diff --git a/.devcontainer/rm1/devcontainer.json b/.devcontainer/rm1/devcontainer.json deleted file mode 100644 index 59e4878e..00000000 --- a/.devcontainer/rm1/devcontainer.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "name": "Edge Node Development Container", - "dockerFile": "../Dockerfile", - - "workspaceMount": "source=${localWorkspaceFolder},target=/edge_node,type=bind,consistency=cached", - "workspaceFolder": "/edge_node", - - "mounts": [ - // Persistent cache - survives container rebuilds -// "source=edge_node_dev_cache,target=/edge_node/_local_cache,type=volume", -// "source=/home/vi/work/ratio1/edge_nodes/edge_node_volumes/devnet/r4_dev/_data,target=/edge_node/_local_cache/,type=volume", - - ], - - "runArgs": [ - // "--gpus=all", // Uncomment for GPU support - "--hostname=rm1", - "--name=rm1", - "--privileged", - "--cgroupns=host", - "--volume=/home/vitalii/remote-dev/projects/RedMesh/.old/edge_nodes_volumes/rm1/_data:/edge_node/_local_cache/", - "--publish=31234:31234", - "--publish=31235:31235", - "--publish=8050:8050", - "--publish=5082:5082" - ], - - "containerEnv": { - "AINODE_DOCKER": "Yes", - "AINODE_DOCKER_SOURCE": "develop", - "EE_ID": "rm1", - "EE_CONFIG": ".config_startup.json", - "EE_ETH_ENABLED": "true", - "EE_EVM_NET": "devnet", - "PYTHONDONTWRITEBYTECODE": "1", - "PYTHONUNBUFFERED": "1" - }, - - // Docker-in-Docker support - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": { - "dockerComposeVersion": "latest" - } - }, - - // Install local packages in editable mode after container creation - "postCreateCommand": "pip install -e ./naeural_core -e ./ratio1_sdk 2>/dev/null || echo 'Local packages not found, using installed versions'; pip install watchdog", - - // Run on every container start - "postStartCommand": "nohup python3 .devcontainer/watch.py > /proc/1/fd/1 2>/proc/1/fd/2 &", - - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "ms-python.debugpy", - "ms-toolsai.jupyter", - "charliermarsh.ruff", - "tamasfe.even-better-toml", - "redhat.vscode-yaml", - "eamodio.gitlens" - ], - "settings": { - "python.defaultInterpreterPath": "/usr/bin/python3", - "python.terminal.activateEnvironment": false, - "editor.formatOnSave": true, - "editor.rulers": [100], - "files.watcherExclude": { - "**/_local_cache/**": true, - "**/__pycache__/**": true, - "**/node_modules/**": true - } - } - } - }, - - "forwardPorts": [5000, 8000, 8080, 9000] -} diff --git a/.devcontainer/watch.py b/.devcontainer/watch.py new file mode 100644 index 00000000..d8d3bf5e --- /dev/null +++ b/.devcontainer/watch.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +Development file watcher for Edge Node hot reload. + +Watches for Python file changes in extensions/ and plugins/ directories, +then automatically restarts the edge node process. + +Usage: + python .devcontainer/dev_watch.py + +Options: + --no-initial Don't start the process immediately, wait for first change + --debounce N Seconds to wait before restarting (default: 1.0) +""" +import subprocess +import sys +import time +import os +import signal +import argparse +from pathlib import Path + +try: + from watchdog.observers import Observer + from watchdog.events import PatternMatchingEventHandler +except ImportError: + print("Installing watchdog...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "watchdog", "-q"]) + from watchdog.observers import Observer + from watchdog.events import PatternMatchingEventHandler + + +class EdgeNodeReloader(PatternMatchingEventHandler): + """Handles file changes and restarts the edge node process.""" + + def __init__(self, debounce_seconds=1.0): + super().__init__( + patterns=["*.py"], + ignore_patterns=["*/__pycache__/*", "*/.git/*", "*/_local_cache/*"], + ignore_directories=True, + case_sensitive=True, + ) + self.process = None + self.last_restart = 0 + self.debounce_seconds = debounce_seconds + self.restart_pending = False + + def start_process(self): + """Start or restart the edge node process.""" + self.stop_process() + + print("\n" + "=" * 60) + print(" Starting edge node...") + print("=" * 60 + "\n") + + self.process = subprocess.Popen( + [sys.executable, "device.py"], + cwd="/edge_node", + preexec_fn=os.setsid, + ) + self.last_restart = time.time() + self.restart_pending = False + + def stop_process(self): + """Stop the running edge node process and all its children.""" + if self.process and self.process.poll() is None: + pgid = os.getpgid(self.process.pid) + print("\n Stopping edge node (PID: {}, PGID: {})...".format(self.process.pid, pgid)) + os.killpg(pgid, signal.SIGTERM) + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + print(" Force killing process group...") + os.killpg(pgid, signal.SIGKILL) + self.process.wait() + print(" Stopped.") + + def _should_restart(self): + """Check if enough time has passed since last restart.""" + return time.time() - self.last_restart >= self.debounce_seconds + + def _trigger_restart(self, event_path): + """Handle a file change event.""" + if not self._should_restart(): + self.restart_pending = True + return + + # Get relative path for cleaner output + try: + rel_path = Path(event_path).relative_to("/edge_node") + except ValueError: + rel_path = event_path + + print("\n File changed: {}".format(rel_path)) + self.start_process() + + def on_modified(self, event): + self._trigger_restart(event.src_path) + + def on_created(self, event): + self._trigger_restart(event.src_path) + + def on_moved(self, event): + self._trigger_restart(event.dest_path) + + def check_pending_restart(self): + """Check and execute pending restart if debounce period passed.""" + if self.restart_pending and self._should_restart(): + print("\n Executing pending restart...") + self.start_process() + + +def main(): + parser = argparse.ArgumentParser(description="Edge Node development watcher") + parser.add_argument("--no-initial", action="store_true", help="Don't start immediately") + parser.add_argument("--debounce", type=float, default=1.0, help="Debounce seconds") + args = parser.parse_args() + + # Directories to watch + watch_dirs = [ + "/edge_node/extensions", + "/edge_node/plugins", + ] + + # Also watch single files + watch_files = [ + "/edge_node/constants.py", + "/edge_node/device.py", + ] + + handler = EdgeNodeReloader(debounce_seconds=args.debounce) + observer = Observer() + + print("\n" + "=" * 60) + print(" Edge Node Development Watcher") + print("=" * 60) + print("\n Watching for changes in:") + + for dir_path in watch_dirs: + path = Path(dir_path) + if path.exists(): + observer.schedule(handler, str(path), recursive=True) + print(" - {}/**/*.py".format(path.name)) + + # Watch parent directory for single files + observer.schedule(handler, "/edge_node", recursive=False) + print(" - constants.py, device.py") + + print("\n Press Ctrl+C to stop.\n") + + # Handle graceful shutdown + def signal_handler(signum, frame): + print("\n\n Shutting down...") + handler.stop_process() + observer.stop() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + observer.start() + + # Start the process initially unless --no-initial + if not args.no_initial: + handler.start_process() + + # Main loop - check for pending restarts + try: + while True: + time.sleep(0.5) + handler.check_pending_restart() + + # Check if process died unexpectedly + if handler.process and handler.process.poll() is not None: + exit_code = handler.process.returncode + if exit_code != 0: + print("\n Process exited with code {}. Waiting for file changes...".format(exit_code)) + handler.process = None + except KeyboardInterrupt: + pass + finally: + handler.stop_process() + observer.stop() + observer.join() + + +if __name__ == "__main__": + main() diff --git a/docs/suricata-security-onion-examples.md b/docs/suricata-security-onion-examples.md new file mode 100644 index 00000000..35aef95c --- /dev/null +++ b/docs/suricata-security-onion-examples.md @@ -0,0 +1,28 @@ +# Suricata Security Onion Correlation Examples + +Use RedMesh lifecycle events as an assessment window when correlating +Suricata alerts in Security Onion. The event payload includes a bounded +time window, authorization context, expected egress metadata, and report +references without exposing target IP values when redaction is enabled. + +Example Security Onion query: + +```text +event.dataset:suricata.eve +AND @timestamp >= window.started_at +AND @timestamp <= window.actual_end_at +AND redmesh.authorization_ref:* +``` + +Useful fields to preserve in analyst notes: + +- `window.started_at` +- `window.actual_end_at` +- `window.grace_seconds` +- `window.clock_skew_seconds` +- `authorization_ref` +- `report_refs.pass_report_cid` + +Treat matches as correlation context for the authorized RedMesh +assessment window. Keep rule tuning and alert handling in the normal SOC +workflow. diff --git a/extensions/business/cybersec/red_mesh/AGENTS.md b/extensions/business/cybersec/red_mesh/AGENTS.md index 256e83d6..33342dbf 100644 --- a/extensions/business/cybersec/red_mesh/AGENTS.md +++ b/extensions/business/cybersec/red_mesh/AGENTS.md @@ -350,3 +350,18 @@ Only append entries for critical or fundamental RedMesh backend changes, discove - Change: added non-blocking SOC event hooks for launcher job-start events, pass completion, finding creation/triage, MISP export status, attestation status, and hard/terminal stop paths. Job/archive/stub models now preserve summary-only `soc_event_status`. - Verification: `python -m pytest extensions/business/cybersec/red_mesh/tests/test_event_lifecycle_hooks.py extensions/business/cybersec/red_mesh/tests/test_misp_export.py extensions/business/cybersec/red_mesh/tests/test_state_machine.py extensions/business/cybersec/red_mesh/tests/test_finalization_aggregation.py extensions/business/cybersec/red_mesh/tests/test_api.py extensions/business/cybersec/red_mesh/tests/test_integration.py extensions/business/cybersec/red_mesh/tests/test_regressions.py extensions/business/cybersec/red_mesh/tests/test_repositories.py -q` passed with 224 tests; `python -m pytest extensions/business/cybersec/red_mesh/tests -q` passed with 1211 tests, 1 skipped, 3 warnings, and 6 subtests. - Horizontal insight: lifecycle hooks should mutate only summary status and call isolated adapters through `services/event_hooks.py`; hook failures must degrade to SOC status/timeline metadata, never scan lifecycle exceptions. + +### 2026-05-13T20:45:39Z + +- Change: hardened OWASP API Top 10 graybox launch/runtime contracts: regular bearer/API-key credentials now flow through the existing encrypted `secret_ref` lane, secret refs fail closed when kind/storage/job ownership is invalid, API-native sessions validate with configured authenticated requests, and flat finding identity includes `scenario_id` plus endpoint evidence. +- Change: API probe families now emit explicit `INFO/inconclusive` findings for missing target inventory, require low-privilege sessions for BOLA/API6 checks, gate higher-risk API4/API8 probes behind operator opt-ins, and treat mutated-but-unverified stateful checks as inconclusive rather than clean. +- Verification: `python -m pytest extensions/business/cybersec/red_mesh/tests/test_secret_isolation.py extensions/business/cybersec/red_mesh/tests/test_api.py extensions/business/cybersec/red_mesh/tests/test_auth.py extensions/business/cybersec/red_mesh/tests/test_target_config.py extensions/business/cybersec/red_mesh/tests/test_graybox_finding.py extensions/business/cybersec/red_mesh/tests/test_stateful_contract.py extensions/business/cybersec/red_mesh/tests/test_probes_api_access.py extensions/business/cybersec/red_mesh/tests/test_probes_api_data.py extensions/business/cybersec/red_mesh/tests/test_probes_api_abuse.py extensions/business/cybersec/red_mesh/tests/test_probes_api_config.py extensions/business/cybersec/red_mesh/tests/test_probes_api_auth.py extensions/business/cybersec/red_mesh/tests/test_finalization_aggregation.py extensions/business/cybersec/red_mesh/tests/test_findings_redaction.py -q` passed with 302 tests and 10 subtests. +- Horizontal insight: API Top 10 graybox coverage is only meaningful when skipped scenarios are reported, low-privilege principals are real, and secret/runtime config boundaries line up from Navigator launch through worker resume and archive/report flattening. + +### 2026-05-14T05:14:40Z + +- Change: closed a secret-ref ownership gap in OWASP API Top 10 graybox worker startup by passing the expected job id explicitly into secret resolution before `JobConfig` coercion can drop non-archived fields. +- Change: moved `PT-OAPI5-03` method-override control traffic fully under the `run_stateful()` gate, so `allow_stateful_probes=false` prevents all mutating requests, and added specific inconclusive-reason callbacks for stateful probes that need attribution-preserving outcomes. +- Change: bounded API4 high-limit probing at an effective request limit of `1000` and changed API3 mass-assignment rollback to fail when the probe introduced a previously absent field instead of writing `false` and claiming rollback success. +- Verification: targeted API/graybox suite passed with `306 passed, 10 subtests`; broad `extensions/business/cybersec/red_mesh/tests -q` run passed `1461` tests and `36` subtests with one unrelated pre-existing failure for missing `docs/suricata-security-onion-examples.md`. +- Horizontal insight: RedMesh stateful probe safety must be enforced at the first target-mutating byte, not only around the vulnerability-attribution request; request-count budgets also need per-request work bounds for resource-consumption probes. diff --git a/extensions/business/cybersec/red_mesh/constants.py b/extensions/business/cybersec/red_mesh/constants.py index df92113e..394af562 100644 --- a/extensions/business/cybersec/red_mesh/constants.py +++ b/extensions/business/cybersec/red_mesh/constants.py @@ -20,6 +20,12 @@ class ScanType(str, Enum): {"key": "_graybox_misconfig", "cls": "misconfig.MisconfigProbes"}, {"key": "_graybox_injection", "cls": "injection.InjectionProbes"}, {"key": "_graybox_business_logic", "cls": "business_logic.BusinessLogicProbes"}, + # OWASP API Top 10 2023 — five themed families (Subphase 1.3). + {"key": "_graybox_api_access", "cls": "api_access.ApiAccessProbes"}, + {"key": "_graybox_api_auth", "cls": "api_auth.ApiAuthProbes"}, + {"key": "_graybox_api_data", "cls": "api_data.ApiDataProbes"}, + {"key": "_graybox_api_config", "cls": "api_config.ApiConfigProbes"}, + {"key": "_graybox_api_abuse", "cls": "api_abuse.ApiAbuseProbes"}, ] # Graybox timing and limits diff --git a/extensions/business/cybersec/red_mesh/docs/adr/2026-05-12-scenario-id-convention.md b/extensions/business/cybersec/red_mesh/docs/adr/2026-05-12-scenario-id-convention.md new file mode 100644 index 00000000..8357e84b --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/adr/2026-05-12-scenario-id-convention.md @@ -0,0 +1,57 @@ +# ADR — Scenario ID convention `PT-OAPI-` for OWASP API Top 10 2023 + +**Status**: Accepted +**Date**: 2026-05-12 +**Context**: Graybox API Top 10 implementation, Subphase 1.0 (see `_todos/2026-05-12-graybox-api-top10-plan-detailed.md`). + +## Decision + +New OWASP API Top 10 (2023) graybox scenarios use the prefix **`PT-OAPI-`** where: + +- `` is the OWASP API category number (1–6, 8, 9 for v1; API7 keeps its legacy ID, API10 is reserved for Phase 9). +- `` is a zero-padded sequence within the category (`01`, `02`, …). + +Examples: `PT-OAPI1-01` (BOLA), `PT-OAPI3-02` (mass assignment), `PT-OAPI5-04` (mutating BFLA), `PT-OAPI9-01` (OpenAPI exposure). + +**Out of scope of this ADR**: any scenario ID for API7 SSRF stays as the existing **`PT-API7-01`** for backward compatibility. Any scenario ID for API10 will be minted in Phase 9, not in v1. + +## Context + +The graybox catalog already uses `PT-A-` for OWASP Web Top 10 2021 scenarios (`PT-A01-01` … `PT-A07-06`). When adding OWASP API Top 10 (2023) coverage we considered several prefixes: + +| Candidate | Pros | Cons | +|---|---|---| +| `PT-API-` | Short, matches OWASP naming directly | One character away from `PT-A0-` — pentesters reading reports under time pressure will misread. `PT-API1-01` vs `PT-A01-01` differ by one character in position 5. | +| `PT-API:2023-` | Year-explicit | Punctuation in ID is hostile to grep, regex, CI test names, JSON keys. | +| `PT-OWASPAPI-` | Fully unambiguous | Long. Inflates inventory tables and PDF columns. | +| **`PT-OAPI-`** | Visually distinct from `PT-A` family. Short. OWASP-API mnemonic. | Slight learning curve (one-time). | + +We chose `PT-OAPI-`. + +## Consequences + +### Affected systems + +1. **Backend catalog** — `extensions/business/cybersec/red_mesh/graybox/scenario_catalog.py` adds 23 new entries under the new prefix (see Subphase 1.2 in the plan). +2. **Inventory regex** — `extensions/business/cybersec/red_mesh/tests/test_detection_inventory.py` widens its scenario-ID matcher to accept `PT-OAPI\d{1,2}-\d+` alongside the existing `PT-A\d+-\d+` and `PT-API7-\d+`. +3. **Frontend (RedMesh-Navigator)** — `lib/domain/knowledge.ts::GRAYBOX_SCENARIOS` registers the new IDs; `OWASP_CATEGORIES` extends to include `API1`–`API9`; a shared `owaspCategoryKey()` helper replaces brittle `owasp_id.slice(0, 3)` usage so `API7:2023` resolves correctly. +4. **PDF report** — `lib/pdf/sections/vulnerabilityAssessment.ts` adds §3.3.3 "OWASP API Top 10" with `owaspCategoryKey(f.owasp_id)?.startsWith('API')` as the dispatch predicate. Legacy `PT-API7-01` MUST appear here. +5. **Operator docs** — `docs/guides/api-security-scanning.md` (Phase 8.6) explains how to read the new IDs. + +### Backward compatibility + +- `PT-A-` (Web Top 10 2021) IDs are unchanged. +- `PT-API7-01` (legacy SSRF) is preserved verbatim — never renamed. Frontend must continue to render it correctly. +- Detection-inventory floor counters are bumped by +23 (graybox floor 80 → ≥103) in Subphase 1.2. + +### Non-decisions (out of scope of this ADR) + +- Whether to deprecate the legacy `PT-A02-12` once `PT-OAPI2-01` is stable. Tracked as Phase 9 F12. +- ATT&CK / CWE / compliance-framework mapping schemes; tracked separately (Subphase 1.2 and Phase 9 F13). +- Whether `PT-API7-01` should eventually be renamed to `PT-OAPI7-01` for consistency. Not in v1; revisit when there is a separate need to migrate the legacy probe. + +## References + +- Plan: `_todos/2026-05-12-graybox-api-top10-plan-detailed.md` (Subphase 1.0, lines 253–280; Subphase 1.2 ID table, lines 315–329). +- OWASP API Security Top 10 2023: https://owasp.org/API-Security/editions/2023/en/0x11-t10/ +- Existing OWASP Web Top 10 2021 scenarios: `extensions/business/cybersec/red_mesh/graybox/scenario_catalog.py`. diff --git a/extensions/business/cybersec/red_mesh/docs/api-security-target-config.md b/extensions/business/cybersec/red_mesh/docs/api-security-target-config.md new file mode 100644 index 00000000..fe6404e1 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/docs/api-security-target-config.md @@ -0,0 +1,192 @@ +# `target_config.api_security` JSON shape + +This is the operator-facing reference for the OWASP API Top 10 graybox scan +configuration. Pass it inside `target_config` on `launch_webapp_scan`. + +Source of truth: `graybox/models/target_config.py` (Subphase 1.1 of the API +Top 10 plan). Scenario IDs live in `graybox/scenario_catalog.py` and the +ADR at `docs/adr/2026-05-12-scenario-id-convention.md`. + +API10 ("Unsafe Consumption of APIs") is **deliberately not present** in v1 +— it is scheduled for Phase 9 once a callback-receiver service exists. + +## Top-level shape + +```json +{ + "target_config": { + "api_security": { + "object_endpoints": [ApiObjectEndpoint], + "property_endpoints": [ApiPropertyEndpoint], + "function_endpoints": [ApiFunctionEndpoint], + "resource_endpoints": [ApiResourceEndpoint], + "business_flows": [ApiBusinessFlow], + "token_endpoints": ApiTokenEndpoint, + "inventory_paths": ApiInventoryPaths, + "ssrf_body_fields": ["url", "webhook", "callback", "image_url", "redirect_uri"], + "sensitive_field_patterns": [], + "tampering_fields": ["is_admin", "is_superuser", "role", "verified", ...], + "debug_path_candidates": ["/debug", "/api/debug", "/api/_routes", ...] + } + } +} +``` + +## Endpoint sub-models + +### `ApiObjectEndpoint` — drives **PT-OAPI1-01** (BOLA) + +```json +{ + "path": "/api/records/{id}/", + "test_ids": [1, 2], + "owner_field": "owner", + "id_param": "id", + "tenant_field": "" +} +``` + +Only `path` is required. Set `tenant_field` for cross-tenant BOLA. + +### `ApiPropertyEndpoint` — drives **PT-OAPI3-01** (excessive exposure) and **PT-OAPI3-02** (mass assignment, stateful) + +```json +{ + "path": "/api/profile/{id}/", + "method_read": "GET", + "method_write": "PATCH", + "test_id": 1, + "id_param": "id" +} +``` + +### `ApiFunctionEndpoint` — drives **PT-OAPI5-01..04** (BFLA) + +```json +{ + "path": "/api/admin/users/{uid}/promote/", + "method": "POST", + "privilege": "admin", + "auth_required_marker": "", + "revert_path": "/api/admin/users/{uid}/demote/", + "revert_body": {"reason": "test"} +} +``` + +`revert_path` is **mandatory** when `method != "GET"` and you want +PT-OAPI5-03 / PT-OAPI5-04 to run with `allow_stateful_probes=true`. +Without it, the stateful probe emits `inconclusive`. + +### `ApiResourceEndpoint` — drives **PT-OAPI4-01..03** + +```json +{ + "path": "/api/records/list/", + "limit_param": "limit", + "baseline_limit": 10, + "abuse_limit": 999999, + "rate_limit_expected": false +} +``` + +Set `rate_limit_expected=true` only on endpoints that genuinely should be +rate-limited — otherwise PT-OAPI4-03 will produce noisy false positives. + +### `ApiBusinessFlow` — drives **PT-OAPI6-01..02** (stateful) + +```json +{ + "path": "/api/auth/signup/", + "method": "POST", + "flow_name": "signup", + "body_template": {"username": "x", "email": "x@x"}, + "verify_path": "/api/users/?username=", + "test_account": "abuse_canary", + "captcha_marker": "", + "mfa_marker": "" +} +``` + +Requires `allow_stateful_probes=true` and a tester-supplied non-privileged +`test_account`. Hard-capped at N=5 attempts per flow. + +### `ApiTokenEndpoint` — drives **PT-OAPI2-01..03** + +```json +{ + "token_path": "/api/token/", + "protected_path": "/api/me/", + "logout_path": "/api/auth/logout/", + "weak_secret_candidates": ["secret", "changeme", "password", ...] +} +``` + +`logout_path` is required for **PT-OAPI2-03** (logout-doesn't-invalidate); +without it, only PT-OAPI2-01 and PT-OAPI2-02 fire. + +### `ApiInventoryPaths` — drives **PT-OAPI9-01..03** + +```json +{ + "openapi_candidates": ["/openapi.json", "/swagger.json", "/v3/api-docs", ...], + "current_version": "/api/v2/", + "version_sibling_candidates": ["/api/v1/", "/api/v0/", "/api/beta/", ...], + "canonical_probe_path": "/api/v2/records/1/", + "private_path_patterns": ["/internal/", "/admin/"], + "deprecated_paths": ["/api/v1/legacy/"] +} +``` + +`canonical_probe_path` should be a known-existing endpoint under +`current_version`; PT-OAPI9-02 cross-checks each sibling version by hitting +the same path under it. + +## Cross-cutting fields + +- **`ssrf_body_fields`**: extends PT-API7-01 to scan JSON body fields by name. +- **`sensitive_field_patterns`**: appended to the built-in regex list used by + PT-OAPI3-01. +- **`tampering_fields`**: property names PT-OAPI3-02 attempts to set via mass + assignment. +- **`debug_path_candidates`**: paths PT-OAPI8-03 probes for debug exposure. + +## Notes on auth + budget (forward references) + +- **Bearer / API-key auth descriptors** (`api_security.auth`) land in + Subphase 1.5. Secret values (`bearer_token`, `api_key`) are top-level + launch parameters, **not** inside `target_config`. +- **`max_total_requests`** lands in Subphase 1.7 as a per-scan request + budget cap. + +## Minimal example + +```json +{ + "target_url": "https://api.example.com", + "official_username": "admin", + "official_password": "...", + "regular_username": "alice", + "regular_password": "...", + "target_config": { + "api_security": { + "object_endpoints": [ + {"path": "/api/records/{id}/", "test_ids": [42, 43], + "tenant_field": "tenant_id"} + ], + "function_endpoints": [ + {"path": "/api/admin/export/", "method": "GET"} + ], + "token_endpoints": { + "token_path": "/api/token/", + "protected_path": "/api/me/", + "logout_path": "/api/auth/logout/" + }, + "inventory_paths": { + "current_version": "/api/v2/", + "canonical_probe_path": "/api/v2/health" + } + } + }, + "allow_stateful_probes": false +} +``` diff --git a/extensions/business/cybersec/red_mesh/graybox/auth.py b/extensions/business/cybersec/red_mesh/graybox/auth.py index 9a502572..c072b9de 100644 --- a/extensions/business/cybersec/red_mesh/graybox/auth.py +++ b/extensions/business/cybersec/red_mesh/graybox/auth.py @@ -1,8 +1,18 @@ """ Authentication manager for graybox scanning. -Handles CSRF auto-detection, login with robust success detection, -session expiry, re-auth, and cleanup. +Orchestrates `AuthStrategy` instances to establish authenticated sessions +for one or more principals (`official`, `regular`). The strategy itself +owns the protocol-level details (form login, Bearer header injection, +API-key placement); the manager owns the lifecycle (expiry, retry, +multi-principal coordination, cleanup). + +For backward compatibility this module continues to expose `AuthManager` +with the same public API used by `graybox/worker.py` and by tests that +patch `extensions...graybox.auth.requests`. Internally it delegates to +`auth_strategies.FormAuth` for the legacy form-login flow; later +subphases route to `BearerAuth` / `ApiKeyAuth` based on +`target_config.api_security.auth.auth_type`. """ import re @@ -11,6 +21,8 @@ import requests from ..constants import GRAYBOX_SESSION_MAX_AGE +from .auth_credentials import Credentials +from .auth_strategies import ApiKeyAuth, BearerAuth, FormAuth from .models.target_config import COMMON_CSRF_FIELDS from .models import GrayboxAuthState @@ -26,10 +38,11 @@ class AuthManager: session expiry, re-auth, and cleanup. """ - def __init__(self, target_url, target_config, verify_tls=True): + def __init__(self, target_url, target_config, verify_tls=True, http_client=None): self.target_url = target_url.rstrip("/") self.target_config = target_config self.verify_tls = verify_tls + self.http_client = http_client self.anon_session = None self.official_session = None @@ -69,8 +82,8 @@ def needs_refresh(self, require_regular=False) -> bool: def ensure_sessions(self, official_creds, regular_creds=None): """Re-authenticate if sessions are stale or not yet created.""" - regular_creds = self._coerce_creds(regular_creds) - require_regular = bool(regular_creds and regular_creds.get("username")) + regular_creds = self._coerce_creds(regular_creds, principal="regular") + require_regular = self._credentials_configured(regular_creds) if not self.needs_refresh(require_regular=require_regular): return True self.cleanup() @@ -83,23 +96,21 @@ def ensure_sessions(self, official_creds, regular_creds=None): def authenticate(self, official_creds, regular_creds=None): """Create fresh sessions for all configured users.""" self.anon_session = self._make_session() - official_creds = self._coerce_creds(official_creds) - regular_creds = self._coerce_creds(regular_creds) + official_creds = self._coerce_creds(official_creds, principal="official") + regular_creds = self._coerce_creds(regular_creds, principal="regular") self._auth_errors = [] self.official_session = self._try_login_with_retry( "official", - official_creds["username"], - official_creds["password"], + official_creds, ) if not self.official_session: return False - if regular_creds and regular_creds.get("username"): + if self._credentials_configured(regular_creds): self.regular_session = self._try_login_with_retry( "regular", - regular_creds["username"], - regular_creds["password"], + regular_creds, ) if not self.regular_session: self._record_auth_error("regular_login_failed") @@ -108,18 +119,41 @@ def authenticate(self, official_creds, regular_creds=None): return True @staticmethod - def _coerce_creds(creds): + def _coerce_creds(creds, principal="official"): if creds is None: return None + if isinstance(creds, Credentials): + creds.principal = creds.principal or principal + return creds + if hasattr(creds, "to_credentials") and callable(creds.to_credentials): + return creds.to_credentials() if isinstance(creds, dict): - return { - "username": creds.get("username", ""), - "password": creds.get("password", ""), - } - return { - "username": getattr(creds, "username", "") or "", - "password": getattr(creds, "password", "") or "", - } + return Credentials( + username=creds.get("username", "") or "", + password=creds.get("password", "") or "", + bearer_token=creds.get("bearer_token", "") or "", + bearer_refresh_token=creds.get("bearer_refresh_token", "") or "", + api_key=creds.get("api_key", "") or "", + principal=creds.get("principal", principal) or principal, + ) + return Credentials( + username=getattr(creds, "username", "") or "", + password=getattr(creds, "password", "") or "", + bearer_token=getattr(creds, "bearer_token", "") or "", + bearer_refresh_token=getattr(creds, "bearer_refresh_token", "") or "", + api_key=getattr(creds, "api_key", "") or "", + principal=getattr(creds, "principal", principal) or principal, + ) + + @staticmethod + def _credentials_configured(creds) -> bool: + if creds is None: + return False + return bool( + creds.has_form_credentials() + or creds.has_bearer_token() + or creds.has_api_key() + ) def cleanup(self): """ @@ -127,12 +161,13 @@ def cleanup(self): Prevents session accumulation on targets with session limits. """ - logout_url = self.target_url + self.target_config.logout_path + logout_url = self._logout_url_for_current_auth() for session in [self.official_session, self.regular_session]: if session is None: continue try: - session.get(logout_url, timeout=5) + if logout_url: + session.get(logout_url, timeout=5) except requests.RequestException: pass finally: @@ -145,36 +180,19 @@ def cleanup(self): self._created_at = 0.0 def preflight_check(self) -> str | None: - """ - Verify target reachability and login page existence. + """Delegate preflight to the configured auth strategy. - Returns error message if preflight fails, None if OK. + Strategy chooses its own preflight semantics — FormAuth requires the + login_path to exist; BearerAuth / ApiKeyAuth (Subphase 1.5 #5-#7) + instead hit a configured authenticated endpoint. """ - # 1. Target reachable? - try: - requests.head( - self.target_url, - timeout=10, - verify=self.verify_tls, - allow_redirects=True, - ) - except requests.RequestException as exc: - return f"Target unreachable: {exc}" - - # 2. Login page exists? - login_url = self.target_url + self.target_config.login_path - try: - resp = requests.get(login_url, timeout=10, verify=self.verify_tls) - if resp.status_code == 404: - return f"Login page not found: {login_url} returned 404" - except requests.RequestException as exc: - return f"Login page unreachable: {exc}" - - return None + return self._build_strategy().preflight() def _make_session(self): s = requests.Session() s.verify = self.verify_tls + if self.http_client is not None: + return self.http_client.wrap_session(s) return s def make_anonymous_session(self): @@ -197,10 +215,10 @@ def try_credentials(self, username, password): def _record_auth_error(self, code): self._auth_errors.append(code) - def _try_login_with_retry(self, principal, username, password): + def _try_login_with_retry(self, principal, creds): retryable_failure = False for attempt in range(1, self.MAX_AUTH_ATTEMPTS + 1): - session, retryable_failure = self._try_login_attempt(username, password) + session, retryable_failure = self._try_login_attempt(creds) if session is not None: return session if not retryable_failure: @@ -218,139 +236,288 @@ def _try_login(self, username, password): """ Attempt login with CSRF auto-detection and robust success detection. """ - session, _ = self._try_login_attempt(username, password) + session, _ = self._try_login_attempt(Credentials(username=username, password=password)) return session - def _try_login_attempt(self, username, password): - """ - Attempt one login and classify whether failure is retryable. - """ - session = self._make_session() - login_url = self.target_url + self.target_config.login_path + def _try_login_attempt(self, creds): + """Attempt one login via the configured strategy. - # GET login page + Returns ``(session, retryable_failure)``. Transport errors raised by + the strategy are translated into ``retryable_failure=True``; auth-level + failures into ``retryable_failure=False``. + """ + strategy = self._build_strategy() try: - resp = session.get(login_url, timeout=10, allow_redirects=True) + session = strategy.authenticate(creds) except requests.RequestException: - session.close() + # Even on transport errors, the strategy may have already seen the + # login page and detected the CSRF field — preserve it. + if strategy.last_detected_csrf_field: + self._detected_csrf_field = strategy.last_detected_csrf_field return None, True + # Always propagate whatever CSRF field the strategy saw, regardless + # of whether the credential check ultimately succeeded. + if strategy.last_detected_csrf_field: + self._detected_csrf_field = strategy.last_detected_csrf_field + if session is not None: + valid, retryable_failure = self._validate_authenticated_session(session) + if not valid: + try: + session.close() + except Exception: + pass + return None, retryable_failure + return session, False + return None, False - # Auto-detect or use configured CSRF field - csrf_field, csrf_token = self._extract_csrf(resp.text) + def _authenticated_probe_path(self) -> str: + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return "" + auth_desc = getattr(api_security, "auth", None) + if auth_desc is None: + return "" + return (getattr(auth_desc, "authenticated_probe_path", "") or "").strip() + + def _authenticated_probe_method(self) -> str: + api_security = getattr(self.target_config, "api_security", None) + auth_desc = getattr(api_security, "auth", None) if api_security is not None else None + method = (getattr(auth_desc, "authenticated_probe_method", "GET") or "GET").upper() + allow_non_readonly = bool( + getattr(auth_desc, "allow_non_readonly_auth_validation_method", False) + ) + if method in ("GET", "HEAD"): + return method + if allow_non_readonly and method in ("POST", "OPTIONS"): + return method + return "GET" + + def _logout_url_for_current_auth(self) -> str: + if self._resolve_auth_type() == "form": + path = getattr(self.target_config, "logout_path", "") or "" + else: + api_security = getattr(self.target_config, "api_security", None) + auth_desc = getattr(api_security, "auth", None) if api_security is not None else None + path = getattr(auth_desc, "api_logout_path", "") or "" + return self.target_url + path if path else "" + + def _auth_descriptor(self): + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return None + return getattr(api_security, "auth", None) + + def _configured_success_statuses(self) -> tuple[int, ...]: + auth_desc = self._auth_descriptor() + if auth_desc is None: + return () + return tuple(getattr(auth_desc, "authenticated_probe_success_statuses", ()) or ()) + + def _configured_success_marker(self) -> str: + auth_desc = self._auth_descriptor() + if auth_desc is None: + return "" + return str(getattr(auth_desc, "authenticated_probe_success_marker", "") or "") + + def _configured_identity_json_path(self) -> str: + auth_desc = self._auth_descriptor() + if auth_desc is None: + return "" + return str(getattr(auth_desc, "authenticated_probe_identity_json_path", "") or "") - payload = { - self.target_config.username_field: username, - self.target_config.password_field: password, - } - headers = {"Referer": login_url} - if csrf_token and csrf_field: - payload[csrf_field] = csrf_token - headers["X-CSRFToken"] = csrf_token + @staticmethod + def _traverse_identity_path(payload, path: str): + """Safely walk a dotted JSON path. + Only supports nested dict key lookups (no array indexing, no + expressions). Returns None if any segment is missing. The whole + traversal is bounded by the depth of the configured path so it + cannot be turned into an arbitrary-expression evaluator. + """ + if not path or not isinstance(payload, dict): + return None + cursor = payload + for segment in path.split("."): + segment = segment.strip() + if not segment or not isinstance(cursor, dict): + return None + if segment not in cursor: + return None + cursor = cursor[segment] + return cursor + + def _anonymous_control_response(self, method: str, url: str): + """Send the same probe request without credentials, no redirects.""" try: - resp = session.post( - login_url, data=payload, headers=headers, - timeout=10, allow_redirects=True, - ) + session = requests.Session() + try: + return session.request( + method, url, timeout=10, allow_redirects=False, verify=self.verify_tls, + ) + finally: + try: + session.close() + except Exception: + pass except requests.RequestException: - session.close() - return None, True - - # Robust success detection - if self._is_login_success(resp, session, login_url): - return session, False - - session.close() - return None, False + return None - def _is_login_success(self, response, session, login_url): - """ - Determine if login succeeded. - - Checks (in order): - 1. HTTP error -> fail - 2. Response body contains failure markers -> fail - 3. JSON error responses -> fail - 4. Redirected away from login page AND cookies present -> success - 5. Non-empty session cookies -> success + def _validate_authenticated_session(self, session) -> tuple[bool, bool]: + """Validate token/key sessions after credentials have been attached. + + Tightened in B2 (PR406 remediation): we no longer follow redirects + or accept any <400 status, since an invalid bearer token frequently + triggers a 302 to a public 200 login page. The flow is: + + 1. Send the probe with allow_redirects=False; reject 3xx/401/403. + 2. Send an anonymous control request to the same path. If the + control is also 2xx, require an explicit success assertion + (status allow-list, marker, or identity JSON path) before + accepting — otherwise the path is effectively public and the + configured token tells us nothing. + 3. If a marker / identity path is configured, both the + authenticated AND anonymous responses must agree with the + assertion (marker present in authenticated body but missing + from anonymous; identity path non-empty when authenticated and + empty/missing when anonymous). """ - if response.status_code >= 400: - return False + if self._resolve_auth_type() == "form": + return True, False + probe_path = self._authenticated_probe_path() + if not probe_path: + return True, False + method = self._authenticated_probe_method().lower() + probe_url = self.target_url + probe_path + try: + req = getattr(session, method, session.get) + resp = req(probe_url, timeout=10, allow_redirects=False) + except requests.RequestException: + return False, True + status = getattr(resp, "status_code", None) + if status is None: + return False, False + # Reject redirects (commonly mask invalid tokens) and explicit + # authentication failures. + if 300 <= status < 400: + return False, False + if status in (401, 403): + return False, False + if status >= 400: + return False, False + + success_statuses = self._configured_success_statuses() + if success_statuses and status not in success_statuses: + return False, False + + marker = self._configured_success_marker() + identity_path = self._configured_identity_json_path() + requires_assertion = bool(marker or identity_path) + + auth_body = self._read_response_body(resp) + auth_json = self._read_response_json(resp, auth_body) + + if requires_assertion: + if marker and marker not in auth_body: + return False, False + if identity_path: + value = self._traverse_identity_path(auth_json, identity_path) + if not value: + return False, False + + control = self._anonymous_control_response(method.upper(), probe_url) + control_status = getattr(control, "status_code", None) if control is not None else None + control_is_success = ( + control_status is not None and 200 <= control_status < 300 + ) - # Check for failure markers in response body. - # Use multi-word phrases to avoid false matches — single words like - # "failed" can appear in legitimate post-login content. - failure_markers = [ - "invalid credentials", "invalid username", "invalid password", - "incorrect password", "login failed", "authentication failed", - "try again", "wrong password", "unable to log in", - "account locked", "account disabled", - ] - body_lower = response.text.lower() - if any(marker in body_lower for marker in failure_markers): - return False + if not control_is_success: + # Anonymous request was rejected (or transport failed) — the + # authenticated 2xx is a meaningful delta. Accept without + # requiring a marker. + return True, False + + # Anonymous request also got 2xx. The endpoint may be public; we + # need an assertion that distinguishes the two responses. + if not requires_assertion: + return False, False + control_body = self._read_response_body(control) + control_json = self._read_response_json(control, control_body) + if marker and marker in control_body: + return False, False + if identity_path: + anon_value = self._traverse_identity_path(control_json, identity_path) + if anon_value: + return False, False + return True, False - # SPA support: check JSON error responses - ct = response.headers.get("content-type", "") - if "application/json" in ct: + @staticmethod + def _read_response_body(resp) -> str: + if resp is None: + return "" + text = getattr(resp, "text", None) + if isinstance(text, str): + return text + content = getattr(resp, "content", b"") or b"" + if isinstance(content, (bytes, bytearray)): try: - data = response.json() - if isinstance(data, dict): - if data.get("error") or data.get("success") is False or data.get("authenticated") is False: - return False - except ValueError: - pass - - has_cookies = bool(session.cookies.get_dict()) - - # Redirect away from login URL — require cookies to confirm - # session was actually established. - if response.url and "login" not in response.url.lower(): - if has_cookies: - return True + return content.decode("utf-8", errors="replace") + except Exception: + return "" + return str(content) - # Redirect chain present and final URL differs AND cookies set - if response.history and login_url not in response.url: - if has_cookies: - return True + @staticmethod + def _read_response_json(resp, body_text: str): + if resp is None: + return None + json_fn = getattr(resp, "json", None) + if callable(json_fn): + try: + return json_fn() + except Exception: + pass + if not body_text: + return None + try: + import json as _json + return _json.loads(body_text) + except Exception: + return None - # Has auth-relevant cookies (even without redirect — SPA logins) - return has_cookies + def _resolve_auth_type(self) -> str: + """Return the configured auth_type, defaulting to ``form``. - def _extract_csrf(self, html): + Targets that don't populate ``target_config.api_security.auth`` + (everything pre-API-Top-10) keep ``form`` and behave identically + to before the refactor. """ - Extract CSRF token from HTML. - - If csrf_field is configured, use it directly. - Otherwise, try common framework field names. - Returns (field_name, token_value) tuple. + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return "form" + auth_desc = getattr(api_security, "auth", None) + if auth_desc is None: + return "form" + return getattr(auth_desc, "auth_type", "form") or "form" + + def _build_strategy(self): + """Construct the auth strategy for this manager based on auth_type. + + ``form`` → FormAuth (existing form-login) + ``bearer`` → BearerAuth (Subphase 1.5 commit #6) + ``api_key``→ ApiKeyAuth (Subphase 1.5 commit #7) """ - if self.target_config.csrf_field: - token = self._find_csrf_value(html, self.target_config.csrf_field) - return (self.target_config.csrf_field, token) - - # Auto-detect: try common CSRF field names - if self._detected_csrf_field: - token = self._find_csrf_value(html, self._detected_csrf_field) - if token: - return (self._detected_csrf_field, token) - - for field_name in COMMON_CSRF_FIELDS: - token = self._find_csrf_value(html, field_name) - if token: - self._detected_csrf_field = field_name - return (field_name, token) - - # Fallback: any hidden input with "csrf" or "token" in name - m = re.search( - r']+type=["\']hidden["\'][^>]+name=["\']([^"\']*(?:csrf|token)[^"\']*)["\'][^>]+value=["\']([^"\']+)', - html or "", re.IGNORECASE, - ) - if m: - self._detected_csrf_field = m.group(1) - return (m.group(1), m.group(2)) - - return (None, None) + auth_type = self._resolve_auth_type() + if auth_type == "form": + return FormAuth(self.target_url, self.target_config, self.verify_tls, self.http_client) + if auth_type == "bearer": + return BearerAuth(self.target_url, self.target_config, self.verify_tls, self.http_client) + if auth_type == "api_key": + return ApiKeyAuth(self.target_url, self.target_config, self.verify_tls, self.http_client) + raise ValueError(f"Unknown auth_type: {auth_type!r}") + + # Form-login internals (``_is_login_success``, ``_extract_csrf``, + # ``_find_csrf_value``) moved into ``auth_strategies.FormAuth`` in + # Subphase 1.5 commit #3. ``extract_csrf_value`` remains a public + # static helper so existing probe-side callers keep working. @staticmethod def extract_csrf_value(html, field_name): @@ -359,21 +526,4 @@ def extract_csrf_value(html, field_name): Used by probes that need to include CSRF tokens in form submissions. """ - return AuthManager._find_csrf_value(html, field_name) - - @staticmethod - def _find_csrf_value(html, field_name): - """Find value of a named hidden input field.""" - # Try name->value order - m = re.search( - rf'name=["\']?{re.escape(field_name)}["\']?\s[^>]*value=["\']([^"\']+)', - html or "", re.IGNORECASE, - ) - if m: - return m.group(1) - # Try value->name order (some frameworks emit attrs differently) - m = re.search( - rf'value=["\']([^"\']+)["\'][^>]*name=["\']?{re.escape(field_name)}["\']?', - html or "", re.IGNORECASE, - ) - return m.group(1) if m else None + return FormAuth._find_csrf_value(html, field_name) diff --git a/extensions/business/cybersec/red_mesh/graybox/auth_credentials.py b/extensions/business/cybersec/red_mesh/graybox/auth_credentials.py new file mode 100644 index 00000000..5266fb47 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/auth_credentials.py @@ -0,0 +1,86 @@ +"""Mutable credential value object for graybox auth strategies. + +`Credentials` holds the secret material a single `AuthStrategy` needs to +authenticate against a target. The orchestrator (`AuthManager`) hands it +to the strategy at ``authenticate()`` time; strategies retain a reference +only for the active session lifetime and call ``clear()`` on cleanup. + +Critically: +- This class never appears in persisted JobConfig payloads. Secrets travel + from the launch API into the R1FS secret payload via + ``services/secrets.py::persist_job_config_with_secrets`` (Subphase 1.5 + commit #8). At worker startup the secrets are resolved out of the secret + payload and packed into a `Credentials` instance. +- ``clear()`` overwrites each field with empty strings so accidental + references (logs, repr, post-hoc serialisation) cannot leak token values. +- ``__repr__`` is overridden to never include secret values. + +Mutable on purpose — `dataclass(frozen=True)` was considered but `clear()` +needs to overwrite fields. The class is treated as conceptually +write-once-then-clear; do not mutate it outside the auth layer. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class Credentials: + """Per-strategy credential bundle. + + Fields are union-typed by strategy: + FormAuth — uses ``username`` + ``password``. + BearerAuth — uses ``bearer_token`` (+ optional ``bearer_refresh_token``). + ApiKeyAuth — uses ``api_key``. + + Strategies must not write back into this object; only the orchestrator + populates it. Strategies may, however, call ``clear()`` on cleanup. + """ + username: str = "" + password: str = "" + bearer_token: str = "" + bearer_refresh_token: str = "" + api_key: str = "" + + # Optional principal label for diagnostics ("official", "regular", ...). + principal: str = "official" + + # Static empty-string marker used by clear(). Defined as a class attribute + # to avoid importing typing.Final each time. + _CLEARED = "" + + def has_form_credentials(self) -> bool: + return bool(self.username) and bool(self.password) + + def has_bearer_token(self) -> bool: + return bool(self.bearer_token) + + def has_api_key(self) -> bool: + return bool(self.api_key) + + def clear(self) -> None: + """Overwrite every credential field. Idempotent. + + Note: Python strings are immutable, so ``clear()`` does not truly + zeroise memory the way a buffer .fill(0) would. We rely instead on + GC + the limited scope of the Credentials object. The point of this + method is to ensure code that re-reads the object (after cleanup) + sees empty values, not historical secrets. + """ + self.username = self._CLEARED + self.password = self._CLEARED + self.bearer_token = self._CLEARED + self.bearer_refresh_token = self._CLEARED + self.api_key = self._CLEARED + + def __repr__(self) -> str: + """Never include secret values in repr() (Subphase 1.5 secret-handling).""" + return ( + "Credentials(" + f"principal={self.principal!r}, " + f"has_form_credentials={self.has_form_credentials()}, " + f"has_bearer_token={self.has_bearer_token()}, " + f"has_api_key={self.has_api_key()}" + ")" + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/auth_strategies.py b/extensions/business/cybersec/red_mesh/graybox/auth_strategies.py new file mode 100644 index 00000000..77b90164 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/auth_strategies.py @@ -0,0 +1,460 @@ +"""Auth strategy pattern for graybox session establishment. + +Defines the `AuthStrategy` ABC and concrete strategies used by the +`AuthManager` orchestrator. Each strategy returns a fully-authenticated +`requests.Session` ready for probe families to use. + +Strategy implementations are introduced incrementally across Subphase 1.5: + 1.5 commit #1 — AuthStrategy ABC + FormAuth. + 1.5 commit #6 — BearerAuth. + 1.5 commit #7 — ApiKeyAuth. + 1.5 commit #3 — `AuthManager` is wired to dispatch to a strategy + selected from `target_config.api_security.auth.auth_type`. + +`Credentials` (Subphase 1.5 commit #2) is the value object the +orchestrator hands to each strategy at `authenticate()` time. Strategies +must NOT capture credentials beyond the active session lifetime — call +``creds.clear()`` on ``cleanup()`` when they own the secret material. +""" + +from __future__ import annotations + +import re +from abc import ABC, abstractmethod +from typing import Optional + +import requests + + +class AuthStrategy(ABC): + """Abstract base class for graybox auth strategies. + + Concrete strategies live alongside this module (`auth_strategies.py`). + They are stateless apart from the active ``requests.Session`` and any + short-lived collaborator references (target URL, verify_tls flag). + + Lifecycle: + 1. ``preflight()`` validates the target is reachable / configured in + the strategy-appropriate way. Returns an error string on failure, + None on success. + 2. ``authenticate(creds)`` returns an authenticated ``Session`` or + None on failure. The orchestrator (`AuthManager`) is responsible + for retries and recording errors. + 3. ``refresh(creds)`` re-establishes the authenticated state when a + session ages out. Returns True on success. + 4. ``cleanup()`` closes the session and zeroises any captured + credential material owned by the strategy. + + Strategies should be cheap to instantiate; the orchestrator may create + multiple instances (for example to hold ``official`` and ``regular`` + sessions concurrently). + """ + + def __init__(self, target_url: str, target_config, verify_tls: bool = True, + http_client=None): + self.target_url = target_url.rstrip("/") + self.target_config = target_config + self.verify_tls = verify_tls + self.http_client = http_client + self._session: Optional[requests.Session] = None + # Strategies may expose protocol-specific diagnostic state here; the + # orchestrator copies it back into AuthManager so probe callers keep + # using the legacy public API. + self.last_detected_csrf_field: Optional[str] = None + + def make_session(self) -> requests.Session: + """Create a fresh, unauthenticated ``requests.Session`` honouring TLS verify.""" + s = requests.Session() + s.verify = self.verify_tls + if self.http_client is not None: + return self.http_client.wrap_session(s) + return s + + @property + def session(self) -> Optional[requests.Session]: + return self._session + + @abstractmethod + def preflight(self) -> Optional[str]: + """Return an error string if preflight fails, None if OK.""" + ... + + @abstractmethod + def authenticate(self, creds) -> Optional[requests.Session]: + """Return an authenticated session or None on failure.""" + ... + + def refresh(self, creds) -> bool: + """Default refresh = re-authenticate. Strategies may override.""" + self.cleanup() + sess = self.authenticate(creds) + return sess is not None + + def cleanup(self) -> None: + """Close the session if owned. Strategies that hold secret material + in addition to the session should override and zeroise it via + ``creds.clear()``. + """ + if self._session is not None: + try: + self._session.close() + except Exception: + pass + self._session = None + + +# Common CSRF field names across frameworks (mirrors COMMON_CSRF_FIELDS in +# target_config.py — kept independent here so the strategy module has no +# upstream dependency on the typed-config package layout). +_FORM_AUTH_CSRF_FIELDS = ( + "csrfmiddlewaretoken", # Django + "csrf_token", # Flask / WTForms + "authenticity_token", # Rails + "_csrf", # Spring Security + "_token", # Laravel +) + +_FORM_AUTH_FAILURE_MARKERS = ( + "invalid credentials", "invalid username", "invalid password", + "incorrect password", "login failed", "authentication failed", + "try again", "wrong password", "unable to log in", + "account locked", "account disabled", +) + + +class FormAuth(AuthStrategy): + """Cookie-session login via HTML form (existing legacy behaviour). + + Wraps the form-login logic that previously lived inline in + `AuthManager._try_login_attempt`. The behaviour and heuristics are + identical — see Subphase 1.5 commit #3 for the wiring into the + orchestrator. + + Public methods: + - ``preflight()`` — verifies target reachability AND that the login + page exists at ``target_config.login_path`` (not 404). + - ``authenticate(creds)`` — GETs the login page, auto-detects the + CSRF field, POSTs ``username``/``password`` from ``creds``, and + heuristically confirms success. + """ + + def preflight(self) -> Optional[str]: + if self.http_client is not None: + session = self.make_session() + try: + session.head(self.target_url, timeout=10, allow_redirects=True) + login_url = self.target_url + self.target_config.login_path + resp = session.get(login_url, timeout=10) + if resp.status_code == 404: + return f"Login page not found: {login_url} returned 404" + except requests.RequestException as exc: + return f"Login page unreachable: {exc}" + finally: + session.close() + return None + + # 1. Target reachable? + try: + requests.head( + self.target_url, + timeout=10, + verify=self.verify_tls, + allow_redirects=True, + ) + except requests.RequestException as exc: + return f"Target unreachable: {exc}" + + # 2. Login page exists? + login_url = self.target_url + self.target_config.login_path + try: + resp = requests.get(login_url, timeout=10, verify=self.verify_tls) + if resp.status_code == 404: + return f"Login page not found: {login_url} returned 404" + except requests.RequestException as exc: + return f"Login page unreachable: {exc}" + + return None + + def authenticate(self, creds) -> Optional[requests.Session]: + """Return an authenticated session or None on auth-level failure. + + Transport errors (``requests.RequestException``) bubble up so the + orchestrator can distinguish retryable transport failures from + definitive credential failures. + """ + session = self.make_session() + login_url = self.target_url + self.target_config.login_path + + # GET login page — transport errors bubble up (retryable). + try: + resp = session.get(login_url, timeout=10, allow_redirects=True) + except requests.RequestException: + session.close() + raise + + # Auto-detect or use configured CSRF field + csrf_field, csrf_token = self._extract_csrf(resp.text) + if csrf_field: + self.last_detected_csrf_field = csrf_field + + payload = { + self.target_config.username_field: creds.username, + self.target_config.password_field: creds.password, + } + headers = {"Referer": login_url} + if csrf_token and csrf_field: + payload[csrf_field] = csrf_token + headers["X-CSRFToken"] = csrf_token + + # POST credentials — transport errors bubble up (retryable). + try: + resp = session.post( + login_url, data=payload, headers=headers, + timeout=10, allow_redirects=True, + ) + except requests.RequestException: + session.close() + raise + + if self._is_login_success(resp, session, login_url): + self._session = session + return session + + session.close() + return None + + # ── Internal helpers (mirrored from legacy AuthManager) ──────────────── + + @staticmethod + def _is_login_success(response, session, login_url): + if response.status_code >= 400: + return False + body_lower = response.text.lower() + if any(marker in body_lower for marker in _FORM_AUTH_FAILURE_MARKERS): + return False + ct = response.headers.get("content-type", "") + if "application/json" in ct: + try: + data = response.json() + if isinstance(data, dict): + if (data.get("error") or data.get("success") is False + or data.get("authenticated") is False): + return False + except ValueError: + pass + has_cookies = bool(session.cookies.get_dict()) + if response.url and "login" not in response.url.lower(): + if has_cookies: + return True + if response.history and login_url not in response.url: + if has_cookies: + return True + return has_cookies + + def _extract_csrf(self, html): + """Return ``(field_name, token_value)`` or ``(None, None)``. + + Honours ``target_config.csrf_field`` when set, otherwise tries the + common framework field names. Falls back to a generic + hidden-input-with-csrf-or-token heuristic. + """ + configured = getattr(self.target_config, "csrf_field", "") or "" + if configured: + return (configured, self._find_csrf_value(html, configured)) + for field_name in _FORM_AUTH_CSRF_FIELDS: + token = self._find_csrf_value(html, field_name) + if token: + return (field_name, token) + m = re.search( + r']+type=["\']hidden["\'][^>]+name=["\']([^"\']*(?:csrf|token)[^"\']*)["\'][^>]+value=["\']([^"\']+)', + html or "", re.IGNORECASE, + ) + if m: + return (m.group(1), m.group(2)) + return (None, None) + + @staticmethod + def _find_csrf_value(html, field_name): + m = re.search( + rf'name=["\']?{re.escape(field_name)}["\']?\s[^>]*value=["\']([^"\']+)', + html or "", re.IGNORECASE, + ) + if m: + return m.group(1) + m = re.search( + rf'value=["\']([^"\']+)["\'][^>]*name=["\']?{re.escape(field_name)}["\']?', + html or "", re.IGNORECASE, + ) + return m.group(1) if m else None + + +class BearerAuth(AuthStrategy): + """Bearer-token auth for API-only targets. + + Reads `creds.bearer_token` and injects it into every request via the + configured header/scheme (default ``Authorization: Bearer ``). + No HTTP traffic is needed during ``authenticate`` itself — the strategy + simply stamps the session with the token. + + ``preflight`` validates that the configured authenticated probe path is + reachable without sending secret material. A 401/403 is acceptable here + because it usually means auth is enforced; the AuthManager validates the + stamped session after ``authenticate``. + """ + + def __init__(self, target_url, target_config, verify_tls=True, http_client=None): + super().__init__(target_url, target_config, verify_tls, http_client) + self._auth_desc = self._resolve_auth_descriptor() + self._creds = None # populated by authenticate(); needed for refresh() + + def _resolve_auth_descriptor(self): + """Pluck `api_security.auth` off the config or fall back to defaults.""" + api_security = getattr(self.target_config, "api_security", None) + if api_security is not None: + auth = getattr(api_security, "auth", None) + if auth is not None: + return auth + # Tests/callers without an ApiSecurityConfig get sensible defaults. + from .models.target_config import AuthDescriptor + return AuthDescriptor() + + def preflight(self) -> Optional[str]: + probe_path = (self._auth_desc.authenticated_probe_path or "").strip() + if not probe_path: + # Caller opted out of pre-auth verification — strategy will fail + # loudly at the first probe call if the token is invalid. + return None + url = self.target_url + probe_path + try: + if self.http_client is not None: + session = self.make_session() + try: + resp = session.head(url, timeout=10, allow_redirects=True) + finally: + session.close() + else: + resp = requests.head(url, timeout=10, verify=self.verify_tls, + allow_redirects=True) + except requests.RequestException as exc: + return f"Authenticated probe path unreachable: {exc}" + return None + + def authenticate(self, creds) -> Optional[requests.Session]: + if not creds.has_bearer_token(): + return None + session = self.make_session() + scheme = self._auth_desc.bearer_scheme or "Bearer" + header_name = self._auth_desc.bearer_token_header_name or "Authorization" + value = f"{scheme} {creds.bearer_token}".strip() if scheme else creds.bearer_token + session.headers[header_name] = value + self._session = session + self._creds = creds + return session + + def refresh(self, creds) -> bool: + """Default behaviour: re-stamp the same token. Phase 9 OAuth2 follow-up + can replace this with a real refresh-grant call against + `bearer_refresh_url` using `creds.bearer_refresh_token`. + """ + self.cleanup() + return self.authenticate(creds) is not None + + def cleanup(self) -> None: + super().cleanup() + if self._creds is not None: + # Don't clear caller-owned creds — AuthManager.cleanup() drives that. + self._creds = None + + +class ApiKeyAuth(AuthStrategy): + """API-key auth for legacy / partner APIs. + + Places ``creds.api_key`` in either: + - a header (default; configured via + ``auth.api_key_header_name`` — e.g. ``X-Api-Key``) + - a query parameter (``auth.api_key_location='query'``; + configured via ``auth.api_key_query_param``). + + Query-parameter placement is supported for legacy interoperability but + is a known anti-pattern (keys leak to access logs, proxies, referrers). + The Subphase 1.6 evidence scrubber redacts the configured query + parameter from finding evidence; the Navigator launch form shows a + warning banner (Subphase 8.5). + """ + + def __init__(self, target_url, target_config, verify_tls=True, http_client=None): + super().__init__(target_url, target_config, verify_tls, http_client) + self._auth_desc = self._resolve_auth_descriptor() + self._creds = None + + def _resolve_auth_descriptor(self): + api_security = getattr(self.target_config, "api_security", None) + if api_security is not None: + auth = getattr(api_security, "auth", None) + if auth is not None: + return auth + from .models.target_config import AuthDescriptor + return AuthDescriptor() + + def preflight(self) -> Optional[str]: + probe_path = (self._auth_desc.authenticated_probe_path or "").strip() + if not probe_path: + return None + url = self.target_url + probe_path + headers = {} + params = {} + if self._auth_desc.api_key_location == "query": + # We have no key here (preflight runs before authenticate's session + # is created); just check the probe path is reachable. + pass + try: + if self.http_client is not None: + session = self.make_session() + try: + resp = session.head( + url, headers=headers, params=params, timeout=10, + allow_redirects=True, + ) + finally: + session.close() + else: + resp = requests.head( + url, headers=headers, params=params, timeout=10, + verify=self.verify_tls, allow_redirects=True, + ) + except requests.RequestException as exc: + return f"Authenticated probe path unreachable: {exc}" + # 401/403 here is informational — we haven't sent the key yet so it + # may simply mean auth is enforced. Real validation happens after + # authenticate(), when probes start hitting endpoints. + return None + + def authenticate(self, creds) -> Optional[requests.Session]: + if not creds.has_api_key(): + return None + session = self.make_session() + location = self._auth_desc.api_key_location or "header" + if location == "header": + header_name = self._auth_desc.api_key_header_name or "X-Api-Key" + session.headers[header_name] = creds.api_key + elif location == "query": + # Stash the param name + value on the session for per-request mixing + # by probes. Cleanest cross-call carrier without a real session + # extension is the session.params attribute used by requests. + param_name = self._auth_desc.api_key_query_param or "api_key" + session.params = {**(session.params or {}), param_name: creds.api_key} + else: + session.close() + return None + self._session = session + self._creds = creds + return session + + def refresh(self, creds) -> bool: + self.cleanup() + return self.authenticate(creds) is not None + + def cleanup(self) -> None: + super().cleanup() + if self._creds is not None: + self._creds = None diff --git a/extensions/business/cybersec/red_mesh/graybox/budget.py b/extensions/business/cybersec/red_mesh/graybox/budget.py new file mode 100644 index 00000000..5f18915a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/budget.py @@ -0,0 +1,64 @@ +"""Per-scan request budget for graybox probes. + +`RequestBudget` is a small mutable object shared by reference across +every probe instance in a single scan. It enforces a global request cap +so a misconfigured ``target_config`` (e.g., 200 endpoints across 5 +families) cannot DoS the target or the scanner. + +Design (Subphase 1.7 of the API Top 10 plan): +- `GrayboxProbeContext` is `frozen=True`, so it cannot itself hold the + counter. The frozen context instead holds a *reference* to a single + RequestBudget shared across all probes. +- Probes consult the budget via `ProbeBase.budget()` before each HTTP + request. When exhausted, probes emit `inconclusive` with reason + ``budget_exhausted`` rather than skipping silently. +- `consume()` is thread-safe (`threading.Lock`) so future parallel + dispatch cannot double-spend. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + + +@dataclass +class RequestBudget: + """Shared mutable request budget. + + Fields: + remaining: requests not yet consumed. + total: original budget (for reporting). + exhausted_count: number of `consume()` calls that returned False + because the budget was empty. Surfaced in worker metrics so + operators can see whether a scan was budget-bound. + + ``_lock`` guards the check-then-decrement to avoid a race when probes + share the budget across threads (v1 dispatch is single-threaded but + the lock costs nothing and makes future parallelisation safe). + """ + remaining: int + total: int + exhausted_count: int = 0 + _lock: threading.Lock = field(default_factory=threading.Lock, + init=False, repr=False, compare=False) + + def consume(self, n: int = 1) -> bool: + """Decrement by ``n`` if available; return False (and bump + ``exhausted_count``) when the budget can't cover the request.""" + if n <= 0: + raise ValueError("RequestBudget.consume requires n > 0") + with self._lock: + if self.remaining < n: + self.exhausted_count += 1 + return False + self.remaining -= n + return True + + def snapshot(self) -> dict: + """Return a JSON-friendly snapshot for worker outcome / metrics.""" + return { + "remaining": self.remaining, + "total": self.total, + "exhausted_count": self.exhausted_count, + } diff --git a/extensions/business/cybersec/red_mesh/graybox/findings.py b/extensions/business/cybersec/red_mesh/graybox/findings.py index f022f786..10b11f15 100644 --- a/extensions/business/cybersec/red_mesh/graybox/findings.py +++ b/extensions/business/cybersec/red_mesh/graybox/findings.py @@ -5,14 +5,145 @@ unified flat finding dict (matching blackbox findings) at the report level via to_flat_finding(). The blackbox Finding in findings.py is NOT modified. + +Subphase 1.6 (centralised evidence scrubber): every finding traversing +to_flat_finding() passes through `scrub_graybox_secrets`, which strips +Authorization/Cookie/JWT/`password=…`/api_key/etc. patterns from the +evidence list, evidence_artifacts request/response snapshots, finding +description, title, and replay_steps. Probes still SHOULD redact at +emission time (via ProbeBase.emit_*), but the storage-boundary +scrubber is defense-in-depth — one forgetful probe author cannot leak +secrets into the archive, LLM input, or PDF. """ from __future__ import annotations +import re +import contextvars from dataclasses import dataclass, asdict, field from typing import Any +# ── Centralised secret scrubber (Subphase 1.6 commit #2) ──────────────── + +# Generic patterns applied to every flat finding regardless of which +# AuthDescriptor was active. Configured names (X-Custom-Key, custom query +# params) are added to the per-call scrub via ``secret_field_names`` +# when ProbeBase.emit_* invokes the scrubber with the live AuthDescriptor. +_SCRUB_PATTERNS = ( + # Whole-header redaction: redact the full value, which spans until the + # next field separator (comma/semicolon/newline) or end of string. + (re.compile(r"(?i)\b(authorization)\s*:\s*[^,\r\n;]+"), r"\1: "), + (re.compile(r"(?i)\b(cookie)\s*:\s*[^,\r\n;]+"), r"\1: "), + (re.compile(r"(?i)\b(set-cookie)\s*:\s*[^,\r\n;]+"), r"\1: "), + # JWT (3 base64url chunks separated by dots, leading eyJ). + (re.compile(r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}"), + ""), + # Bearer schema in body / URL: keep prefix only. + (re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{8,}"), "Bearer "), + # Common name=value forms (cookie / form / URL query). + (re.compile(r"(?i)\b(password|secret|token|api_key|apikey)=([^&\s\";,]+)"), + r"\1="), + # JSON-style key:value. + (re.compile(r'(?i)"(password|secret|token|api_key|bearer_token|api[\w_-]*key)"\s*:\s*"[^"]+"'), + r'"\1": ""'), +) + + +_FINDING_SECRET_FIELD_NAMES = contextvars.ContextVar( + "redmesh_graybox_finding_secret_field_names", + default=(), +) + + +def _merged_secret_field_names(extra=()) -> tuple[str, ...]: + names = [] + for name in tuple(_FINDING_SECRET_FIELD_NAMES.get(()) or ()) + tuple(extra or ()): + if isinstance(name, str) and name and name not in names: + names.append(name) + return tuple(names) + + +class FindingRedactionContext: + """Temporarily add configured auth field names to finding serialization.""" + + def __init__(self, *, secret_field_names=()): + self.secret_field_names = tuple( + name for name in (secret_field_names or ()) + if isinstance(name, str) and name + ) + self._token = None + + def __enter__(self): + self._token = _FINDING_SECRET_FIELD_NAMES.set(self.secret_field_names) + return self + + def __exit__(self, exc_type, exc, tb): + if self._token is not None: + _FINDING_SECRET_FIELD_NAMES.reset(self._token) + return False + + +def current_finding_secret_field_names() -> tuple[str, ...]: + """Return configured names currently active for finding redaction.""" + return tuple(_FINDING_SECRET_FIELD_NAMES.get(()) or ()) + + +def scrub_graybox_secrets(value: Any, *, secret_field_names: tuple[str, ...] = ()) -> Any: + """Recursively redact known secret patterns from ``value``. + + Accepts strings, lists, tuples, dicts. Non-string leaves pass through. + ``secret_field_names`` is a tuple of additional case-insensitive names + (e.g. configured API-key header / query param names) to scrub on top of + the generic pattern set. + """ + secret_field_names = tuple(secret_field_names or ()) + if isinstance(value, str): + out = value + for pat, repl in _SCRUB_PATTERNS: + out = pat.sub(repl, out) + for name in secret_field_names: + if not name: + continue + esc = re.escape(name) + # name=val → name= + out = re.sub(rf"(?i)\b({esc})=([^&\s\";]+)", r"\1=", out) + # name: val (header form) → name: + out = re.sub(rf"(?i)\b({esc})\s*:\s*\S+", r"\1: ", out) + # JSON "name":"val" + out = re.sub(rf'(?i)"({esc})"\s*:\s*"[^"]+"', r'"\1": ""', out) + return out + if isinstance(value, list): + return [scrub_graybox_secrets(v, secret_field_names=secret_field_names) for v in value] + if isinstance(value, tuple): + return tuple(scrub_graybox_secrets(v, secret_field_names=secret_field_names) for v in value) + if isinstance(value, dict): + return {k: scrub_graybox_secrets(v, secret_field_names=secret_field_names) for k, v in value.items()} + return value + + +def _scrub_flat_finding(flat: dict, *, secret_field_names=()) -> dict: + """Final storage-boundary pass on a flat finding dict. + + Targets the fields most likely to carry secret values: + - title, description, evidence, replay_steps + - evidence_artifacts (request/response snapshots, evidence_items) + Other fields (severity, owasp_id, scenario_id, etc.) are policy-bound + and pass through unchanged. + """ + secret_field_names = _merged_secret_field_names(secret_field_names) + for key in ("title", "description", "evidence", "replay_steps", "remediation"): + if key in flat: + flat[key] = scrub_graybox_secrets( + flat[key], secret_field_names=secret_field_names, + ) + if "evidence_artifacts" in flat and isinstance(flat["evidence_artifacts"], list): + flat["evidence_artifacts"] = scrub_graybox_secrets( + flat["evidence_artifacts"], secret_field_names=secret_field_names, + ) + return flat + + @dataclass(frozen=True) class GrayboxEvidenceArtifact: """Typed graybox evidence payload kept alongside legacy string summaries.""" @@ -68,6 +199,11 @@ class GrayboxFinding: error: str | None = None # non-None if probe had an error cvss_score: float | None = None cvss_vector: str = "" + # OWASP API Top 10 — Subphase 1.8. Stateful-probe rollback outcome. + # Populated by ProbeBase.run_stateful; remains "" for non-stateful + # findings. Renders as a badge in the Navigator UI (Phase 8.3) and in + # the PDF report when revert_failed (Phase 8.4 red-bordered note). + rollback_status: str = "" # "" | "reverted" | "revert_failed" | "no_revert_needed" @classmethod def from_dict(cls, payload: dict[str, Any]) -> "GrayboxFinding": @@ -81,14 +217,17 @@ def from_dict(cls, payload: dict[str, Any]) -> "GrayboxFinding": ] return cls(**data) - def to_dict(self) -> dict[str, Any]: + def to_dict(self, *, secret_field_names=()) -> dict[str, Any]: """JSON-safe serialization.""" payload = asdict(self) payload["evidence_artifacts"] = [ GrayboxEvidenceArtifact.from_value(item).to_dict() for item in self.evidence_artifacts ] - return payload + return scrub_graybox_secrets( + payload, + secret_field_names=_merged_secret_field_names(secret_field_names), + ) def _normalized_evidence_artifacts(self) -> list[GrayboxEvidenceArtifact]: return [GrayboxEvidenceArtifact.from_value(item) for item in self.evidence_artifacts] @@ -103,7 +242,8 @@ def _flat_evidence_summary(self) -> str: ] return "; ".join(artifact_summaries) - def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: + def to_flat_finding(self, port: int, protocol: str, probe_name: str, + *, secret_field_names=()) -> dict: """ Normalize to the unified flat finding dict schema used in PassReport.findings. @@ -114,7 +254,16 @@ def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: canon_title = self.title.lower().strip() cwe_joined = ", ".join(self.cwe) cwe_canonical = ", ".join(sorted({item.strip() for item in self.cwe if isinstance(item, str) and item.strip()})) - id_input = f"{port}:{probe_name}:{cwe_canonical}:{canon_title}" + evidence_identity = [] + for item in self.evidence: + if not isinstance(item, str): + continue + if item.startswith(("endpoint=", "path=", "protected_path=", "token_path=", "flow=", "test_id=")): + evidence_identity.append(item) + id_input = ( + f"{port}:{probe_name}:{self.scenario_id}:{cwe_canonical}:" + f"{canon_title}:{'|'.join(sorted(evidence_identity))}" + ) finding_id = hashlib.sha256(id_input.encode()).hexdigest()[:16] # Map status -> confidence and effective severity @@ -127,7 +276,7 @@ def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: # override severity to INFO so they don't inflate finding_counts effective_severity = "INFO" if self.status == "not_vulnerable" else self.severity.upper() - return { + flat = { "finding_id": finding_id, "probe_type": "graybox", "severity": effective_severity, @@ -152,9 +301,14 @@ def to_flat_finding(self, port: int, protocol: str, probe_name: str) -> dict: "attack_ids": list(self.attack), "cvss_score": self.cvss_score, "cvss_vector": self.cvss_vector, + "rollback_status": self.rollback_status, } + return _scrub_flat_finding(flat, secret_field_names=secret_field_names) @classmethod - def flat_from_dict(cls, payload: dict[str, Any], port: int, protocol: str, probe_name: str) -> dict[str, Any]: + def flat_from_dict(cls, payload: dict[str, Any], port: int, protocol: str, + probe_name: str, *, secret_field_names=()) -> dict[str, Any]: """Normalize a persisted graybox finding dict into the flat report contract.""" - return cls.from_dict(payload).to_flat_finding(port, protocol, probe_name) + return cls.from_dict(payload).to_flat_finding( + port, protocol, probe_name, secret_field_names=secret_field_names, + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/http_client.py b/extensions/business/cybersec/red_mesh/graybox/http_client.py new file mode 100644 index 00000000..78924e08 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/http_client.py @@ -0,0 +1,291 @@ +"""Scoped graybox HTTP client. + +Centralizes host/path scope checks for runtime graybox traffic while +keeping the existing probe-facing ``requests.Session`` shape. +""" + +from __future__ import annotations + +import posixpath +from urllib.parse import unquote, urlsplit, urlunsplit + +import requests + + +class GrayboxScopeError(requests.RequestException): + """Raised before any outbound request when scope validation fails.""" + + +def _decode_repeated(value: str, rounds: int = 3) -> str: + current = value + for _ in range(rounds): + decoded = unquote(current) + if decoded == current: + break + current = decoded + return current + + +def _normalize_path(path: str) -> str: + raw = str(path or "").strip() + if not raw: + return "" + decoded = _decode_repeated(raw) + parsed = urlsplit(decoded) + path = parsed.path if parsed.scheme or parsed.netloc else decoded.split("?", 1)[0] + if not path.startswith("/"): + path = "/" + path + parts = [part for part in path.split("/") if part] + if any(part == ".." for part in parts): + raise GrayboxScopeError(f"path traversal is outside graybox scope: {raw}") + normalized = posixpath.normpath(path) + if normalized == ".": + normalized = "/" + if path.endswith("/") and not normalized.endswith("/"): + normalized += "/" + return normalized if normalized.startswith("/") else "/" + normalized + + +def _split_target(target_url: str): + parsed = urlsplit(target_url) + scheme = parsed.scheme or "http" + hostname = (parsed.hostname or "").lower() + port = parsed.port or (443 if scheme == "https" else 80) + return parsed, scheme, hostname, port + + +def normalize_request_url(target_url: str, url_or_path: str) -> str: + target, scheme, hostname, port = _split_target(target_url) + raw = str(url_or_path or "").strip() + parsed = urlsplit(raw) + if parsed.scheme or parsed.netloc: + req_scheme = parsed.scheme or scheme + req_host = (parsed.hostname or "").lower() + req_port = parsed.port or (443 if req_scheme == "https" else 80) + if req_host != hostname or req_port != port or req_scheme != scheme: + raise GrayboxScopeError(f"cross-origin graybox request blocked: {raw}") + path = _normalize_path(parsed.path or "/") + return urlunsplit((scheme, target.netloc, path, parsed.query, "")) + path = _normalize_path(raw or "/") + return urlunsplit((scheme, target.netloc, path, parsed.query, "")) + + +def path_in_scope(path: str, scope: str) -> bool: + path = _normalize_path(path) + scope = _normalize_path(scope) + if not scope or scope == "/": + return True + if path == scope.rstrip("/"): + return True + prefix = scope if scope.endswith("/") else scope + "/" + return path.startswith(prefix) + + +def path_scopes_from_allowlist(target_url: str, entries) -> list[str]: + scopes = [] + _target, scheme, hostname, port = _split_target(target_url) + for entry in entries or []: + raw = str(entry or "").strip() + if not raw: + continue + parsed = urlsplit(raw) + if parsed.scheme or parsed.netloc: + req_scheme = parsed.scheme or scheme + req_host = (parsed.hostname or "").lower() + req_port = parsed.port or (443 if req_scheme == "https" else 80) + if req_scheme == scheme and req_host == hostname and req_port == port and parsed.path: + scopes.append(_normalize_path(parsed.path)) + continue + if raw.startswith("/"): + scopes.append(_normalize_path(raw)) + deduped = [] + for scope in scopes: + if scope not in deduped: + deduped.append(scope) + return deduped + + +def _append_path(paths, value): + value = str(value or "").strip() + if value: + paths.append(value) + + +def collect_target_config_paths(config: dict) -> list[str]: + """Collect known request paths from canonical GrayboxTargetConfig dict.""" + if not isinstance(config, dict): + return [] + paths = [] + for key in ( + "login_path", "logout_path", "password_reset_path", + "password_reset_confirm_path", + ): + _append_path(paths, config.get(key)) + + access = config.get("access_control") or {} + for item in access.get("idor_endpoints") or []: + _append_path(paths, item.get("path") if isinstance(item, dict) else "") + for item in access.get("admin_endpoints") or []: + _append_path(paths, item.get("path") if isinstance(item, dict) else "") + + misconfig = config.get("misconfig") or {} + for path in misconfig.get("debug_paths") or []: + _append_path(paths, path) + jwt_cfg = misconfig.get("jwt_endpoints") or {} + _append_path(paths, jwt_cfg.get("token_path")) + _append_path(paths, jwt_cfg.get("protected_path")) + + injection = config.get("injection") or {} + for section in ( + "ssrf_endpoints", "xss_endpoints", "ssti_endpoints", + "cmd_endpoints", "header_endpoints", "json_type_endpoints", + ): + for item in injection.get(section) or []: + _append_path(paths, item.get("path") if isinstance(item, dict) else "") + + business = config.get("business_logic") or {} + for section in ("workflow_endpoints", "record_endpoints"): + for item in business.get(section) or []: + _append_path(paths, item.get("path") if isinstance(item, dict) else "") + + api = config.get("api_security") or {} + for section in ( + "object_endpoints", "property_endpoints", "function_endpoints", + "resource_endpoints", + ): + for item in api.get(section) or []: + if not isinstance(item, dict): + continue + _append_path(paths, item.get("path")) + _append_path(paths, item.get("revert_path")) + for flow in api.get("business_flows") or []: + if not isinstance(flow, dict): + continue + _append_path(paths, flow.get("path")) + _append_path(paths, flow.get("verify_path")) + _append_path(paths, flow.get("revert_path")) + token = api.get("token_endpoints") or {} + _append_path(paths, token.get("token_path")) + _append_path(paths, token.get("protected_path")) + _append_path(paths, token.get("logout_path")) + auth = api.get("auth") or {} + _append_path(paths, auth.get("authenticated_probe_path")) + _append_path(paths, auth.get("api_logout_path")) + inventory = api.get("inventory_paths") or {} + for path in inventory.get("openapi_candidates") or []: + _append_path(paths, path) + for path in inventory.get("version_sibling_candidates") or []: + _append_path(paths, path) + for path in inventory.get("deprecated_paths") or []: + _append_path(paths, path) + _append_path(paths, inventory.get("canonical_probe_path")) + for path in api.get("debug_path_candidates") or []: + _append_path(paths, path) + + return paths + + +def validate_target_config_paths(target_url: str, target_config: dict, allowlist) -> list[str]: + scopes = path_scopes_from_allowlist(target_url, allowlist) + if not scopes: + return [] + errors = [] + for raw_path in collect_target_config_paths(target_config): + try: + url = normalize_request_url(target_url, raw_path) + path = urlsplit(url).path + except GrayboxScopeError as exc: + errors.append(str(exc)) + continue + if not any(path_in_scope(path, scope) for scope in scopes): + errors.append(f"configured path {raw_path!r} is outside authorized scope {scopes}") + return errors + + +class ScopedSession: + """Small proxy that preserves the ``requests.Session`` API used by probes.""" + + def __init__(self, session, client: "GrayboxHttpClient"): + object.__setattr__(self, "_session", session) + object.__setattr__(self, "_client", client) + + def __getattr__(self, name): + return getattr(self._session, name) + + def __setattr__(self, name, value): + if name in {"_session", "_client"}: + object.__setattr__(self, name, value) + else: + setattr(self._session, name, value) + + def request(self, method, url, **kwargs): + return self._client.request(self._session, method, url, **kwargs) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def post(self, url, **kwargs): + return self.request("POST", url, **kwargs) + + def put(self, url, **kwargs): + return self.request("PUT", url, **kwargs) + + def patch(self, url, **kwargs): + return self.request("PATCH", url, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + def head(self, url, **kwargs): + return self.request("HEAD", url, **kwargs) + + def options(self, url, **kwargs): + return self.request("OPTIONS", url, **kwargs) + + def close(self): + return self._session.close() + + +class GrayboxHttpClient: + """Runtime host/path scope guard for graybox HTTP traffic.""" + + def __init__(self, target_url: str, *, allowlist=None, target_config=None): + self.target_url = target_url.rstrip("/") + self.scopes = path_scopes_from_allowlist(target_url, allowlist) + discovery = getattr(target_config, "discovery", None) + scope_prefix = getattr(discovery, "scope_prefix", "") if discovery else "" + if scope_prefix and not self.scopes: + self.scopes = [_normalize_path(scope_prefix)] + + def wrap_session(self, session): + if isinstance(session, ScopedSession): + return session + return ScopedSession(session, self) + + def validate_url(self, url_or_path: str) -> str: + url = normalize_request_url(self.target_url, url_or_path) + path = urlsplit(url).path + if self.scopes and not any(path_in_scope(path, scope) for scope in self.scopes): + raise GrayboxScopeError(f"out-of-scope graybox request blocked: {path}") + return url + + def request(self, session, method, url, **kwargs): + allow_redirects = bool(kwargs.pop("allow_redirects", False)) + safe_url = self.validate_url(url) + if not allow_redirects: + return session.request(method, safe_url, allow_redirects=False, **kwargs) + current_url = safe_url + response = None + for _ in range(5): + response = session.request(method, current_url, allow_redirects=False, **kwargs) + if response.status_code not in (301, 302, 303, 307, 308): + return response + location = response.headers.get("Location", "") + if not location: + return response + current_url = self.validate_url(location) + if response.status_code in (301, 302, 303) and method != "HEAD": + method = "GET" + kwargs.pop("data", None) + kwargs.pop("json", None) + return response diff --git a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py index 6d8b1b30..05e8f70a 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/runtime.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/runtime.py @@ -2,20 +2,40 @@ from dataclasses import dataclass, field +from ..auth_credentials import Credentials + @dataclass(frozen=True) class GrayboxCredential: username: str = "" password: str = "" + bearer_token: str = "" + bearer_refresh_token: str = "" + api_key: str = "" + principal: str = "official" @property def is_configured(self) -> bool: - return bool(self.username) + return bool(self.username or self.bearer_token or self.api_key) + + def to_credentials(self) -> Credentials: + return Credentials( + username=self.username, + password=self.password, + bearer_token=self.bearer_token, + bearer_refresh_token=self.bearer_refresh_token, + api_key=self.api_key, + principal=self.principal, + ) def to_dict(self) -> dict: return { "username": self.username, - "password": self.password, + "has_password": bool(self.password), + "has_bearer_token": bool(self.bearer_token), + "has_bearer_refresh_token": bool(self.bearer_refresh_token), + "has_api_key": bool(self.api_key), + "principal": self.principal, } @@ -29,15 +49,27 @@ class GrayboxCredentialSet: @classmethod def from_job_config(cls, job_config) -> GrayboxCredentialSet: regular = None - if getattr(job_config, "regular_username", ""): + if ( + getattr(job_config, "regular_username", "") + or getattr(job_config, "regular_bearer_token", "") + or getattr(job_config, "regular_api_key", "") + ): regular = GrayboxCredential( username=getattr(job_config, "regular_username", "") or "", password=getattr(job_config, "regular_password", "") or "", + bearer_token=getattr(job_config, "regular_bearer_token", "") or "", + bearer_refresh_token=getattr(job_config, "regular_bearer_refresh_token", "") or "", + api_key=getattr(job_config, "regular_api_key", "") or "", + principal="regular", ) return cls( official=GrayboxCredential( username=getattr(job_config, "official_username", "") or "", password=getattr(job_config, "official_password", "") or "", + bearer_token=getattr(job_config, "bearer_token", "") or "", + bearer_refresh_token=getattr(job_config, "bearer_refresh_token", "") or "", + api_key=getattr(job_config, "api_key", "") or "", + principal="official", ), regular=regular, weak_candidates=list(getattr(job_config, "weak_candidates", None) or []), @@ -77,6 +109,15 @@ class GrayboxProbeContext: discovered_forms: list[str] = field(default_factory=list) regular_username: str = "" allow_stateful: bool = False + # OWASP API Top 10 — Subphase 1.7. Reference (not value) to a shared + # mutable RequestBudget. The frozen dataclass owns the binding; the + # budget object itself mutates as probes consume. + request_budget: object = None + allowed_scenario_ids: tuple[str, ...] | None = None + rollback_journal: object = None + job_id: str = "" + worker_id: str = "" + assignment_revision: int = 0 def to_kwargs(self) -> dict: return { @@ -88,6 +129,12 @@ def to_kwargs(self) -> dict: "discovered_forms": list(self.discovered_forms), "regular_username": self.regular_username, "allow_stateful": self.allow_stateful, + "request_budget": self.request_budget, + "allowed_scenario_ids": self.allowed_scenario_ids, + "rollback_journal": self.rollback_journal, + "job_id": self.job_id, + "worker_id": self.worker_id, + "assignment_revision": self.assignment_revision, } diff --git a/extensions/business/cybersec/red_mesh/graybox/models/target_config.py b/extensions/business/cybersec/red_mesh/graybox/models/target_config.py index f7a0983c..5e9dcb86 100644 --- a/extensions/business/cybersec/red_mesh/graybox/models/target_config.py +++ b/extensions/business/cybersec/red_mesh/graybox/models/target_config.py @@ -10,7 +10,7 @@ from __future__ import annotations -from dataclasses import dataclass, asdict, field +from dataclasses import dataclass, asdict, field, fields from typing import Any @@ -24,6 +24,212 @@ ] +_SECRET_BODY_KEY_PARTS = ( + "password", "passwd", "pwd", "secret", "api_key", "apikey", + "authorization", "cookie", "credential", +) +_SECRET_BODY_TOKEN_KEYS = {"token", "access_token", "refresh_token", "id_token"} +_SECRET_BODY_VALUE_MARKERS = ( + "bearer ", "basic ", "apikey=", "api_key=", "access_token=", + "refresh_token=", "client_secret=", "-----begin ", +) +_SAFE_SECRET_BODY_PREFIX = "__redmesh_" +_ALLOWED_SECRET_REF_PREFIXES = ( + ("api_security", "token_endpoints", "token_request_body"), +) +_ALLOWED_SECRET_REF_LIST_FIELDS = { + ("api_security", "function_endpoints"): {"revert_body"}, + ("api_security", "business_flows"): {"body_template", "revert_body"}, +} + + +def _ensure_mapping(d, context: str) -> dict: + if d is None: + return {} + if not isinstance(d, dict): + raise TypeError(f"{context} must be an object") + return d + + +def _coerce_success_status_tuple(value) -> tuple[int, ...]: + """Normalize an authenticated-probe success-status list into a tuple of ints. + + Accepts a list/tuple of integers (or numeric strings). Anything that does + not coerce cleanly is silently dropped — the upstream contract is that + callers either configure valid statuses or leave the field empty. + """ + if value in (None, "", (), []): + return () + if isinstance(value, (str, bytes)): + return () + try: + iterator = iter(value) + except TypeError: + return () + out: list[int] = [] + for item in iterator: + try: + coerced = int(item) + except (TypeError, ValueError): + continue + if 100 <= coerced <= 599: + out.append(coerced) + return tuple(out) + + +def _checked_dict(cls, d, context: str = "") -> dict: + context = context or cls.__name__ + d = _ensure_mapping(d, context) + allowed = {f.name for f in fields(cls)} + unknown = sorted((key for key in d.keys() if key not in allowed), key=str) + if unknown: + unknown_text = ", ".join(str(key) for key in unknown) + raise ValueError(f"{context} has unknown field(s): {unknown_text}") + return d + + +def _looks_like_secret_body_key(key) -> bool: + normalized = str(key or "").strip().lower().replace("-", "_") + if normalized in _SECRET_BODY_TOKEN_KEYS: + return True + if normalized.endswith("_token") or normalized.endswith("_api_key"): + return True + return any(part in normalized for part in _SECRET_BODY_KEY_PARTS) + + +def _looks_like_secret_body_value(value) -> bool: + if not isinstance(value, str): + return False + normalized = value.strip().lower() + if not normalized: + return False + if any(marker in normalized for marker in _SECRET_BODY_VALUE_MARKERS): + return True + # Compact JWT-looking strings are too easy to leak through examples. + return normalized.startswith("eyj") and normalized.count(".") >= 2 + + +def _is_typed_secret_ref(value) -> bool: + if not isinstance(value, dict): + return False + return ( + set(value.keys()) == {"secret_ref"} and + isinstance(value.get("secret_ref"), str) and + bool(value.get("secret_ref").strip()) + ) + + +def _is_safe_secret_body_placeholder(value) -> bool: + return ( + isinstance(value, str) and + value.startswith(_SAFE_SECRET_BODY_PREFIX) and + value.endswith("__") + ) + + +def _is_allowed_secret_ref_path(path: tuple) -> bool: + for prefix in _ALLOWED_SECRET_REF_PREFIXES: + if path[:len(prefix)] == prefix: + return True + for list_prefix, allowed_fields in _ALLOWED_SECRET_REF_LIST_FIELDS.items(): + if len(path) < len(list_prefix) + 2: + continue + if path[:len(list_prefix)] != list_prefix: + continue + if not isinstance(path[len(list_prefix)], int): + continue + if path[len(list_prefix) + 1] in allowed_fields: + return True + return False + + +def iter_target_config_secret_refs(value, path: tuple = ()): + """Yield ``(path, ref_name)`` for typed target-config secret refs.""" + if _is_typed_secret_ref(value): + yield path, value["secret_ref"].strip() + return + if isinstance(value, dict): + for key, item in value.items(): + yield from iter_target_config_secret_refs(item, path + (key,)) + return + if isinstance(value, list): + for idx, item in enumerate(value): + yield from iter_target_config_secret_refs(item, path + (idx,)) + + +def collect_target_config_secret_refs(value) -> list[str]: + refs = [] + seen = set() + for _path, ref in iter_target_config_secret_refs(value): + if ref and ref not in seen: + seen.add(ref) + refs.append(ref) + return refs + + +def validate_target_config_secret_ref_positions(value): + for path, ref in iter_target_config_secret_refs(value): + if not _is_allowed_secret_ref_path(path): + path_text = ".".join(str(part) for part in path) + raise ValueError( + f"{path_text} uses secret_ref {ref!r} outside an approved request body" + ) + + +def resolve_target_config_secret_refs(value, secret_values: dict): + """Return a copy with typed secret refs replaced by runtime values.""" + if _is_typed_secret_ref(value): + ref = value["secret_ref"].strip() + if ref not in (secret_values or {}): + raise KeyError(ref) + return secret_values[ref] + if isinstance(value, dict): + return { + key: resolve_target_config_secret_refs(item, secret_values) + for key, item in value.items() + } + if isinstance(value, list): + return [ + resolve_target_config_secret_refs(item, secret_values) + for item in value + ] + return value + + +def _reject_inline_secrets(value, context: str): + """Reject raw secret material in request-body-like config payloads. + + Request bodies are persisted as part of JobConfig.target_config. They + may contain non-secret test data, but credentials must move through an + explicit secret reference so archives and reports remain publish-safe. + """ + if _is_typed_secret_ref(value): + return + if isinstance(value, dict): + for key, item in value.items(): + item_context = f"{context}.{key}" + if _is_typed_secret_ref(item): + continue + if _looks_like_secret_body_key(key): + if _is_safe_secret_body_placeholder(item): + continue + raise ValueError( + f"{item_context} contains secret-looking data; use secret_ref" + ) + if _looks_like_secret_body_value(item): + raise ValueError( + f"{item_context} contains secret-looking data; use secret_ref" + ) + _reject_inline_secrets(item, item_context) + return + if isinstance(value, list): + for idx, item in enumerate(value): + _reject_inline_secrets(item, f"{context}[{idx}]") + return + if _looks_like_secret_body_value(value): + raise ValueError(f"{context} contains secret-looking data; use secret_ref") + + # ── Typed endpoint configs (E4) ────────────────────────────────────────── @dataclass(frozen=True) @@ -36,6 +242,7 @@ class IdorEndpoint: @classmethod def from_dict(cls, d: dict) -> IdorEndpoint: + d = _checked_dict(cls, d) return cls( path=d["path"], test_ids=d.get("test_ids", [1, 2]), @@ -53,6 +260,7 @@ class AdminEndpoint: @classmethod def from_dict(cls, d: dict) -> AdminEndpoint: + d = _checked_dict(cls, d) return cls( path=d["path"], method=d.get("method", "GET"), @@ -69,6 +277,7 @@ class WorkflowEndpoint: @classmethod def from_dict(cls, d: dict) -> WorkflowEndpoint: + d = _checked_dict(cls, d) return cls( path=d["path"], method=d.get("method", "POST"), @@ -84,6 +293,7 @@ class SsrfEndpoint: @classmethod def from_dict(cls, d: dict) -> SsrfEndpoint: + d = _checked_dict(cls, d) return cls(path=d["path"], param=d.get("param", "url")) @@ -97,6 +307,7 @@ class AccessControlConfig: @classmethod def from_dict(cls, d: dict) -> AccessControlConfig: + d = _checked_dict(cls, d) return cls( idor_endpoints=[IdorEndpoint.from_dict(e) for e in d.get("idor_endpoints", [])], admin_endpoints=[AdminEndpoint.from_dict(e) for e in d.get("admin_endpoints", [])], @@ -113,6 +324,7 @@ class JwtEndpoint: @classmethod def from_dict(cls, d: dict) -> JwtEndpoint: + d = _checked_dict(cls, d) return cls( token_path=d.get("token_path", ""), protected_path=d.get("protected_path", ""), @@ -132,6 +344,7 @@ class MisconfigConfig: @classmethod def from_dict(cls, d: dict) -> MisconfigConfig: + d = _checked_dict(cls, d) return cls( debug_paths=d.get("debug_paths", cls.__dataclass_fields__["debug_paths"].default_factory()), jwt_endpoints=JwtEndpoint.from_dict(d.get("jwt_endpoints", {})), @@ -151,6 +364,7 @@ class ReflectiveEndpoint: @classmethod def from_dict(cls, d: dict) -> ReflectiveEndpoint: + d = _checked_dict(cls, d) return cls(path=d["path"], param=d.get("param", "msg")) @@ -162,6 +376,7 @@ class JsonLookupEndpoint: @classmethod def from_dict(cls, d: dict) -> JsonLookupEndpoint: + d = _checked_dict(cls, d) return cls(path=d["path"], field=d.get("field", "id")) @@ -177,6 +392,7 @@ class InjectionConfig: @classmethod def from_dict(cls, d: dict) -> InjectionConfig: + d = _checked_dict(cls, d) return cls( ssrf_endpoints=[SsrfEndpoint.from_dict(e) for e in d.get("ssrf_endpoints", [])], xss_endpoints=[ReflectiveEndpoint.from_dict(e) for e in d.get("xss_endpoints", [])], @@ -198,6 +414,7 @@ class RecordEndpoint: @classmethod def from_dict(cls, d: dict) -> RecordEndpoint: + d = _checked_dict(cls, d) return cls( path=d["path"], method=d.get("method", "POST"), @@ -215,6 +432,7 @@ class BusinessLogicConfig: @classmethod def from_dict(cls, d: dict) -> BusinessLogicConfig: + d = _checked_dict(cls, d) return cls( workflow_endpoints=[WorkflowEndpoint.from_dict(e) for e in d.get("workflow_endpoints", [])], record_endpoints=[RecordEndpoint.from_dict(e) for e in d.get("record_endpoints", [])], @@ -230,6 +448,7 @@ class DiscoveryConfig: @classmethod def from_dict(cls, d: dict) -> DiscoveryConfig: + d = _checked_dict(cls, d) return cls( scope_prefix=d.get("scope_prefix", ""), max_pages=d.get("max_pages", 50), @@ -237,6 +456,460 @@ def from_dict(cls, d: dict) -> DiscoveryConfig: ) +# ── OWASP API Top 10 2023 endpoint configs ────────────────────────────── +# +# Used by the five API probe families introduced in v1 +# (`api_access`, `api_auth`, `api_data`, `api_config`, `api_abuse`). +# See `docs/adr/2026-05-12-scenario-id-convention.md` and the plan at +# `_todos/2026-05-12-graybox-api-top10-plan-detailed.md` (Subphase 1.1). +# +# `ApiOutboundEndpoint` is deliberately absent — API10 is deferred to +# Phase 9 (callback-receiver infrastructure required). + +@dataclass(frozen=True) +class ApiObjectEndpoint: + """API object endpoint for BOLA testing (PT-OAPI1-01). + + Probe iterates ``test_ids`` against ``path`` (a template containing + ``{id_param}``), as `regular_session`, expects ownership mismatch. + """ + path: str # e.g. "/api/records/{id}/" + test_ids: list[int] = field(default_factory=lambda: [1, 2]) + owner_field: str = "owner" + id_param: str = "id" + tenant_field: str = "" # optional, for cross-tenant BOLA + expected_owner: str = "" # expected low-privilege owner value + expected_tenant: str = "" # expected low-privilege tenant value + + @classmethod + def from_dict(cls, d: dict) -> ApiObjectEndpoint: + d = _checked_dict(cls, d) + return cls( + path=d["path"], + test_ids=d.get("test_ids", [1, 2]), + owner_field=d.get("owner_field", "owner"), + id_param=d.get("id_param", "id"), + tenant_field=d.get("tenant_field", ""), + expected_owner=d.get("expected_owner", ""), + expected_tenant=d.get("expected_tenant", ""), + ) + + +@dataclass(frozen=True) +class ApiPropertyEndpoint: + """API property endpoint for BOPLA testing (PT-OAPI3-01 read, PT-OAPI3-02 write). + + Read probe scans the JSON response for sensitive field names. Write + probe (stateful) attempts to set extra fields from ``tampering_fields`` + on the object identified by ``test_id`` and verifies via re-GET. + """ + path: str # e.g. "/api/profile/{id}/" + method_read: str = "GET" + method_write: str = "PATCH" + test_id: int = 1 # designated object for write test + id_param: str = "id" + + @classmethod + def from_dict(cls, d: dict) -> ApiPropertyEndpoint: + d = _checked_dict(cls, d) + return cls( + path=d["path"], + method_read=d.get("method_read", "GET"), + method_write=d.get("method_write", "PATCH"), + test_id=d.get("test_id", 1), + id_param=d.get("id_param", "id"), + ) + + +@dataclass(frozen=True) +class ApiFunctionEndpoint: + """API function endpoint for BFLA testing (PT-OAPI5-01..04). + + ``method == "GET"`` entries are tested read-only in Phase 2.3 + (PT-OAPI5-01 / PT-OAPI5-02). Non-GET entries require both + ``allow_stateful_probes=True`` AND ``revert_path``/``revert_body`` + (Phase 3.4, PT-OAPI5-03 / PT-OAPI5-04, stateful contract). + """ + path: str # e.g. "/api/admin/users/{uid}/promote/" + method: str = "GET" + privilege: str = "admin" # "admin", "user", "anon" + auth_required_marker: str = "" # body substring expected on 401/403 + revert_path: str = "" # e.g. ".../demote/" — required for stateful + revert_body: dict = field(default_factory=dict) + allow_malformed_json_probe: bool = False # opt-in for PT-OAPI8-04 malformed JSON POST + + @classmethod + def from_dict(cls, d: dict) -> ApiFunctionEndpoint: + d = _checked_dict(cls, d) + _reject_inline_secrets( + d.get("revert_body", {}), + "ApiFunctionEndpoint.revert_body", + ) + return cls( + path=d["path"], + method=d.get("method", "GET"), + privilege=d.get("privilege", "admin"), + auth_required_marker=d.get("auth_required_marker", ""), + revert_path=d.get("revert_path", ""), + revert_body=d.get("revert_body", {}), + allow_malformed_json_probe=d.get("allow_malformed_json_probe", False), + ) + + +@dataclass(frozen=True) +class ApiResourceEndpoint: + """API resource endpoint for bounded resource-consumption testing (PT-OAPI4-*). + + Bounded by construction — no stress testing. Total requests across the + family stop at ``max_total_requests`` (per scan, see ApiSecurityConfig) + or earlier if a 429 is observed. + + ``rate_limit_expected`` defaults to False — only set True when the + endpoint is genuinely supposed to be rate-limited; otherwise the + PT-OAPI4-03 (no-rate-limit) probe will produce noisy false positives. + """ + path: str # e.g. "/api/records/list/" + limit_param: str = "limit" + baseline_limit: int = 10 + abuse_limit: int = 999_999 + rate_limit_expected: bool = False + allow_high_limit_probe: bool = False + allow_oversized_payload_probe: bool = False + oversized_payload_bytes: int = 65_536 + + @classmethod + def from_dict(cls, d: dict) -> ApiResourceEndpoint: + d = _checked_dict(cls, d) + return cls( + path=d["path"], + limit_param=d.get("limit_param", "limit"), + baseline_limit=d.get("baseline_limit", 10), + abuse_limit=d.get("abuse_limit", 999_999), + rate_limit_expected=d.get("rate_limit_expected", False), + allow_high_limit_probe=d.get("allow_high_limit_probe", False), + allow_oversized_payload_probe=d.get("allow_oversized_payload_probe", False), + oversized_payload_bytes=d.get("oversized_payload_bytes", 65_536), + ) + + +@dataclass(frozen=True) +class ApiBusinessFlow: + """Sensitive business-flow endpoint for abuse testing (PT-OAPI6-*). + + All checks are stateful by definition — they create or replay data. + ``test_account`` is a tester-supplied non-privileged identity used so + the official user is never touched by abuse probes. + """ + path: str # e.g. "/api/auth/signup/" + method: str = "POST" + flow_name: str = "signup" # "signup", "password_reset", "purchase", etc. + body_template: dict = field(default_factory=dict) + verify_path: str = "" # endpoint to verify duplicate creation + verify_method: str = "GET" + revert_path: str = "" # cleanup endpoint required before mutation + revert_method: str = "POST" + revert_body: dict = field(default_factory=dict) + test_account: str = "" # non-privileged identity used during abuse test + allow_static_test_account_body: bool = False + captcha_marker: str = "" # body substring indicating CAPTCHA challenge + mfa_marker: str = "" # body substring indicating MFA challenge + + @classmethod + def from_dict(cls, d: dict) -> ApiBusinessFlow: + d = _checked_dict(cls, d) + _reject_inline_secrets( + d.get("body_template", {}), + "ApiBusinessFlow.body_template", + ) + _reject_inline_secrets( + d.get("revert_body", {}), + "ApiBusinessFlow.revert_body", + ) + return cls( + path=d["path"], + method=d.get("method", "POST"), + flow_name=d.get("flow_name", "signup"), + body_template=d.get("body_template", {}), + verify_path=d.get("verify_path", ""), + verify_method=d.get("verify_method", "GET"), + revert_path=d.get("revert_path", ""), + revert_method=d.get("revert_method", "POST"), + revert_body=d.get("revert_body", {}), + test_account=d.get("test_account", ""), + allow_static_test_account_body=d.get("allow_static_test_account_body", False), + captcha_marker=d.get("captcha_marker", ""), + mfa_marker=d.get("mfa_marker", ""), + ) + + +@dataclass(frozen=True) +class ApiTokenEndpoint: + """Token endpoint for broken-auth testing (PT-OAPI2-01..03). + + ``token_path`` issues a JWT given credentials; ``protected_path`` accepts + it. ``logout_path`` is required for PT-OAPI2-03 (logout-doesn't-invalidate, + stateful — revert is re-authentication). + + ``weak_secret_candidates`` is an inline dictionary used by PT-OAPI2-02. + Defaults are deliberately tiny — extend per engagement, or use a + Phase 9 wordlist follow-up. + """ + token_path: str = "" # e.g. "/api/token/" + protected_path: str = "" # e.g. "/api/me/" + logout_path: str = "" # e.g. "/api/auth/logout/" — required for PT-OAPI2-03 + token_request_method: str = "POST" + token_request_body: dict = field(default_factory=dict) + token_response_field: str = "" + weak_secret_candidates: list[str] = field(default_factory=lambda: [ + "secret", "changeme", "password", "1234567890", + "jwt", "key", "topsecret", "default", + ]) + + @classmethod + def from_dict(cls, d: dict) -> ApiTokenEndpoint: + d = _checked_dict(cls, d) + _reject_inline_secrets( + d.get("token_request_body", {}), + "ApiTokenEndpoint.token_request_body", + ) + defaults = cls.__dataclass_fields__["weak_secret_candidates"].default_factory() + return cls( + token_path=d.get("token_path", ""), + protected_path=d.get("protected_path", ""), + logout_path=d.get("logout_path", ""), + token_request_method=d.get("token_request_method", "POST"), + token_request_body=d.get("token_request_body", {}), + token_response_field=d.get("token_response_field", ""), + weak_secret_candidates=d.get("weak_secret_candidates", defaults), + ) + + +@dataclass(frozen=True) +class ApiInventoryPaths: + """Inventory-related paths for API9 testing. + + ``openapi_candidates`` are probed by PT-OAPI9-01 looking for an exposed + OpenAPI/Swagger document. ``current_version`` + sibling probing drives + PT-OAPI9-02 (version sprawl); ``deprecated_paths`` drives PT-OAPI9-03. + ``private_path_patterns`` is used as the substring/glob set indicating + paths in the spec that shouldn't be publicly exposed. + """ + openapi_candidates: list[str] = field(default_factory=lambda: [ + "/openapi.json", "/swagger.json", "/v3/api-docs", + "/api/swagger.json", "/api-docs", "/swagger-ui.html", + ]) + current_version: str = "" # e.g. "/api/v2/" + version_sibling_candidates: list[str] = field(default_factory=lambda: [ + "/api/v1/", "/api/v0/", "/api/beta/", "/api/internal/", "/api/legacy/", + ]) + canonical_probe_path: str = "" # canonical endpoint under current_version used to verify a sibling responds + private_path_patterns: list[str] = field(default_factory=list) + deprecated_paths: list[str] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict) -> ApiInventoryPaths: + d = _checked_dict(cls, d) + fields_ = cls.__dataclass_fields__ + return cls( + openapi_candidates=d.get( + "openapi_candidates", + fields_["openapi_candidates"].default_factory(), + ), + current_version=d.get("current_version", ""), + version_sibling_candidates=d.get( + "version_sibling_candidates", + fields_["version_sibling_candidates"].default_factory(), + ), + canonical_probe_path=d.get("canonical_probe_path", ""), + private_path_patterns=d.get("private_path_patterns", []), + deprecated_paths=d.get("deprecated_paths", []), + ) + + +@dataclass(frozen=True) +class AuthDescriptor: + """Non-secret auth configuration for graybox session establishment. + + Secret values (`bearer_token`, `api_key`, `bearer_refresh_token`) are + **never** carried in this object or anywhere inside ``target_config``. + They travel as top-level launch parameters and are stored in the R1FS + secret payload — see Subphase 1.5 commit #8. + + Fields: + auth_type: Selects the AuthStrategy at runtime. ``form`` is the + default and keeps existing behaviour. ``bearer`` and + ``api_key`` add API-native auth in Subphase 1.5. + bearer_token_header_name: HTTP header used for Bearer tokens. Default + ``Authorization``; rare APIs use ``X-Auth-Token`` etc. + bearer_scheme: Scheme prefix for Bearer tokens. Default ``Bearer``; + some APIs use ``Token`` or empty (raw token). + bearer_refresh_url: Optional. If set, BearerAuth will POST here to + refresh an expired token (Phase 9 OAuth2 will replace this + with a proper grant flow). + api_key_header_name: Header name for API-key auth, e.g. ``X-Api-Key``. + api_key_query_param: Query-parameter name for API-key auth when + ``api_key_location='query'``. + api_key_location: ``header`` (default) or ``query``. Query is allowed + for legacy APIs only; evidence scrubbers will redact the + configured param name from URLs at the finding boundary. + authenticated_probe_path: Path used by strategy preflight when + ``auth_type != 'form'`` to verify the credentials work + before any probe runs (e.g. ``/api/me``). + authenticated_probe_method: HTTP method for authenticated validation. + Defaults to GET because many APIs reject HEAD even when + credentials are valid. + allow_unverified_auth: Explicit opt-out for Bearer/API-key + validation. When true and no authenticated_probe_path is + configured, auth-dependent API probes emit auth_unverified + inconclusive findings instead of clean/vulnerable claims. + allow_non_readonly_auth_validation_method: Explicit opt-in for + validation methods outside GET/HEAD. Use only for + documented safe validation endpoints. + api_logout_path: Optional explicit logout endpoint for API-native + sessions. Form scans continue using ``logout_path``. + authenticated_probe_success_statuses: Optional explicit allow-list of + HTTP statuses that prove the session is authenticated. + Required when the probe path is also accessible + anonymously, so that the response distinguishes the two. + When empty, validation falls back to ``2xx + non-3xx`` + plus the anonymous-control delta check below. + authenticated_probe_success_marker: Optional case-sensitive substring + that must appear in the authenticated response body and + NOT in the anonymous-control response body, used to + confirm the endpoint reflects the authenticated principal. + authenticated_probe_identity_json_path: Dotted JSON path (no array + indexing, no expressions) within the response body that + must resolve to a non-empty value when authenticated and + not when anonymous (e.g. ``user.id``). + """ + auth_type: str = "form" # "form" | "bearer" | "api_key" + bearer_token_header_name: str = "Authorization" + bearer_scheme: str = "Bearer" + bearer_refresh_url: str = "" + api_key_header_name: str = "X-Api-Key" + api_key_query_param: str = "api_key" + api_key_location: str = "header" # "header" | "query" + authenticated_probe_path: str = "" + authenticated_probe_method: str = "GET" + allow_unverified_auth: bool = False + allow_non_readonly_auth_validation_method: bool = False + api_logout_path: str = "" + authenticated_probe_success_statuses: tuple[int, ...] = () + authenticated_probe_success_marker: str = "" + authenticated_probe_identity_json_path: str = "" + + @classmethod + def from_dict(cls, d: dict) -> AuthDescriptor: + d = _checked_dict(cls, d) + return cls( + auth_type=d.get("auth_type", "form"), + bearer_token_header_name=d.get("bearer_token_header_name", "Authorization"), + bearer_scheme=d.get("bearer_scheme", "Bearer"), + bearer_refresh_url=d.get("bearer_refresh_url", ""), + api_key_header_name=d.get("api_key_header_name", "X-Api-Key"), + api_key_query_param=d.get("api_key_query_param", "api_key"), + api_key_location=d.get("api_key_location", "header"), + authenticated_probe_path=d.get("authenticated_probe_path", ""), + authenticated_probe_method=d.get("authenticated_probe_method", "GET"), + allow_unverified_auth=d.get("allow_unverified_auth", False), + allow_non_readonly_auth_validation_method=d.get( + "allow_non_readonly_auth_validation_method", False, + ), + api_logout_path=d.get("api_logout_path", ""), + authenticated_probe_success_statuses=_coerce_success_status_tuple( + d.get("authenticated_probe_success_statuses", ()), + ), + authenticated_probe_success_marker=str( + d.get("authenticated_probe_success_marker", "") or "" + ), + authenticated_probe_identity_json_path=str( + d.get("authenticated_probe_identity_json_path", "") or "" + ), + ) + + +@dataclass(frozen=True) +class ApiSecurityConfig: + """Aggregated config for the five OWASP API Top 10 graybox probe families. + + Probes draw from exactly the section they own: + - api_access → object_endpoints (BOLA), function_endpoints (BFLA) + - api_auth → token_endpoints (broken auth) + - api_data → property_endpoints (BOPLA read/write) + - api_config → inventory_paths, debug_path_candidates (misconfig/inventory) + - api_abuse → resource_endpoints, business_flows + + ``ssrf_body_fields`` extends the legacy PT-API7-01 SSRF probe (lives in + injection.py, kept under its legacy ID) to scan JSON body fields by name. + + ``sensitive_field_patterns`` augments the built-in default patterns used + by PT-OAPI3-01 (excessive property exposure). Entries are merged with, + not replacing, the defaults. + + ``tampering_fields`` lists property names PT-OAPI3-02 will attempt to set + via mass assignment. + + Auth descriptor (`auth`) and per-scan request budget + (`max_total_requests`) land in Subphases 1.5 and 1.7 respectively; + added here as future hooks would couple this subphase to those. + """ + object_endpoints: list[ApiObjectEndpoint] = field(default_factory=list) + property_endpoints: list[ApiPropertyEndpoint] = field(default_factory=list) + function_endpoints: list[ApiFunctionEndpoint] = field(default_factory=list) + resource_endpoints: list[ApiResourceEndpoint] = field(default_factory=list) + business_flows: list[ApiBusinessFlow] = field(default_factory=list) + token_endpoints: ApiTokenEndpoint = field(default_factory=ApiTokenEndpoint) + inventory_paths: ApiInventoryPaths = field(default_factory=ApiInventoryPaths) + auth: AuthDescriptor = field(default_factory=AuthDescriptor) + + ssrf_body_fields: list[str] = field(default_factory=lambda: [ + "url", "webhook", "callback", "image_url", "redirect_uri", + ]) + sensitive_field_patterns: list[str] = field(default_factory=list) + tampering_fields: list[str] = field(default_factory=lambda: [ + "is_admin", "is_superuser", "role", "verified", "email_verified", + "tenant_id", "owner_id", "balance", + ]) + debug_path_candidates: list[str] = field(default_factory=lambda: [ + "/debug", "/api/debug", "/api/_routes", + "/actuator", "/actuator/env", "/q/dev", "/__debug__", + ]) + # OWASP API Top 10 — Subphase 1.7. Per-scan request budget cap. Each + # `ProbeBase.budget()` call decrements a shared `RequestBudget`; once + # exhausted, probes emit `inconclusive` with reason `budget_exhausted` + # rather than continuing to issue requests. + max_total_requests: int = 1000 + + @classmethod + def from_dict(cls, d: dict) -> ApiSecurityConfig: + d = _checked_dict(cls, d) + fields_ = cls.__dataclass_fields__ + return cls( + object_endpoints=[ApiObjectEndpoint.from_dict(e) for e in d.get("object_endpoints", [])], + property_endpoints=[ApiPropertyEndpoint.from_dict(e) for e in d.get("property_endpoints", [])], + function_endpoints=[ApiFunctionEndpoint.from_dict(e) for e in d.get("function_endpoints", [])], + resource_endpoints=[ApiResourceEndpoint.from_dict(e) for e in d.get("resource_endpoints", [])], + business_flows=[ApiBusinessFlow.from_dict(e) for e in d.get("business_flows", [])], + token_endpoints=ApiTokenEndpoint.from_dict(d.get("token_endpoints", {})), + inventory_paths=ApiInventoryPaths.from_dict(d.get("inventory_paths", {})), + auth=AuthDescriptor.from_dict(d.get("auth", {})), + ssrf_body_fields=d.get( + "ssrf_body_fields", + fields_["ssrf_body_fields"].default_factory(), + ), + sensitive_field_patterns=d.get("sensitive_field_patterns", []), + tampering_fields=d.get( + "tampering_fields", + fields_["tampering_fields"].default_factory(), + ), + debug_path_candidates=d.get( + "debug_path_candidates", + fields_["debug_path_candidates"].default_factory(), + ), + max_total_requests=d.get("max_total_requests", 1000), + ) + + # ── Main config ───────────────────────────────────────────────────────── @dataclass(frozen=True) @@ -257,6 +930,7 @@ class GrayboxTargetConfig: injection: InjectionConfig = field(default_factory=InjectionConfig) business_logic: BusinessLogicConfig = field(default_factory=BusinessLogicConfig) discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) + api_security: ApiSecurityConfig = field(default_factory=ApiSecurityConfig) # Login endpoint configuration (shared across probes) login_path: str = "/auth/login/" @@ -272,12 +946,14 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, d: dict) -> GrayboxTargetConfig: + d = _checked_dict(cls, d) return cls( access_control=AccessControlConfig.from_dict(d.get("access_control", {})), misconfig=MisconfigConfig.from_dict(d.get("misconfig", {})), injection=InjectionConfig.from_dict(d.get("injection", {})), business_logic=BusinessLogicConfig.from_dict(d.get("business_logic", {})), discovery=DiscoveryConfig.from_dict(d.get("discovery", {})), + api_security=ApiSecurityConfig.from_dict(d.get("api_security", {})), login_path=d.get("login_path", "/auth/login/"), logout_path=d.get("logout_path", "/auth/logout/"), password_reset_path=d.get("password_reset_path", ""), diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/api_abuse.py b/extensions/business/cybersec/red_mesh/graybox/probes/api_abuse.py new file mode 100644 index 00000000..2d244896 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/api_abuse.py @@ -0,0 +1,511 @@ +"""API abuse probes — OWASP API4 (Resource Consumption) and API6 (Business Flows).""" + +import re + +import requests + +from .base import ProbeBase + + +MAX_HIGH_LIMIT_PROBE_LIMIT = 1_000 +_EXACT_TEMPLATE_RE = re.compile(r"^\{([a-zA-Z_][a-zA-Z0-9_]*)\}$") +_ALLOWED_TEMPLATE_KEYS = ("test_account", "run_id", "job_id") + + +class ApiAbuseProbes(ProbeBase): + """OWASP API4 + API6 graybox probes. + + Scenarios implemented (Subphases 3.2 + 3.3): + PT-OAPI4-01 — pagination cap missing (bounded; non-stateful) + PT-OAPI4-02 — oversized payload accepted (bounded; non-stateful) + PT-OAPI4-03 — rate limit absent (bounded; requires `rate_limit_expected=True`) + PT-OAPI6-01 — business flow lacks rate limit (STATEFUL) + PT-OAPI6-02 — business flow lacks uniqueness check (STATEFUL) + """ + + requires_auth = True + requires_regular_session = False + is_stateful = False + probe_key = "_graybox_api_abuse" + + def run(self): + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return self.findings + return self.run_runtime_scenarios(self.probe_key) + + def _session(self): + return self.auth.official_session or self.auth.regular_session + + def _low_priv_session(self): + return self.auth.regular_session + + @staticmethod + def _bounded_int(value, *, default: int, minimum: int = 1, + maximum: int = MAX_HIGH_LIMIT_PROBE_LIMIT) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + parsed = default + return max(minimum, min(parsed, maximum)) + + def _flow_request(self, session, method, url, body, timeout=10): + req = getattr(session, (method or "POST").lower(), session.post) + if (method or "POST").upper() in ("GET", "DELETE"): + return req(url, params=dict(body or {}), timeout=timeout) + return req(url, json=dict(body or {}), timeout=timeout) + + def _flow_template_context(self, flow): + job_id = self.job_id or "local" + run_id = f"{job_id}:{self.assignment_revision or 0}" + return { + "test_account": flow.test_account, + "run_id": run_id, + "job_id": job_id, + } + + def _render_template_value(self, value, context): + if isinstance(value, str): + match = _EXACT_TEMPLATE_RE.match(value) + if match: + key = match.group(1) + if key not in _ALLOWED_TEMPLATE_KEYS: + raise ValueError(f"unsupported_template_key:{key}") + return context[key], key == "test_account" + if "{" in value or "}" in value: + raise ValueError("unsupported_template_expression") + return value, False + if isinstance(value, dict): + out = {} + used_test_account = False + for key, item in value.items(): + rendered, used = self._render_template_value(item, context) + out[key] = rendered + used_test_account = used_test_account or used + return out, used_test_account + if isinstance(value, list): + out = [] + used_test_account = False + for item in value: + rendered, used = self._render_template_value(item, context) + out.append(rendered) + used_test_account = used_test_account or used + return out, used_test_account + return value, False + + def _render_flow_payloads(self, flow): + context = self._flow_template_context(flow) + try: + body, body_uses_test_account = self._render_template_value( + flow.body_template or {}, context, + ) + revert_body, revert_uses_test_account = self._render_template_value( + flow.revert_body or {}, context, + ) + except ValueError as exc: + return None, None, str(exc) + unsafe_static_body = bool( + getattr(flow, "allow_static_test_account_body", False) + ) + if not body_uses_test_account and not unsafe_static_body: + return None, None, "test_account_placeholder_required" + if flow.revert_body and not revert_uses_test_account and not unsafe_static_body: + return None, None, "revert_test_account_placeholder_required" + return body, revert_body, "" + + def _flow_verify(self, session, flow): + if not flow.verify_path: + return True + if not self.budget(): + raise RuntimeError("budget_exhausted") + self.safety.throttle() + resp = self._flow_request( + session, + flow.verify_method, + self.target_url + flow.verify_path, + {}, + timeout=10, + ) + return resp.status_code < 400 + + def _flow_revert(self, session, flow, revert_body): + if not flow.revert_path: + return False + if not self.cleanup_budget(): + return False + self.safety.throttle() + resp = self._flow_request( + session, + flow.revert_method, + self.target_url + flow.revert_path, + revert_body, + timeout=10, + ) + return resp.status_code < 400 + + def _flow_revert_fn(self, session, flow, revert_body): + if not flow.revert_path: + return None + + def revert(_baseline, _flow=flow, _revert_body=revert_body): + return self._flow_revert(session, _flow, _revert_body) + + return revert + + def _flow_replay_steps(self, flow, url, action): + steps = [ + f"{action}: {(flow.method or 'POST').upper()} {url}", + ] + if flow.revert_path: + steps.append( + "rollback: " + f"{(flow.revert_method or 'POST').upper()} " + f"{self.target_url + flow.revert_path}" + ) + return steps + + # ── PT-OAPI4-01 — no pagination cap ──────────────────────────────── + + def _test_no_pagination_cap(self): + if not self.scenario_enabled("PT-OAPI4-01"): + return + title = "API endpoint lacks pagination cap" + owasp = "API4:2023" + if not self.target_config.api_security.resource_endpoints: + self.emit_inconclusive( + "PT-OAPI4-01", title, owasp, "no_configured_resource_endpoints", + ) + return + session = self._session() + if session is None: + self.emit_inconclusive("PT-OAPI4-01", title, owasp, "no_authenticated_session") + return + for ep in self.target_config.api_security.resource_endpoints: + if not getattr(ep, "allow_high_limit_probe", False): + self.emit_inconclusive( + "PT-OAPI4-01", title, owasp, "high_limit_probe_not_authorized", + ) + continue + if not self.budget(2): + self.emit_inconclusive("PT-OAPI4-01", title, owasp, "budget_exhausted") + return + url = self.target_url + ep.path + baseline_limit = self._bounded_int(ep.baseline_limit, default=10) + abuse_limit = self._bounded_int(ep.abuse_limit, default=MAX_HIGH_LIMIT_PROBE_LIMIT) + if abuse_limit <= baseline_limit: + self.emit_inconclusive( + "PT-OAPI4-01", title, owasp, "invalid_limit_bounds", + ) + continue + self.safety.throttle() + try: + baseline = session.get( + url, params={ep.limit_param: baseline_limit}, timeout=10, + ) + except requests.RequestException: + continue + self.safety.throttle() + try: + abuse = session.get( + url, params={ep.limit_param: abuse_limit}, timeout=10, + ) + except requests.RequestException: + continue + if baseline.status_code >= 400 or abuse.status_code >= 400: + continue + base_size = len((baseline.text or "").encode()) + abuse_size = len((abuse.text or "").encode()) + if abuse_size > 5 * max(1, base_size): + self.emit_vulnerable( + "PT-OAPI4-01", title, "MEDIUM", owasp, ["CWE-770"], + [f"endpoint={url}", f"requested_limit={ep.abuse_limit}", + f"effective_limit={abuse_limit}", + f"baseline_size_bytes={base_size}", + f"abuse_size_bytes={abuse_size}"], + remediation=( + "Cap pagination server-side. Reject limit values above a " + "configured maximum (typically 100–1000)." + ), + ) + else: + self.emit_clean( + "PT-OAPI4-01", title, owasp, + [f"endpoint={url}", "size_growth_within_cap"], + ) + + # ── PT-OAPI4-02 — oversized payload ──────────────────────────────── + + def _test_oversized_payload(self): + if not self.scenario_enabled("PT-OAPI4-02"): + return + title = "API endpoint accepts oversized payload" + owasp = "API4:2023" + if not self.target_config.api_security.resource_endpoints: + self.emit_inconclusive( + "PT-OAPI4-02", title, owasp, "no_configured_resource_endpoints", + ) + return + session = self._session() + if session is None: + self.emit_inconclusive("PT-OAPI4-02", title, owasp, "no_authenticated_session") + return + for ep in self.target_config.api_security.resource_endpoints: + if not getattr(ep, "allow_oversized_payload_probe", False): + self.emit_inconclusive( + "PT-OAPI4-02", title, owasp, "oversized_payload_probe_not_authorized", + ) + continue + if not self.budget(): + self.emit_inconclusive("PT-OAPI4-02", title, owasp, "budget_exhausted") + return + body_bytes = max(1, min(int(getattr(ep, "oversized_payload_bytes", 65_536) or 65_536), 262_144)) + big = "A" * body_bytes + url = self.target_url + ep.path + self.safety.throttle() + try: + resp = session.post(url, json={"x": big}, timeout=15) + except requests.RequestException: + continue + if resp.status_code == 429: + return + if resp.status_code < 400: + self.emit_vulnerable( + "PT-OAPI4-02", title, "MEDIUM", owasp, ["CWE-770"], + [f"endpoint={url}", f"body_bytes={body_bytes}", + f"response_status={resp.status_code}"], + remediation=( + "Enforce a request-body size limit at the reverse-proxy or " + "framework layer." + ), + ) + + # ── PT-OAPI4-03 — no rate limit ──────────────────────────────────── + + def _test_no_rate_limit(self): + if not self.scenario_enabled("PT-OAPI4-03"): + return + title = "API endpoint lacks rate limit" + owasp = "API4:2023" + if not self.target_config.api_security.resource_endpoints: + self.emit_inconclusive( + "PT-OAPI4-03", title, owasp, "no_configured_resource_endpoints", + ) + return + session = self._session() + if session is None: + self.emit_inconclusive("PT-OAPI4-03", title, owasp, "no_authenticated_session") + return + for ep in self.target_config.api_security.resource_endpoints: + if not ep.rate_limit_expected: + continue # FP guard — only fire when operator marked endpoint + n = 0 + saw_429 = False + saw_retry_after = False + saw_ratelimit_header = False + for _ in range(10): + if not self.budget(): + self.emit_inconclusive("PT-OAPI4-03", title, owasp, "budget_exhausted") + break + self.safety.throttle() + try: + resp = session.get(self.target_url + ep.path, timeout=10) + except requests.RequestException: + break + n += 1 + if resp.status_code == 429: + saw_429 = True + break + if resp.headers.get("Retry-After"): + saw_retry_after = True + if any(h.lower().startswith("x-ratelimit") for h in resp.headers): + saw_ratelimit_header = True + if n >= 5 and not (saw_429 or saw_retry_after or saw_ratelimit_header): + self.emit_vulnerable( + "PT-OAPI4-03", title, "LOW", owasp, ["CWE-770"], + [f"endpoint={self.target_url + ep.path}", + f"requests_sent={n}", + "rate_limit_signals=absent"], + remediation=( + "Apply rate limiting (token bucket / leaky bucket / sliding " + "window) at the gateway. Return 429 + Retry-After when the " + "limit is reached." + ), + ) + + # ── PT-OAPI6-01 — flow no rate limit (STATEFUL) ──────────────────── + + def _test_flow_no_rate_limit(self): + if not self.scenario_enabled("PT-OAPI6-01"): + return + title = "API business flow lacks rate limit / abuse controls" + owasp = "API6:2023" + if not self.target_config.api_security.business_flows: + self.emit_inconclusive( + "PT-OAPI6-01", title, owasp, "no_configured_business_flows", + ) + return + session = self._low_priv_session() + if session is None: + self.emit_inconclusive("PT-OAPI6-01", title, owasp, "no_low_privileged_session") + return + for flow in self.target_config.api_security.business_flows: + if not flow.test_account: + self.emit_inconclusive("PT-OAPI6-01", title, owasp, "no_test_account_configured") + continue + body, revert_body, template_error = self._render_flow_payloads(flow) + if template_error: + self.emit_inconclusive("PT-OAPI6-01", title, owasp, template_error) + continue + url = self.target_url + flow.path + probe_state = {} + + def baseline(_flow=flow): + return {"flow_name": _flow.flow_name} + + def mutate(_baseline, _flow=flow, _url=url, _body=body, + _probe_state=probe_state): + attempts = 0 + captcha = False + mfa = False + for _ in range(5): + if not self.budget(): + raise RuntimeError("budget_exhausted") + self.safety.throttle() + try: + resp = self._flow_request( + session, _flow.method, _url, _body, timeout=10, + ) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + attempts += 1 + if resp.status_code == 429: + break + body = (resp.text or "")[:2000].lower() + if _flow.captcha_marker and _flow.captcha_marker.lower() in body: + captcha = True + if _flow.mfa_marker and _flow.mfa_marker.lower() in body: + mfa = True + _probe_state["attempts"] = attempts + _probe_state["captcha"] = captcha + _probe_state["mfa"] = mfa + return attempts >= 5 and not (captcha or mfa) + + def verify(baseline_, _flow=flow, _probe_state=probe_state): + state = _probe_state + signals_confirmed = state.get("attempts", 0) >= 5 and not ( + state.get("captcha") or state.get("mfa") + ) + if not signals_confirmed: + return False + try: + return self._flow_verify(session, _flow) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + + self.run_stateful( + "PT-OAPI6-01", + baseline_fn=baseline, + mutate_fn=mutate, + verify_fn=verify, + revert_fn=self._flow_revert_fn(session, flow, revert_body), + finding_kwargs={ + "title": title, "owasp": owasp, "severity": "MEDIUM", + "cwe": ["CWE-799", "CWE-840"], + "evidence": [f"flow={flow.flow_name}", f"endpoint={url}", + "attempts=5"], + "replay_steps": self._flow_replay_steps(flow, url, "repeat 5 times"), + "remediation": ( + "Add an abuse-prevention layer to sensitive flows: per-account " + "quota, CAPTCHA challenge after N attempts, or MFA when the " + "operation impacts billing / identity. Pure rate-limit at the " + "IP layer is insufficient." + ), + }, + no_mutation_reason_fn=lambda base: "abuse_signals_not_confirmed", + ) + + # ── PT-OAPI6-02 — flow no uniqueness check (STATEFUL) ────────────── + + def _test_flow_no_uniqueness(self): + if not self.scenario_enabled("PT-OAPI6-02"): + return + title = "API business flow lacks uniqueness check" + owasp = "API6:2023" + if not self.target_config.api_security.business_flows: + self.emit_inconclusive( + "PT-OAPI6-02", title, owasp, "no_configured_business_flows", + ) + return + session = self._low_priv_session() + if session is None: + self.emit_inconclusive("PT-OAPI6-02", title, owasp, "no_low_privileged_session") + return + for flow in self.target_config.api_security.business_flows: + if not flow.test_account: + self.emit_inconclusive("PT-OAPI6-02", title, owasp, "no_test_account_configured") + continue + body, revert_body, template_error = self._render_flow_payloads(flow) + if template_error: + self.emit_inconclusive("PT-OAPI6-02", title, owasp, template_error) + continue + url = self.target_url + flow.path + probe_state = {} + + def baseline(_flow=flow): + return {"flow_name": _flow.flow_name} + + def mutate(_b, _flow=flow, _url=url, _body=body, + _probe_state=probe_state): + if not self.budget(2): + raise RuntimeError("budget_exhausted") + # Track whether the first mutating request was already issued so a + # transport failure on the second send still triggers revert + # (PR406 B3): leaving the target with the first duplicate in place + # while reporting "no mutation needed" is unsafe. + mutation_sent = False + try: + self.safety.throttle() + r1 = self._flow_request( + session, _flow.method, _url, _body, timeout=10, + ) + mutation_sent = True + self.safety.throttle() + r2 = self._flow_request( + session, _flow.method, _url, _body, timeout=10, + ) + except requests.RequestException: + if mutation_sent: + return self.MUTATION_ATTEMPTED_UNKNOWN + return False + _probe_state["both_2xx"] = ( + r1.status_code < 400 and r2.status_code < 400 + ) + return _probe_state["both_2xx"] + + def verify(_b, _flow=flow, _probe_state=probe_state): + if not _probe_state.get("both_2xx", False): + return False + try: + return self._flow_verify(session, _flow) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + + self.run_stateful( + "PT-OAPI6-02", + baseline_fn=baseline, + mutate_fn=mutate, + verify_fn=verify, + revert_fn=self._flow_revert_fn(session, flow, revert_body), + finding_kwargs={ + "title": title, "owasp": owasp, "severity": "MEDIUM", + "cwe": ["CWE-840"], + "evidence": [f"flow={flow.flow_name}", f"endpoint={url}", + "duplicate_accepted=true"], + "replay_steps": self._flow_replay_steps(flow, url, "submit twice"), + "remediation": ( + "Enforce uniqueness server-side (e.g., unique constraint on " + "username/email/voucher-code). Return 409 Conflict on duplicate." + ), + }, + no_mutation_reason_fn=lambda base: "duplicate_submission_not_accepted", + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/api_access.py b/extensions/business/cybersec/red_mesh/graybox/probes/api_access.py new file mode 100644 index 00000000..2bcd71f0 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/api_access.py @@ -0,0 +1,620 @@ +"""API access-control probes — OWASP API1 (BOLA) and API5 (BFLA).""" + +import re + +import requests + +from .base import ProbeBase + + +# Sensitive-field-name patterns that escalate a BOLA finding to CRITICAL +# when present in the leaked response (Subphase 2.1 design § FP guards + +# severity). Field NAMES only — values never inspected here; the +# centralised scrubber strips secret values at the storage boundary. +_BOLA_PII_FIELD_PATTERNS = ( + re.compile(r"(?i)\b(email|e_mail)\b"), + re.compile(r"(?i)\b(ssn|social_security)\b"), + re.compile(r"(?i)\b(token|api_key|password|secret)\b"), + re.compile(r"(?i)\b(credit_?card|cc_number|cc_num|card_number)\b"), + re.compile(r"(?i)\b(phone|mobile|telephone)\b"), +) + + +class ApiAccessProbes(ProbeBase): + """OWASP API1 (BOLA) + API5 (BFLA) graybox probes. + + Scenarios: + PT-OAPI1-01 — API object-level authorization bypass (BOLA, read) + — implemented in Subphase 2.1. + PT-OAPI5-01 — Function-level authorization bypass (regular as admin, + read) — Subphase 2.3. + PT-OAPI5-02 — Function-level authorization bypass (anonymous as user, + read) — Subphase 2.3. + PT-OAPI5-03 — Method-override authorization bypass — Subphase 3.4. + PT-OAPI5-04 — Function-level authorization bypass (regular as admin, + mutating; stateful, requires revert plan) — Subphase 3.4. + """ + + requires_auth = True + requires_regular_session = False + is_stateful = False + probe_key = "_graybox_api_access" + + def run(self): + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return self.findings + return self.run_runtime_scenarios(self.probe_key) + + # ── PT-OAPI1-01 — API object-level authorization bypass (BOLA) ────── + + def _test_api_bola(self): + if not self.scenario_enabled("PT-OAPI1-01"): + return + """For each configured ApiObjectEndpoint, iterate ``test_ids`` against + ``path`` (template) using the regular_session (or official_session if + no regular configured). Vulnerable iff response is 200 + JSON + + ``owner_field`` mismatches the authenticated username (or + ``tenant_field`` mismatches the expected tenant). + + Severity: + HIGH by default. + CRITICAL when leaked response contains PII-ish field NAMES. + """ + api_security = self.target_config.api_security + endpoints = api_security.object_endpoints + if not endpoints: + self.emit_inconclusive( + "PT-OAPI1-01", + "API object-level authorization bypass (BOLA)", + "API1:2023", + "no_configured_object_endpoints", + ) + return + session = self.auth.regular_session + if session is None: + self.emit_inconclusive( + "PT-OAPI1-01", + "API object-level authorization bypass (BOLA)", + "API1:2023", + "no_low_privileged_session", + ) + return + + found_any = False + for ep in endpoints: + for test_id in ep.test_ids: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI1-01", + "API object-level authorization bypass (BOLA)", + "API1:2023", + "budget_exhausted", + ) + return + url = self._render_object_url(ep, test_id) + self.safety.throttle() + try: + resp = session.get(url, timeout=10, allow_redirects=False) + except requests.RequestException as exc: + # Single-endpoint transport error → continue with next id. + # _record_error would also work but inflates noise. + continue + + outcome = self._evaluate_bola_response(ep, test_id, url, resp) + if outcome in ("vulnerable", "clean", "inconclusive"): + found_any = True + + if not found_any: + # Every iteration was inconclusive (HTML, 4xx, etc.) OR the config + # listed zero test_ids. Surface a single inconclusive so the + # operator knows the probe attempted but couldn't draw a conclusion. + self.emit_inconclusive( + "PT-OAPI1-01", + "API object-level authorization bypass (BOLA)", + "API1:2023", + "no_evaluable_responses", + ) + + def _render_object_url(self, ep, test_id): + """Substitute {id_param} into ep.path. Falls back to {id} for + backward compatibility with the typical Django/Flask convention.""" + path = ep.path + if "{" + ep.id_param + "}" in path: + path = path.replace("{" + ep.id_param + "}", str(test_id)) + elif "{id}" in path: + path = path.replace("{id}", str(test_id)) + else: + path = path.rstrip("/") + "/" + str(test_id) + return self.target_url + path + + def _evaluate_bola_response(self, ep, test_id, url, resp): + """Return ``"vulnerable"`` / ``"clean"`` / ``"skip"`` and emit the + appropriate finding for the single-id evaluation.""" + title = "API object-level authorization bypass (BOLA)" + owasp = "API1:2023" + cwe = ["CWE-639", "CWE-284"] + + # FP guard 1: skip non-API responses (web IDOR is AccessControlProbes' job). + content_type = (resp.headers.get("content-type") or "").lower() + if "application/json" not in content_type: + return "skip" + # FP guard 2: skip 4xx/5xx — endpoint forbade us, that's correct. + if resp.status_code >= 400: + return "skip" + # FP guard 3: must parse as JSON. + try: + data = resp.json() + except (ValueError, requests.exceptions.JSONDecodeError): + return "skip" + if not isinstance(data, dict): + return "skip" + tenant_field = (ep.tenant_field or "").strip() + owner_present = ep.owner_field in data + tenant_present = bool(tenant_field and tenant_field in data) + # FP guard 4: an expected owner/tenant field must be present. + if not owner_present and not tenant_present: + return "skip" + + expected_owner = (getattr(ep, "expected_owner", "") or self.regular_username or "").strip() + expected_tenant = (getattr(ep, "expected_tenant", "") or "").strip() + if not expected_owner and not expected_tenant: + self.emit_inconclusive( + "PT-OAPI1-01", title, owasp, "no_expected_owner_or_tenant", + ) + return "inconclusive" + + owner_value = str(data.get(ep.owner_field)) if owner_present else "" + tenant_value = str(data.get(tenant_field)) if tenant_present else "" + owner_mismatch = bool(owner_present and expected_owner + and owner_value != expected_owner) + tenant_mismatch = bool( + tenant_present and expected_tenant and tenant_value != expected_tenant + ) + if not owner_mismatch and not tenant_mismatch: + compared = bool((owner_present and expected_owner) + or (tenant_present and expected_tenant)) + if not compared: + self.emit_inconclusive( + "PT-OAPI1-01", title, owasp, "no_comparable_expected_owner_or_tenant", + ) + return "inconclusive" + + if owner_mismatch or tenant_mismatch: + sensitive_fields = self._collect_sensitive_field_names(data) + severity = "CRITICAL" if sensitive_fields else "HIGH" + evidence = [ + f"endpoint={url}", + "response_status=200", + "content_type=application/json", + f"owner_field={ep.owner_field}", + f"owner_value={owner_value}", + f"expected_owner={expected_owner}", + f"test_id={test_id}", + ] + if tenant_mismatch: + evidence.append(f"tenant_field={tenant_field}") + evidence.append(f"tenant_value={tenant_value}") + evidence.append(f"expected_tenant={expected_tenant}") + if sensitive_fields: + evidence.append("pii_fields=" + ",".join(sorted(sensitive_fields))) + replay = [ + "Authenticate as the regular (low-privileged) user.", + f"GET {url}", + f"Observe the response carries {ep.owner_field}={owner_value!r} " + "even though the requester is not the owner.", + ] + self.emit_vulnerable( + "PT-OAPI1-01", title, severity, owasp, cwe, evidence, + replay_steps=replay, + remediation=( + "Enforce per-object authorization on the endpoint: verify that " + "the requester owns the object (or shares its tenant) before " + "returning it. A path/query parameter is not an authorization " + "claim." + ), + ) + return "vulnerable" + + self.emit_clean( + "PT-OAPI1-01", title, owasp, + [f"endpoint={url}", "response_status=200", + f"owner_field={ep.owner_field}", + f"owner_value={owner_value}", + f"expected_owner={expected_owner}"], + ) + return "clean" + + # ── PT-OAPI5-01 — BFLA: regular user reaches admin function ───────── + + def _test_bfla_regular_as_admin(self): + if not self.scenario_enabled("PT-OAPI5-01"): + return + """For each ApiFunctionEndpoint with method == GET (read-only), + GET it as the regular_session and expect ≥401/403. + + Vulnerable iff status < 400 (no auth gate). Mutating endpoints + (method != GET) are deferred to PT-OAPI5-04 in Subphase 3.4 — they + require the stateful contract + a configured revert plan. + """ + api_security = self.target_config.api_security + endpoints = api_security.function_endpoints + if not endpoints: + self.emit_inconclusive( + "PT-OAPI5-01", + "API function-level authorization bypass (regular as admin, read)", + "API5:2023", + "no_configured_function_endpoints", + ) + return + session = self.auth.regular_session + if session is None: + self.emit_inconclusive( + "PT-OAPI5-01", + "API function-level authorization bypass (regular as admin, read)", + "API5:2023", + "no_regular_session", + ) + return + + found_any = self._run_function_endpoints( + endpoints, session, "regular", + scenario_id="PT-OAPI5-01", + title="API function-level authorization bypass (regular as admin, read)", + ) + if not found_any: + self.emit_inconclusive( + "PT-OAPI5-01", + "API function-level authorization bypass (regular as admin, read)", + "API5:2023", + "no_evaluable_function_endpoints", + ) + + # ── PT-OAPI5-02 — BFLA: anonymous user reaches user function ──────── + + def _test_bfla_anon_as_user(self): + if not self.scenario_enabled("PT-OAPI5-02"): + return + """Anonymous (unauthenticated) GET against each function endpoint. + + Same mechanics as PT-OAPI5-01 but uses + `auth.make_anonymous_session()` so caller cookies / Bearer headers + are not present. + """ + api_security = self.target_config.api_security + endpoints = api_security.function_endpoints + if not endpoints: + self.emit_inconclusive( + "PT-OAPI5-02", + "API function-level authorization bypass (anonymous as user, read)", + "API5:2023", + "no_configured_function_endpoints", + ) + return + if not hasattr(self.auth, "make_anonymous_session"): + self.emit_inconclusive( + "PT-OAPI5-02", + "API function-level authorization bypass (anonymous as user, read)", + "API5:2023", + "auth_manager_missing_anonymous_session", + ) + return + session = self.auth.make_anonymous_session() + found_any = self._run_function_endpoints( + endpoints, session, "anonymous", + scenario_id="PT-OAPI5-02", + title="API function-level authorization bypass (anonymous as user, read)", + ) + try: + session.close() + except Exception: + pass + if not found_any: + self.emit_inconclusive( + "PT-OAPI5-02", + "API function-level authorization bypass (anonymous as user, read)", + "API5:2023", + "no_evaluable_function_endpoints", + ) + + # ── Shared BFLA evaluator ──────────────────────────────────────────── + + def _run_function_endpoints(self, endpoints, session, principal, *, + scenario_id, title): + """Iterate function endpoints in read-only mode; emit per-endpoint + finding. Returns True iff at least one endpoint yielded a definitive + (vulnerable or clean) result.""" + cwe = ["CWE-285", "CWE-862"] + owasp = "API5:2023" + found_any = False + + for ep in endpoints: + # Phase 2.3 covers read-only (method=GET) only. Mutating methods + # are deferred to PT-OAPI5-03 / PT-OAPI5-04 (stateful, Phase 3.4). + if (ep.method or "GET").upper() not in ("GET", "HEAD"): + continue + + if not self.budget(): + self.emit_inconclusive( + scenario_id, title, owasp, "budget_exhausted", + ) + return found_any + + url = self.target_url + ep.path + self.safety.throttle() + try: + resp = session.get(url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + + status = resp.status_code + # Auth gate working as intended. + if status in (401, 403): + self.emit_clean( + scenario_id, title, owasp, + [f"endpoint={url}", f"principal={principal}", + f"response_status={status}", + "marker=auth_gate_returned_4xx"], + ) + found_any = True + continue + # Other 4xx/5xx — endpoint refused for other reasons; skip. + if status >= 400: + continue + + # 2xx/3xx without an auth-required marker = vulnerable. + body_lower = (resp.text or "").lower()[:2000] + marker = (ep.auth_required_marker or "").lower().strip() + marker_present = bool(marker and marker in body_lower) + if marker_present: + self.emit_clean( + scenario_id, title, owasp, + [f"endpoint={url}", f"principal={principal}", + f"response_status={status}", + "marker=configured_auth_required_marker_present"], + ) + found_any = True + continue + + # Severity: HIGH baseline; CRITICAL when path matches /admin or + # function_endpoint is explicitly tagged privilege=admin. + privilege = (ep.privilege or "").lower() + severity = "CRITICAL" if (privilege == "admin" + or "/admin" in ep.path.lower()) else "HIGH" + evidence = [ + f"endpoint={url}", f"principal={principal}", + f"response_status={status}", + f"method={(ep.method or 'GET').upper()}", + "marker_absent=true", + ] + replay = [ + f"Authenticate as the {principal} user (or none for anonymous).", + f"GET {url}", + "Observe a 2xx response — the endpoint did not enforce its " + "intended authorization.", + ] + self.emit_vulnerable( + scenario_id, title, severity, owasp, cwe, evidence, + replay_steps=replay, + remediation=( + "Add the appropriate authorization decorator/middleware on the " + "endpoint. For administrative functions verify that the caller " + "has the required role; for user-only functions require an " + "authenticated session. Returning 2xx to the wrong principal " + "leaks data or exposes side effects." + ), + ) + found_any = True + + return found_any + + # ── PT-OAPI5-03 — Method-override bypass (STATEFUL) ──────────────── + + def _test_bfla_method_override(self): + if not self.scenario_enabled("PT-OAPI5-03"): + return + title = "API method-override authorization bypass" + owasp = "API5:2023" + api_security = self.target_config.api_security + if not api_security.function_endpoints: + self.emit_inconclusive( + "PT-OAPI5-03", title, owasp, "no_configured_function_endpoints", + ) + return + session = self.auth.regular_session + if session is None: + self.emit_inconclusive("PT-OAPI5-03", title, owasp, "no_regular_session") + return + + for ep in api_security.function_endpoints: + method = (ep.method or "GET").upper() + if method == "GET": + # Method-override target should be a method-restricted endpoint + # — GET-only endpoints have nothing to override. + continue + if not ep.revert_path: + self.emit_inconclusive( + "PT-OAPI5-03", title, owasp, "no_revert_path_configured", + ) + continue + + url = self.target_url + ep.path + revert_url = self.target_url + ep.revert_path + method_fn = getattr(session, method.lower(), session.post) + evidence = [f"endpoint={url}", + "override_header=X-HTTP-Method-Override: GET"] + + def baseline(_ep=ep, _url=url): + return {"method": method, "ep_path": _ep.path} + + def mutate(base, _ep=ep, _url=url, _method_fn=method_fn, + _evidence=evidence): + if not self.budget(): + return False + self.safety.throttle() + try: + plain_resp = _method_fn(_url, timeout=10, allow_redirects=False) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + base["plain_status"] = plain_resp.status_code + _evidence.append(f"plain_status={plain_resp.status_code}") + if plain_resp.status_code < 400: + base["plain_mutating_method_allowed"] = True + return True + if plain_resp.status_code not in (401, 403): + base["plain_mutating_method_unexpected_status"] = plain_resp.status_code + return False + + if not self.budget(): + return False + self.safety.throttle() + try: + resp = session.post( + _url, headers={"X-HTTP-Method-Override": "GET"}, + timeout=10, allow_redirects=False, + ) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + base["override_status"] = resp.status_code + _evidence.append(f"override_status={resp.status_code}") + return resp.status_code < 400 + + def verify(base): + if base.get("plain_mutating_method_allowed"): + return False + return base.get("override_status", 999) < 400 + + def revert(base, _revert_url=revert_url, _ep=ep): + return self._revert_function_endpoint(session, _revert_url, _ep) + + def mutation_unverified_reason(base, rollback_status): + if base.get("plain_mutating_method_allowed"): + if rollback_status == "revert_failed": + return "plain_mutating_method_allowed_revert_failed" + return "plain_mutating_method_allowed" + return "" + + def no_mutation_reason(base): + status = base.get("plain_mutating_method_unexpected_status") + if status is not None: + return f"plain_mutating_method_status_{status}" + return "" + + self.run_stateful( + "PT-OAPI5-03", + baseline_fn=baseline, + mutate_fn=mutate, + verify_fn=verify, + revert_fn=revert, + finding_kwargs={ + "title": title, "owasp": owasp, "severity": "HIGH", + "cwe": ["CWE-285", "CWE-862"], + "evidence": evidence, + "remediation": ( + "Disable HTTP method override entirely or restrict it to " + "internal services. Authorization must be enforced on the " + "effective method used." + ), + }, + mutation_unverified_reason_fn=mutation_unverified_reason, + no_mutation_reason_fn=no_mutation_reason, + ) + + # ── PT-OAPI5-04 — Regular user reaches admin function (MUTATING) ─── + + def _test_bfla_regular_as_admin_mutating(self): + if not self.scenario_enabled("PT-OAPI5-04"): + return + title = "API function-level authorization bypass (regular as admin, mutating)" + owasp = "API5:2023" + api_security = self.target_config.api_security + if not api_security.function_endpoints: + self.emit_inconclusive( + "PT-OAPI5-04", title, owasp, "no_configured_function_endpoints", + ) + return + session = self.auth.regular_session + if session is None: + self.emit_inconclusive("PT-OAPI5-04", title, owasp, "no_regular_session") + return + + for ep in api_security.function_endpoints: + method = (ep.method or "GET").upper() + if method == "GET": + continue + if not ep.revert_path: + self.emit_inconclusive( + "PT-OAPI5-04", title, owasp, "no_revert_path_configured", + ) + continue + + url = self.target_url + ep.path + revert_url = self.target_url + ep.revert_path + method_fn = getattr(session, method.lower(), session.post) + + def baseline(_ep=ep): + return {"method": method, "ep_path": _ep.path} + + def mutate(base, _url=url, _method_fn=method_fn): + if not self.budget(): + return False + self.safety.throttle() + try: + resp = _method_fn(_url, timeout=10) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + base["mutate_status"] = resp.status_code + return resp.status_code < 400 + + def verify(base): + return base.get("mutate_status", 999) < 400 + + def revert(base, _revert_url=revert_url, _ep=ep): + return self._revert_function_endpoint(session, _revert_url, _ep) + + privilege = (ep.privilege or "").lower() + severity = ("CRITICAL" + if privilege == "admin" or "/admin" in ep.path.lower() + else "HIGH") + self.run_stateful( + "PT-OAPI5-04", + baseline_fn=baseline, + mutate_fn=mutate, + verify_fn=verify, + revert_fn=revert, + finding_kwargs={ + "title": title, "owasp": owasp, "severity": severity, + "cwe": ["CWE-285", "CWE-862"], + "evidence": [f"endpoint={url}", f"method={method}", + "principal=regular"], + "remediation": ( + "Verify the caller's role on every mutating endpoint. The " + "URL alone is not an authorization claim — admin actions " + "must check the session/JWT role on the server." + ), + }, + ) + + def _revert_function_endpoint(self, session, revert_url, ep) -> bool: + if not self.cleanup_budget(): + return False + self.safety.throttle() + try: + resp = session.post(revert_url, json=ep.revert_body or {}, timeout=10) + except requests.RequestException: + return False + return resp.status_code < 400 + + @staticmethod + def _collect_sensitive_field_names(payload): + """Return the subset of top-level keys in ``payload`` whose names + match a PII pattern. Values are never inspected.""" + found = set() + for key in (payload.keys() if isinstance(payload, dict) else ()): + if not isinstance(key, str): + continue + for pat in _BOLA_PII_FIELD_PATTERNS: + if pat.search(key): + found.add(key) + break + return found diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/api_auth.py b/extensions/business/cybersec/red_mesh/graybox/probes/api_auth.py new file mode 100644 index 00000000..7ba708d1 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/api_auth.py @@ -0,0 +1,363 @@ +"""API authentication probes — OWASP API2 (Broken Authentication).""" + +import base64 +import hashlib +import hmac +import json + +import requests + +from .base import ProbeBase + + +def _b64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode() + + +def _b64url_decode(s: str) -> bytes: + pad = "=" * (-len(s) % 4) + return base64.urlsafe_b64decode(s + pad) + + +def _forge_jwt(header: dict, payload: dict, secret: str | None = None) -> str: + h = _b64url(json.dumps(header, separators=(",", ":")).encode()) + p = _b64url(json.dumps(payload, separators=(",", ":")).encode()) + signing_input = f"{h}.{p}".encode() + if header.get("alg") == "none": + return f"{h}.{p}." + if header.get("alg") == "HS256" and secret is not None: + sig = hmac.new(secret.encode(), signing_input, hashlib.sha256).digest() + return f"{h}.{p}.{_b64url(sig)}" + return f"{h}.{p}." + + +def _decode_jwt_payload(token: str) -> dict | None: + try: + parts = token.split(".") + if len(parts) < 2: + return None + return json.loads(_b64url_decode(parts[1])) + except Exception: + return None + + +class ApiAuthProbes(ProbeBase): + """OWASP API2 (Broken Authentication) graybox probes. + + Scenarios: + PT-OAPI2-01 — JWT missing-signature (alg=none) accepted. + PT-OAPI2-02 — JWT signed with weak HMAC secret. + PT-OAPI2-03 — Token not invalidated on logout (stateful, re-auth revert). + """ + + requires_auth = True + requires_regular_session = False + is_stateful = False + probe_key = "_graybox_api_auth" + + def run(self): + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return self.findings + return self.run_runtime_scenarios(self.probe_key) + + # ── helpers ──────────────────────────────────────────────────────── + + def _obtain_token(self, *, consume_budget: bool = True): + """Return (token, raw_payload) from token_path or configured bearer session.""" + tok = self.target_config.api_security.token_endpoints + session = self.auth.official_session or self.auth.regular_session + if session is None: + return None, None + if not tok.token_path: + token = self._configured_session_bearer_token(session) + return (token, {"source": "configured_bearer_token"}) if token else (None, None) + if consume_budget and not self.budget(): + return None, None + url = self.target_url + tok.token_path + method = (getattr(tok, "token_request_method", "POST") or "POST").upper() + body = dict(getattr(tok, "token_request_body", {}) or {}) + self.safety.throttle() + try: + req = getattr(session, method.lower(), session.post) + if method in ("GET", "DELETE"): + resp = req(url, params=body, timeout=10) + else: + resp = req(url, json=body if body else None, timeout=10) + except requests.RequestException: + return None, None + if resp.status_code >= 400: + return None, None + try: + data = resp.json() + except (ValueError, requests.exceptions.JSONDecodeError): + return None, None + field = (getattr(tok, "token_response_field", "") or "").strip() + token = data.get(field) if field else None + token = token or data.get("token") or data.get("access_token") or data.get("jwt") or "" + return token, data + + def _auth_descriptor(self): + api_security = getattr(self.target_config, "api_security", None) + auth = getattr(api_security, "auth", None) if api_security is not None else None + if auth is None: + from ..models.target_config import AuthDescriptor + return AuthDescriptor() + return auth + + def _configured_session_bearer_token(self, session) -> str: + auth = self._auth_descriptor() + header_name = getattr(auth, "bearer_token_header_name", "Authorization") or "Authorization" + raw = "" + try: + raw = (session.headers or {}).get(header_name, "") or "" + except Exception: + raw = "" + scheme = getattr(auth, "bearer_scheme", "Bearer") or "" + if scheme and raw.lower().startswith((scheme + " ").lower()): + raw = raw[len(scheme):].strip() + return raw if raw.count(".") == 2 else "" + + def _auth_headers_for_token(self, token: str) -> dict: + auth = self._auth_descriptor() + header_name = getattr(auth, "bearer_token_header_name", "Authorization") or "Authorization" + scheme = getattr(auth, "bearer_scheme", "Bearer") or "Bearer" + value = f"{scheme} {token}".strip() if scheme else token + return {header_name: value} + + # ── PT-OAPI2-01 — alg=none ──────────────────────────────────────── + + def _test_jwt_alg_none(self): + if not self.scenario_enabled("PT-OAPI2-01"): + return + title = "API JWT missing-signature accepted (alg=none)" + owasp = "API2:2023" + tok = self.target_config.api_security.token_endpoints + if not tok.protected_path: + self.emit_inconclusive( + "PT-OAPI2-01", title, owasp, "no_protected_path_configured", + ) + return + real_token, _ = self._obtain_token() + if not real_token: + self.emit_inconclusive( + "PT-OAPI2-01", title, owasp, "token_issuance_failed", + ) + return + original_payload = _decode_jwt_payload(real_token) or {} + forged_payload = dict(original_payload) + forged_payload["is_admin"] = True + forged = _forge_jwt({"alg": "none", "typ": "JWT"}, forged_payload) + + url = self.target_url + tok.protected_path + if not self.budget(): + return + self.safety.throttle() + session = self.auth.make_anonymous_session() + try: + resp = session.get( + url, headers=self._auth_headers_for_token(forged), + timeout=10, + allow_redirects=False, + ) + except requests.RequestException: + self.emit_inconclusive( + "PT-OAPI2-01", title, owasp, "protected_path_transport_error", + ) + return + finally: + session.close() + + if resp.status_code < 400: + self.emit_vulnerable( + "PT-OAPI2-01", title, + "CRITICAL", owasp, ["CWE-347", "CWE-327"], + [f"token_path={tok.token_path}", + f"protected_path={tok.protected_path}", + "forged_alg=none", + f"forged_claim=is_admin", + f"server_returned_status={resp.status_code}"], + remediation=( + "Reject JWTs with alg=none unconditionally. Verify the signing " + "algorithm against an explicit allowlist before signature " + "verification." + ), + ) + else: + self.emit_clean( + "PT-OAPI2-01", title, owasp, + [f"forged_alg=none", f"server_returned_status={resp.status_code}"], + ) + + # ── PT-OAPI2-02 — weak HMAC secret ─────────────────────────────── + + def _test_jwt_weak_hmac(self): + if not self.scenario_enabled("PT-OAPI2-02"): + return + title = "API JWT signed with weak HMAC secret" + owasp = "API2:2023" + tok = self.target_config.api_security.token_endpoints + if not tok.protected_path: + self.emit_inconclusive( + "PT-OAPI2-02", title, owasp, "no_protected_path_configured", + ) + return + real_token, _ = self._obtain_token() + if not real_token: + self.emit_inconclusive( + "PT-OAPI2-02", title, owasp, "token_issuance_failed", + ) + return + parts = real_token.split(".") + if len(parts) != 3: + self.emit_inconclusive( + "PT-OAPI2-02", title, owasp, "token_not_jwt_shape", + ) + return + header_b64, payload_b64, sig_b64 = parts + signing_input = f"{header_b64}.{payload_b64}".encode() + try: + sig = _b64url_decode(sig_b64) + except Exception: + self.emit_inconclusive( + "PT-OAPI2-02", title, owasp, "token_signature_not_base64", + ) + return + + candidates = list( + self.target_config.api_security.token_endpoints.weak_secret_candidates + ) + for secret in candidates: + if not secret: + continue + try: + expected = hmac.new(secret.encode(), signing_input, + hashlib.sha256).digest() + except Exception: + continue + if hmac.compare_digest(expected, sig): + self.emit_vulnerable( + "PT-OAPI2-02", title, + "HIGH", owasp, ["CWE-327", "CWE-521"], + [f"weak_secret_was=", + f"token_prefix={real_token[:8]}", + "verification=local_HS256_compare"], + remediation=( + "Rotate the JWT signing secret to a high-entropy value (≥32 " + "random bytes). Store the secret in a secret manager, not in " + "source / env defaults / framework boilerplate." + ), + ) + return + self.emit_clean( + "PT-OAPI2-02", title, owasp, + [f"candidates_tested={len(candidates)}", + f"token_prefix={real_token[:8]}"], + ) + + # ── PT-OAPI2-03 — Logout doesn't invalidate (STATEFUL) ─────────── + + def _test_token_logout_invalidation(self): + if not self.scenario_enabled("PT-OAPI2-03"): + return + title = "API token not invalidated on logout" + owasp = "API2:2023" + tok = self.target_config.api_security.token_endpoints + if not tok.protected_path: + self.emit_inconclusive( + "PT-OAPI2-03", title, owasp, "no_protected_path_configured", + ) + return + if not tok.logout_path: + self.emit_inconclusive( + "PT-OAPI2-03", title, owasp, "no_logout_path_configured", + ) + return + if not tok.token_path: + self.emit_inconclusive( + "PT-OAPI2-03", title, owasp, "disposable_logout_token_required", + ) + return + + real_token = [None] + no_mutation_reason = [""] + + def baseline(): + t, _ = self._obtain_token() + real_token[0] = t + if not t: + raise RuntimeError("token_issuance_failed") + return t + + def mutate(base): + if not self.budget(): + no_mutation_reason[0] = "budget_exhausted" + return False + url = self.target_url + tok.logout_path + self.safety.throttle() + session = self.auth.make_anonymous_session() + try: + resp = session.post( + url, headers=self._auth_headers_for_token(base), + timeout=10, allow_redirects=False, + ) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + finally: + session.close() + accepted = resp.status_code < 400 + if not accepted: + no_mutation_reason[0] = f"logout_status={resp.status_code}" + return accepted + + def verify(base): + if not self.budget(): + raise RuntimeError("budget_exhausted") + url = self.target_url + tok.protected_path + session = self.auth.make_anonymous_session() + try: + resp = session.get( + url, headers=self._auth_headers_for_token(base), + timeout=10, allow_redirects=False, + ) + except requests.RequestException as exc: + raise RuntimeError("protected_path_transport_error") from exc + finally: + session.close() + # Vulnerable iff protected path STILL accepts the supposedly-revoked token. + return resp.status_code < 400 + + def revert(base): + fresh_token, _ = self._obtain_token(consume_budget=False) + return bool(fresh_token) + + self.run_stateful( + "PT-OAPI2-03", + baseline_fn=baseline, + mutate_fn=mutate, + verify_fn=verify, + revert_fn=revert, + no_mutation_reason_fn=lambda base: ( + no_mutation_reason[0] or "logout_request_not_accepted" + ), + clean_when_verify_false=True, + finding_kwargs={ + "title": title, "owasp": owasp, "severity": "MEDIUM", + "cwe": ["CWE-613"], + "evidence": [f"token_path={tok.token_path}", + f"logout_path={tok.logout_path}", + f"protected_path={tok.protected_path}"], + "replay_steps": [ + "POST to token_path and capture the issued bearer token.", + "POST to logout_path with that token.", + "GET protected_path with the same token after logout.", + "Observe the protected path still returns 2xx — the token " + "was not invalidated.", + ], + "remediation": ( + "Track issued JWTs server-side (e.g., a revocation list keyed " + "on `jti`) and reject revoked tokens on every request. " + "Pure-stateless JWTs cannot enforce logout." + ), + }, + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/api_config.py b/extensions/business/cybersec/red_mesh/graybox/probes/api_config.py new file mode 100644 index 00000000..e529d74a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/api_config.py @@ -0,0 +1,501 @@ +"""API misconfiguration + inventory probes — OWASP API8 and API9.""" + +import re + +import requests + +from .base import ProbeBase + + +_DEBUG_BODY_MARKERS = ( + re.compile(r"(?i)\btraceback\b"), + re.compile(r"(?i)\bstack trace\b"), + re.compile(r"(?i)\bdebug\b"), + re.compile(r"(?i)\bDEBUG\s*=\s*True"), + re.compile(r"(?i)at\s+/(?:usr|home|opt|app)/"), + re.compile(r"(?i)urlpattern"), + re.compile(r"\"swagger\"\s*:"), + re.compile(r"\"openapi\"\s*:"), +) + +_VERBOSE_ERROR_MARKERS = ( + re.compile(r"(?i)\bTraceback\b"), + re.compile(r"(?i)Exception"), + re.compile(r"(?i)Stack trace"), + re.compile(r"(?i)at\s+/(?:usr|home|opt|app)/"), + re.compile(r"(?i)line\s+\d+"), + re.compile(r"(?i)Werkzeug|Flask|Django|FastAPI"), +) + + +class ApiConfigProbes(ProbeBase): + """OWASP API8 + API9 graybox probes. + + Scenarios implemented (Subphases 2.4 + 2.5): + PT-OAPI8-01 / 02 / 03 / 04 / 05 — Subphase 2.4 + PT-OAPI9-01 / 02 / 03 — Subphase 2.5 + """ + + requires_auth = True + requires_regular_session = False + is_stateful = False + probe_key = "_graybox_api_config" + + def run(self): + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return self.findings + return self.run_runtime_scenarios(self.probe_key) + + # ── helpers ──────────────────────────────────────────────────────── + + def _session(self): + return self.auth.official_session or self.auth.regular_session + + def _anon_session(self): + if hasattr(self.auth, "make_anonymous_session"): + try: + return self.auth.make_anonymous_session() + except Exception: + return None + return None + + # ── PT-OAPI8-01 — Permissive CORS ───────────────────────────────── + + def _test_cors_misconfig(self): + if not self.scenario_enabled("PT-OAPI8-01"): + return + api_security = self.target_config.api_security + if not api_security.function_endpoints: + self.emit_inconclusive( + "PT-OAPI8-01", "API permissive CORS configuration", + "API8:2023", "no_configured_function_endpoints", + ) + return + session = self._session() + if session is None: + self.emit_inconclusive( + "PT-OAPI8-01", "API permissive CORS configuration", + "API8:2023", "no_authenticated_session", + ) + return + found_any = False + for ep in api_security.function_endpoints: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI8-01", "API permissive CORS configuration", + "API8:2023", "budget_exhausted", + ) + return + url = self.target_url + ep.path + self.safety.throttle() + try: + resp = session.get( + url, headers={"Origin": "https://evil.example"}, + timeout=10, allow_redirects=False, + ) + except requests.RequestException: + continue + + acao = resp.headers.get("Access-Control-Allow-Origin", "") + acac = (resp.headers.get("Access-Control-Allow-Credentials") or "").lower() + origin_echoes_evil = "evil.example" in acao + wildcard_with_creds = acao == "*" and acac == "true" + + if wildcard_with_creds or (origin_echoes_evil and acac == "true"): + severity = "HIGH" + elif acao == "*": + severity = "LOW" + else: + self.emit_clean( + "PT-OAPI8-01", "API permissive CORS configuration", "API8:2023", + [f"endpoint={url}", f"acao={acao or ''}", + f"acac={acac or ''}"], + ) + found_any = True + continue + + self.emit_vulnerable( + "PT-OAPI8-01", "API permissive CORS configuration", + severity, "API8:2023", ["CWE-942"], + [f"endpoint={url}", f"acao={acao}", f"acac={acac}", + f"sent_origin=https://evil.example"], + remediation=( + "Replace permissive CORS with an explicit allowlist of trusted " + "origins. Never echo an arbitrary Origin alongside " + "Access-Control-Allow-Credentials: true." + ), + ) + found_any = True + if not found_any: + self.emit_inconclusive( + "PT-OAPI8-01", "API permissive CORS configuration", + "API8:2023", "no_evaluable_responses", + ) + + # ── PT-OAPI8-02 — Missing security headers ──────────────────────── + + def _test_security_headers(self): + if not self.scenario_enabled("PT-OAPI8-02"): + return + api_security = self.target_config.api_security + if not api_security.function_endpoints: + self.emit_inconclusive( + "PT-OAPI8-02", "API response missing security headers", + "API8:2023", "no_configured_function_endpoints", + ) + return + session = self._session() + if session is None: + self.emit_inconclusive( + "PT-OAPI8-02", "API response missing security headers", + "API8:2023", "no_authenticated_session", + ) + return + for ep in api_security.function_endpoints: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI8-02", "API response missing security headers", + "API8:2023", "budget_exhausted", + ) + return + url = self.target_url + ep.path + self.safety.throttle() + try: + resp = session.get(url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + missing = [] + headers_lower = {k.lower(): v for k, v in resp.headers.items()} + if "x-content-type-options" not in headers_lower: + missing.append("X-Content-Type-Options") + if self.target_url.startswith("https") and \ + "strict-transport-security" not in headers_lower: + missing.append("Strict-Transport-Security") + if "cache-control" not in headers_lower: + missing.append("Cache-Control") + if missing: + self.emit_vulnerable( + "PT-OAPI8-02", "API response missing security headers", + "LOW", "API8:2023", ["CWE-693"], + [f"endpoint={url}", "missing_headers=" + ",".join(missing)], + remediation=( + "Set the missing security headers via middleware. " + "X-Content-Type-Options: nosniff and a sensible Cache-Control " + "are appropriate on every API response; " + "Strict-Transport-Security is mandatory over HTTPS." + ), + ) + else: + self.emit_clean( + "PT-OAPI8-02", "API response missing security headers", + "API8:2023", + [f"endpoint={url}", "all_expected_headers_present"], + ) + + # ── PT-OAPI8-03 — Debug endpoint exposed ───────────────────────── + + def _test_debug_endpoint_exposed(self): + if not self.scenario_enabled("PT-OAPI8-03"): + return + api_security = self.target_config.api_security + session = self._session() + if session is None: + self.emit_inconclusive( + "PT-OAPI8-03", "API debug endpoint exposed", + "API8:2023", "no_authenticated_session", + ) + return + for path in api_security.debug_path_candidates: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI8-03", "API debug endpoint exposed", + "API8:2023", "budget_exhausted", + ) + return + url = self.target_url + path + self.safety.throttle() + try: + resp = session.get(url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + if resp.status_code >= 400: + continue + body = (resp.text or "")[:2000] + if any(p.search(body) for p in _DEBUG_BODY_MARKERS): + self.emit_vulnerable( + "PT-OAPI8-03", "API debug endpoint exposed", + "MEDIUM", "API8:2023", ["CWE-200", "CWE-215"], + [f"endpoint={url}", f"response_status={resp.status_code}", + "debug_markers_present=true"], + remediation=( + "Remove debug / introspection endpoints from production " + "deployments. If they must exist, gate them behind a " + "non-public network or strong authentication." + ), + ) + + # ── PT-OAPI8-04 — Verbose error response ───────────────────────── + + def _test_verbose_error(self): + if not self.scenario_enabled("PT-OAPI8-04"): + return + api_security = self.target_config.api_security + if not api_security.function_endpoints: + self.emit_inconclusive( + "PT-OAPI8-04", "API verbose error response leaks internals", + "API8:2023", "no_configured_function_endpoints", + ) + return + session = self._session() + if session is None: + self.emit_inconclusive( + "PT-OAPI8-04", "API verbose error response leaks internals", + "API8:2023", "no_authenticated_session", + ) + return + opted_in = [ep for ep in api_security.function_endpoints + if getattr(ep, "allow_malformed_json_probe", False)] + if not opted_in: + self.emit_inconclusive( + "PT-OAPI8-04", "API verbose error response leaks internals", + "API8:2023", "malformed_json_probe_not_authorized", + ) + return + for ep in opted_in: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI8-04", "API verbose error response leaks internals", + "API8:2023", "budget_exhausted", + ) + return + url = self.target_url + ep.path + self.safety.throttle() + try: + resp = session.post( + url, data='{"x":', headers={"Content-Type": "application/json"}, + timeout=10, allow_redirects=False, + ) + except requests.RequestException: + continue + body = (resp.text or "")[:2000] + if any(p.search(body) for p in _VERBOSE_ERROR_MARKERS): + self.emit_vulnerable( + "PT-OAPI8-04", "API verbose error response leaks internals", + "MEDIUM", "API8:2023", ["CWE-209"], + [f"endpoint={url}", f"response_status={resp.status_code}", + "stack_trace_or_framework_marker=present"], + remediation=( + "Catch unhandled exceptions and return a generic error body. " + "Detailed exception traces belong in server logs, not API " + "responses." + ), + ) + + # ── PT-OAPI8-05 — Unexpected methods ───────────────────────────── + + def _test_unexpected_methods(self): + if not self.scenario_enabled("PT-OAPI8-05"): + return + api_security = self.target_config.api_security + if not api_security.function_endpoints: + self.emit_inconclusive( + "PT-OAPI8-05", "API advertises unexpected HTTP methods", + "API8:2023", "no_configured_function_endpoints", + ) + return + session = self._session() + if session is None: + self.emit_inconclusive( + "PT-OAPI8-05", "API advertises unexpected HTTP methods", + "API8:2023", "no_authenticated_session", + ) + return + risky = {"TRACE", "PUT", "DELETE", "PATCH"} + for ep in api_security.function_endpoints: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI8-05", "API advertises unexpected HTTP methods", + "API8:2023", "budget_exhausted", + ) + return + url = self.target_url + ep.path + self.safety.throttle() + try: + resp = session.options(url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + allow = (resp.headers.get("Allow") or "").upper() + advertised = {m.strip() for m in allow.split(",") if m.strip()} + offenders = advertised & risky + # Skip when ep itself uses a mutating method legitimately. + expected = {(ep.method or "GET").upper()} + surprising = offenders - expected + if surprising: + self.emit_vulnerable( + "PT-OAPI8-05", "API advertises unexpected HTTP methods", + "LOW", "API8:2023", ["CWE-693"], + [f"endpoint={url}", "allow_header=" + allow, + "unexpected_methods=" + ",".join(sorted(surprising))], + remediation=( + "Restrict the endpoint's accepted HTTP methods to what it " + "actually uses. TRACE is rarely needed in production; " + "DELETE / PUT / PATCH should be present only on resources " + "that genuinely require them." + ), + ) + + # ── PT-OAPI9-01 — OpenAPI exposed ──────────────────────────────── + + def _test_openapi_exposed(self): + if not self.scenario_enabled("PT-OAPI9-01"): + return + api_security = self.target_config.api_security + inv = api_security.inventory_paths + session = self._anon_session() or self._session() + if session is None: + return + for path in inv.openapi_candidates: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI9-01", "API OpenAPI/Swagger specification publicly exposed", + "API9:2023", "budget_exhausted", + ) + return + url = self.target_url + path + self.safety.throttle() + try: + resp = session.get(url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + if resp.status_code >= 400: + continue + try: + data = resp.json() + except (ValueError, requests.exceptions.JSONDecodeError): + continue + if not isinstance(data, dict): + continue + if not (data.get("openapi") or data.get("swagger")): + continue + + spec_paths = list((data.get("paths") or {}).keys()) + private = [] + for p in spec_paths: + for pat in inv.private_path_patterns: + if pat in p: + private.append(p) + break + severity = "MEDIUM" if private else "LOW" + ev = [f"path={url}", f"status={resp.status_code}", + f"spec_paths_count={len(spec_paths)}", + f"private_paths_count={len(private)}"] + if private: + ev.append("private_path_examples=" + ",".join(private[:3])) + self.emit_vulnerable( + "PT-OAPI9-01", "API OpenAPI/Swagger specification publicly exposed", + severity, "API9:2023", ["CWE-1059", "CWE-538"], ev, + remediation=( + "Gate the OpenAPI/Swagger doc behind authentication, or " + "publish only a curated subset of the spec covering public " + "endpoints. Treat the unfiltered spec as if it were the source " + "code — it advertises every internal route." + ), + ) + return # one spec is enough + self.emit_clean( + "PT-OAPI9-01", "API OpenAPI/Swagger specification publicly exposed", + "API9:2023", ["no_exposed_spec_at_candidates"], + ) + + # ── PT-OAPI9-02 — Version sprawl ───────────────────────────────── + + def _test_version_sprawl(self): + if not self.scenario_enabled("PT-OAPI9-02"): + return + api_security = self.target_config.api_security + inv = api_security.inventory_paths + if not inv.current_version or not inv.canonical_probe_path: + self.emit_inconclusive( + "PT-OAPI9-02", "API legacy version still live (version sprawl)", + "API9:2023", "no_current_version_or_canonical_probe_path", + ) + return + session = self._session() + if session is None: + return + current = inv.current_version.rstrip("/") + canonical = inv.canonical_probe_path + if not canonical.startswith("/"): + canonical = "/" + canonical + + for sibling in inv.version_sibling_candidates: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI9-02", "API legacy version still live (version sprawl)", + "API9:2023", "budget_exhausted", + ) + return + sib = sibling.rstrip("/") + if sib == current: + continue + sib_path = canonical.replace(current, sib, 1) + sib_url = self.target_url + sib_path + self.safety.throttle() + try: + resp = session.get(sib_url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + if 200 <= resp.status_code < 300: + self.emit_vulnerable( + "PT-OAPI9-02", "API legacy version still live (version sprawl)", + "MEDIUM", "API9:2023", ["CWE-1059", "CWE-538"], + [f"current_version={current}", f"sibling={sib}", + f"sibling_url={sib_url}", + f"sibling_status={resp.status_code}"], + remediation=( + "Decommission legacy API versions or gate them behind a " + "deprecation policy. Live siblings often skip the security " + "fixes applied to the current version." + ), + ) + + # ── PT-OAPI9-03 — Deprecated still live ───────────────────────── + + def _test_deprecated_live(self): + if not self.scenario_enabled("PT-OAPI9-03"): + return + api_security = self.target_config.api_security + inv = api_security.inventory_paths + if not inv.deprecated_paths: + self.emit_inconclusive( + "PT-OAPI9-03", "API deprecated path still serving requests", + "API9:2023", "no_deprecated_paths_configured", + ) + return + session = self._session() + if session is None: + return + for path in inv.deprecated_paths: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI9-03", "API deprecated path still serving requests", + "API9:2023", "budget_exhausted", + ) + return + url = self.target_url + path + self.safety.throttle() + try: + resp = session.get(url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + if 200 <= resp.status_code < 300: + self.emit_vulnerable( + "PT-OAPI9-03", "API deprecated path still serving requests", + "MEDIUM", "API9:2023", ["CWE-1059"], + [f"endpoint={url}", f"status={resp.status_code}"], + remediation=( + "Return 410 Gone (or a hard redirect to the supported " + "endpoint) on deprecated paths." + ), + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/api_data.py b/extensions/business/cybersec/red_mesh/graybox/probes/api_data.py new file mode 100644 index 00000000..629d9459 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/probes/api_data.py @@ -0,0 +1,296 @@ +"""API data-exposure probes — OWASP API3 (BOPLA).""" + +import re + +import requests + +from .base import ProbeBase + + +# Built-in sensitive property-name regexes for PT-OAPI3-01. Operators +# can extend via `target_config.api_security.sensitive_field_patterns`. +_DEFAULT_SENSITIVE_PATTERNS = ( + re.compile(r"(?i)\bpassword"), + re.compile(r"(?i)_hash\b"), + re.compile(r"(?i)\btoken\b"), + re.compile(r"(?i)\bsecret\b"), + re.compile(r"(?i)\bapi[_-]?key\b"), + re.compile(r"(?i)\bprivate[_-]?key\b"), + re.compile(r"(?i)\bmfa[_-]?secret\b"), + re.compile(r"(?i)\brecovery[_-]?code"), + re.compile(r"(?i)_ssn\b"), + re.compile(r"(?i)_cc[_-]?number\b"), + re.compile(r"(?i)\bis[_-]?admin\b"), + re.compile(r"(?i)\bis[_-]?superuser\b"), +) + + +class ApiDataProbes(ProbeBase): + """OWASP API3 (Broken Object Property Level Authorization) probes. + + Scenarios: + PT-OAPI3-01 — API response leaks sensitive properties (Subphase 2.2). + PT-OAPI3-02 — API accepts mass assignment of privileged properties + (stateful; Subphase 3.1; uses ProbeBase.run_stateful). + """ + + requires_auth = True + requires_regular_session = False + is_stateful = False + probe_key = "_graybox_api_data" + + def run(self): + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return self.findings + return self.run_runtime_scenarios(self.probe_key) + + # ── PT-OAPI3-01 — Excessive property exposure ───────────────────── + + def _test_api_property_exposure(self): + if not self.scenario_enabled("PT-OAPI3-01"): + return + api_security = self.target_config.api_security + endpoints = api_security.property_endpoints + if not endpoints: + self.emit_inconclusive( + "PT-OAPI3-01", "API response leaks sensitive properties", + "API3:2023", "no_configured_property_endpoints", + ) + return + session = self.auth.regular_session or self.auth.official_session + if session is None: + self.emit_inconclusive( + "PT-OAPI3-01", "API response leaks sensitive properties", + "API3:2023", "no_authenticated_session", + ) + return + + patterns = list(_DEFAULT_SENSITIVE_PATTERNS) + for raw in getattr(api_security, "sensitive_field_patterns", []) or []: + try: + patterns.append(re.compile(raw, re.IGNORECASE)) + except re.error: + continue + + found_any = False + for ep in endpoints: + if not self.budget(): + self.emit_inconclusive( + "PT-OAPI3-01", "API response leaks sensitive properties", + "API3:2023", "budget_exhausted", + ) + return + url = self._render_url(ep.path, ep.id_param, ep.test_id) + self.safety.throttle() + try: + resp = session.get(url, timeout=10, allow_redirects=False) + except requests.RequestException: + continue + + if resp.status_code >= 400: + continue + ct = (resp.headers.get("content-type") or "").lower() + if "application/json" not in ct: + continue + try: + data = resp.json() + except (ValueError, requests.exceptions.JSONDecodeError): + continue + if not isinstance(data, dict): + continue + + leaks = self._find_sensitive_keys(data, patterns) + if leaks: + self.emit_vulnerable( + "PT-OAPI3-01", "API response leaks sensitive properties", + "HIGH", "API3:2023", ["CWE-213", "CWE-915"], + [f"endpoint={url}", "response_status=200", + "sensitive_fields_present=" + ",".join(sorted(leaks))], + replay_steps=[ + "Authenticate as the regular user.", + f"GET {url}", + "Observe response carries sensitive property names: " + + ",".join(sorted(leaks)), + ], + remediation=( + "Strip sensitive properties (password hashes, MFA secrets, " + "API keys, role flags) from response serialisers. Use an " + "explicit allowlist of fields per role rather than excluding " + "individual sensitive ones." + ), + ) + else: + self.emit_clean( + "PT-OAPI3-01", "API response leaks sensitive properties", + "API3:2023", + [f"endpoint={url}", "response_status=200", + "no_sensitive_fields_present"], + ) + found_any = True + + if not found_any: + self.emit_inconclusive( + "PT-OAPI3-01", "API response leaks sensitive properties", + "API3:2023", "no_evaluable_responses", + ) + + # ── PT-OAPI3-02 — Mass-assignment write (Subphase 3.1, STATEFUL) ── + + def _test_api_property_tampering(self): + if not self.scenario_enabled("PT-OAPI3-02"): + return + api_security = self.target_config.api_security + title = "API accepts mass assignment of privileged properties" + owasp = "API3:2023" + if not api_security.property_endpoints: + self.emit_inconclusive( + "PT-OAPI3-02", title, owasp, "no_configured_property_endpoints", + ) + return + + session = self.auth.regular_session + if session is None: + self.emit_inconclusive( + "PT-OAPI3-02", title, owasp, "no_low_privileged_session", + ) + return + + tampering_fields = list( + getattr(api_security, "tampering_fields", []) or [] + ) + if not tampering_fields: + self.emit_inconclusive( + "PT-OAPI3-02", title, owasp, "no_tampering_fields_configured", + ) + return + + for ep in api_security.property_endpoints: + target_field = tampering_fields[0] # one field per endpoint + read_url = self._render_url(ep.path, ep.id_param, ep.test_id) + method = (ep.method_write or "PATCH").upper() + + def baseline(_ep=ep, _url=read_url): + if not self.budget(): + raise RuntimeError("budget_exhausted") + self.safety.throttle() + resp = session.get(_url, timeout=10, allow_redirects=False) + try: + return resp.json() if resp.status_code < 400 else None + except (ValueError, requests.exceptions.JSONDecodeError): + return None + + def mutate(base, _ep=ep, _url=read_url, _method=method, + _field=target_field): + if base is None: + return False + if not self.budget(): + raise RuntimeError("budget_exhausted") + self.safety.throttle() + payload = {_field: True} + try: + if _method == "PATCH": + resp = session.patch(_url, json=payload, timeout=10) + elif _method == "PUT": + resp = session.put(_url, json=payload, timeout=10) + else: + resp = session.post(_url, json=payload, timeout=10) + except requests.RequestException: + return self.MUTATION_ATTEMPTED_UNKNOWN + return resp.status_code < 400 + + def verify(base, _ep=ep, _url=read_url, _field=target_field): + if not self.budget(): + raise RuntimeError("budget_exhausted") + self.safety.throttle() + try: + resp = session.get(_url, timeout=10, allow_redirects=False) + except requests.RequestException: + return False + if resp.status_code >= 400: + return False + try: + data = resp.json() + except (ValueError, requests.exceptions.JSONDecodeError): + return False + if not isinstance(data, dict): + return False + before = (base or {}).get(_field) + after = data.get(_field) + return after is True and after != before + + def revert(base, _ep=ep, _url=read_url, _method=method, + _field=target_field): + if base is None: + return False + if _field not in base: + return False + if not self.cleanup_budget(): + return False + before = base.get(_field) + try: + if _method == "PATCH": + resp = session.patch(_url, json={_field: before}, timeout=10) + elif _method == "PUT": + resp = session.put(_url, json={_field: before}, timeout=10) + else: + resp = session.post(_url, json={_field: before}, timeout=10) + except requests.RequestException: + return False + return resp.status_code < 400 + + self.run_stateful( + "PT-OAPI3-02", + baseline_fn=baseline, + mutate_fn=mutate, + verify_fn=verify, + revert_fn=revert, + finding_kwargs={ + "title": title, "owasp": owasp, "severity": "HIGH", + "cwe": ["CWE-915"], + "evidence": [f"endpoint={read_url}", f"tampered_field={target_field}"], + "replay_steps": [ + "Authenticate as a non-privileged user.", + f"{method} {read_url}", + f'Body includes `{{"{target_field}": true}}` along with the ' + "field the operator is allowed to change.", + f"GET {read_url} and confirm `{target_field}` flipped to True.", + ], + "remediation": ( + "Use an explicit allowlist of writable fields per role. Never " + "pass user input through to ORM .update(**request.data); " + "deserialise into a typed schema first and reject unknown fields." + ), + }, + ) + + # ── helpers ──────────────────────────────────────────────────────── + + def _render_url(self, path, id_param, test_id): + """Substitute {id} into the endpoint path AND prepend target_url + so probes can pass the result directly to session.get/post/patch. + + Previously this was a @staticmethod returning just the path, which + caused PT-OAPI3-01 / PT-OAPI3-02 to call session.get('/api/...') + with no scheme — requests raised MissingSchema and the probe + emitted `baseline_failed` instead of evaluating the response. + """ + if "{" + id_param + "}" in path: + path = path.replace("{" + id_param + "}", str(test_id)) + elif "{id}" in path: + path = path.replace("{id}", str(test_id)) + return self.target_url + path + + @staticmethod + def _find_sensitive_keys(payload, patterns): + found = set() + if not isinstance(payload, dict): + return found + for key in payload.keys(): + if not isinstance(key, str): + continue + for pat in patterns: + if pat.search(key): + found.add(key) + break + return found diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/base.py b/extensions/business/cybersec/red_mesh/graybox/probes/base.py index 8e0fbd50..dadc78bd 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/base.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/base.py @@ -10,6 +10,7 @@ from ..findings import GrayboxFinding from ..models import GrayboxProbeContext, GrayboxProbeRunResult +from ..rollback import MUTATION_ATTEMPTED_UNKNOWN, StatefulMutationPlan class ProbeBase: @@ -31,7 +32,10 @@ class ProbeBase: def __init__(self, target_url, auth_manager, target_config, safety, discovered_routes=None, discovered_forms=None, - regular_username="", allow_stateful=False): + regular_username="", allow_stateful=False, + request_budget=None, allowed_scenario_ids=None, + rollback_journal=None, job_id="", worker_id="", + assignment_revision=0): self.target_url = target_url.rstrip("/") self.auth = auth_manager self.target_config = target_config @@ -40,6 +44,16 @@ def __init__(self, target_url, auth_manager, target_config, safety, self.discovered_forms = discovered_forms or [] self.regular_username = regular_username self._allow_stateful = allow_stateful + # OWASP API Top 10 — Subphase 1.7. Optional shared RequestBudget. + # When None, `self.budget()` always returns True (no enforcement). + self.request_budget = request_budget + self.allowed_scenario_ids = ( + None if allowed_scenario_ids is None else set(allowed_scenario_ids) + ) + self.rollback_journal = rollback_journal + self.job_id = job_id + self.worker_id = worker_id + self.assignment_revision = assignment_revision self.findings: list[GrayboxFinding] = [] @classmethod @@ -57,12 +71,78 @@ def run_safe(self, probe_name, probe_fn): """ try: probe_fn() - except requests.exceptions.ConnectionError: - self._record_error(probe_name, "target_unreachable") - except requests.exceptions.Timeout: - self._record_error(probe_name, "request_timeout") + except requests.exceptions.ConnectionError as exc: + self._record_error(probe_name, self._error_with_detail("target_unreachable", exc)) + except requests.exceptions.Timeout as exc: + self._record_error(probe_name, self._error_with_detail("request_timeout", exc)) except Exception as exc: - self._record_error(probe_name, self.safety.sanitize_error(str(exc))) + self._record_error(probe_name, self._sanitize_error(str(exc))) + + def scenario_enabled(self, scenario_id: str) -> bool: + """Return whether this worker is allowed to execute ``scenario_id``.""" + if self.allowed_scenario_ids is None: + return True + return scenario_id in self.allowed_scenario_ids + + def _api_auth_unverified(self) -> bool: + """Return True when API auth was explicitly accepted without validation.""" + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return False + auth = getattr(api_security, "auth", None) + if auth is None: + return False + auth_type = getattr(auth, "auth_type", "form") or "form" + if auth_type not in ("bearer", "api_key"): + return False + probe_path = (getattr(auth, "authenticated_probe_path", "") or "").strip() + return bool(getattr(auth, "allow_unverified_auth", False)) and not probe_path + + @staticmethod + def _is_api_security_scenario(scenario_id: str) -> bool: + return scenario_id.startswith("PT-OAPI") or scenario_id == "PT-API7-01" + + def _emit_auth_unverified(self, scenario_id: str): + if any( + f.scenario_id == scenario_id and "auth_unverified" in str(f.evidence) + for f in self.findings + ): + return + try: + from ..scenario_catalog import graybox_scenario + entry = graybox_scenario(scenario_id) or {} + except ImportError: + entry = {} + self.emit_inconclusive( + scenario_id, + entry.get("title") or scenario_id, + entry.get("owasp") or "", + "auth_unverified", + ) + + def run_safe_scenario(self, scenario_id: str, probe_name: str, probe_fn): + """Run a scenario only when the worker assignment permits it.""" + if not self.scenario_enabled(scenario_id): + return + if self._is_api_security_scenario(scenario_id) and self._api_auth_unverified(): + self._emit_auth_unverified(scenario_id) + return + self.run_safe(probe_name, probe_fn) + + def run_runtime_scenarios(self, probe_key: str): + """Run assigned runtime-manifest scenarios for one probe family.""" + from ..scenario_runtime import runtime_scenarios_for_probe + + for scenario in runtime_scenarios_for_probe(probe_key): + if not self.scenario_enabled(scenario.scenario_id): + continue + runner = getattr(self, scenario.runner) + self.run_safe_scenario( + scenario.scenario_id, + scenario.runner.lstrip("_"), + runner, + ) + return self.findings def build_result(self, outcome: str = "completed", artifacts=None) -> GrayboxProbeRunResult: """Return a typed probe result without changing legacy run() contracts.""" @@ -72,8 +152,242 @@ def build_result(self, outcome: str = "completed", artifacts=None) -> GrayboxPro outcome=outcome, ) + # ── Stateful probe contract (Subphase 1.8) ────────────────────────── + # + # Every mutating check must implement: baseline → mutate → verify + # → revert → cleanup-evidence. `StatefulProbeMixin.run_stateful` + # orchestrates the four steps and the helper below builds the matching + # finding. The lint test in test_stateful_contract.py asserts that no + # stateful probe bypasses this path. + STATEFUL_PROBE_LINT_MARKER = "uses_run_stateful" + MUTATION_ATTEMPTED_UNKNOWN = MUTATION_ATTEMPTED_UNKNOWN + + def run_stateful(self, scenario_id, *, baseline_fn, mutate_fn, + verify_fn, revert_fn, finding_kwargs=None, + skip_reason_no_revert="no_revert_path_configured", + mutation_unverified_reason_fn=None, + no_mutation_reason_fn=None, + mutation_plan=None, + clean_when_verify_false=False): + """Run a four-step stateful check. + + Steps: + 1. baseline_fn() -> baseline state (any pickle-safe value). + 2. mutate_fn(baseline) -> True if the mutation appeared to land. + 3. verify_fn(baseline) -> True if state actually changed + (i.e. the vulnerability is confirmed). + 4. revert_fn(baseline) -> True if the revert succeeded. + + Emits one GrayboxFinding via emit_vulnerable / emit_clean with the + `rollback_status` field populated on the finding. If the probe is + not gated on `allow_stateful=True`, emits inconclusive + (`stateful_probes_disabled`). If `revert_fn` is None, emits + inconclusive (`no_revert_path_configured` by default). + + `finding_kwargs` supplies the title/severity/owasp/etc. for the + vulnerable case. The clean case reuses ``title`` and ``owasp``. + """ + finding_kwargs = dict(finding_kwargs or {}) + title = finding_kwargs.pop("title", scenario_id) + owasp = finding_kwargs.pop("owasp", "") + + if not self.scenario_enabled(scenario_id): + return False + if not self._allow_stateful: + self.emit_inconclusive(scenario_id, title, owasp, + "stateful_probes_disabled") + return False + if revert_fn is None: + self.emit_inconclusive(scenario_id, title, owasp, skip_reason_no_revert) + return False + + # 1. Baseline. + try: + baseline = baseline_fn() + except Exception as exc: + self.emit_inconclusive( + scenario_id, title, owasp, + f"baseline_failed:{self.safety.sanitize_error(str(exc))}", + ) + return False + + # 2. Mutate. Journal before invoking mutate_fn so a timeout/crash + # after the outbound request still leaves a cleanup record. + journal_record_id = "" + if self.rollback_journal is not None: + plan = mutation_plan + if plan is None: + plan = StatefulMutationPlan( + scenario_id=scenario_id, + principal=getattr(self, "regular_username", "") or "", + ) + journal_record_id = self.rollback_journal.record_pending(scenario_id, plan) + mutated = False + mutation_attempted_unknown = False + try: + mutate_result = mutate_fn(baseline) + if mutate_result == MUTATION_ATTEMPTED_UNKNOWN: + mutated = True + mutation_attempted_unknown = True + else: + mutated = bool(mutate_result) + except Exception as exc: + self.emit_inconclusive( + scenario_id, title, owasp, + f"mutate_failed:{self.safety.sanitize_error(str(exc))}", + ) + if journal_record_id: + self.rollback_journal.update_status( + journal_record_id, "mutation_failed", + ) + return False + + # 3. Verify. Only literal True confirms; the MUTATION_ATTEMPTED_UNKNOWN + # sentinel (and any non-bool truthy value) must NEVER become a + # vulnerable finding because Python truthiness collapsed uncertainty + # into "confirmed" (PR406 B4). + confirmed = False + verify_failed_reason = "" + if mutated: + try: + verify_result = verify_fn(baseline) + if verify_result is True: + confirmed = True + else: + confirmed = False + if verify_result == MUTATION_ATTEMPTED_UNKNOWN or mutation_attempted_unknown: + verify_failed_reason = "mutation_attempted_unknown" + mutation_attempted_unknown = True + else: + verify_failed_reason = "mutation_unverified" + except Exception as exc: + confirmed = False + detail = self._sanitize_error(str(exc)) + verify_failed_reason = f"verify_failed:{detail}" if detail else "verify_failed" + + # 4. Revert (always attempt — even if not confirmed, the mutate may + # have left the target in an unintended state). + rollback_status = "no_revert_needed" if not mutated else "revert_failed" + if mutated: + try: + if revert_fn(baseline): + rollback_status = "reverted" + except Exception: + rollback_status = "revert_failed" + if journal_record_id: + journal_status = { + "no_revert_needed": "not_attempted", + "reverted": "reverted", + "revert_failed": "manual_cleanup_required", + }.get(rollback_status, rollback_status) + self.rollback_journal.update_status( + journal_record_id, + journal_status, + rollback_status=rollback_status, + ) + + # 5. Emit. Confirmed = vulnerable. A mutation that cannot be verified + # is inconclusive, not clean: the target may have changed, or request + # budget/transport may have prevented confirmation. + if confirmed: + severity = finding_kwargs.pop("severity", "HIGH") + # Severity bump on revert failure: HIGH→CRITICAL, MEDIUM→HIGH. + if rollback_status == "revert_failed": + severity = {"HIGH": "CRITICAL", "MEDIUM": "HIGH"}.get(severity, severity) + cwe = finding_kwargs.pop("cwe", []) + evidence = list(finding_kwargs.pop("evidence", [])) + remediation = finding_kwargs.pop("remediation", "") + if rollback_status == "revert_failed": + remediation = ( + (remediation + " ").strip() + + " Manual cleanup required — see Replay Steps." + ) + self.emit_vulnerable( + scenario_id, title, severity, owasp, cwe, evidence, + remediation=remediation, + rollback_status=rollback_status, + **finding_kwargs, + ) + return True + elif mutated: + reason = verify_failed_reason or "mutation_unverified" + if clean_when_verify_false and reason == "mutation_unverified": + self.emit_clean( + scenario_id, title, owasp, + list(finding_kwargs.get("evidence", []) or []), + rollback_status=rollback_status, + ) + return False + if callable(mutation_unverified_reason_fn): + try: + reason = mutation_unverified_reason_fn(baseline, rollback_status) or reason + except Exception as exc: + detail = self._sanitize_error(str(exc)) + reason = f"verify_reason_failed:{detail}" if detail else reason + self.emit_inconclusive( + scenario_id, title, owasp, + reason, + rollback_status=rollback_status, + ) + return False + else: + reason = "" + if callable(no_mutation_reason_fn): + try: + reason = no_mutation_reason_fn(baseline) or "" + except Exception as exc: + detail = self._sanitize_error(str(exc)) + reason = f"no_mutation_reason_failed:{detail}" if detail else "" + if reason: + self.emit_inconclusive( + scenario_id, title, owasp, reason, + rollback_status=rollback_status, + ) + return False + self.emit_clean( + scenario_id, title, owasp, + [], + rollback_status=rollback_status, + ) + return False + + def budget(self, n: int = 1) -> bool: + """Consume ``n`` requests from the shared per-scan RequestBudget. + + Returns False (and records an exhaustion event on the budget object) + when the budget can't cover the request. Probes that hit this should + stop iteration and emit `inconclusive` with reason + ``budget_exhausted``. Returns True when no budget is configured + (legacy callers / tests without a budget). + """ + if self.request_budget is None: + return True + return self.request_budget.consume(n) + + def cleanup_budget(self, n: int = 1) -> bool: + """Return True for cleanup/revert requests. + + Cleanup requests are deliberately exempt from the normal probe + request budget; budget exhaustion must not prevent rollback. + """ + return True + + def request(self, session, method: str, url: str, **kwargs): + """Probe-facing HTTP helper. + + Worker-created sessions are scoped by GrayboxHttpClient, so routing + calls through the session keeps scope enforcement centralized while + preserving the existing requests-like API. + """ + return session.request(method, url, **kwargs) + + def stateful_request(self, session, method: str, url: str, **kwargs): + """Issue a state-changing request through the scoped session wrapper.""" + return self.request(session, method, url, **kwargs) + def _record_error(self, probe_name, error_msg): """Store a non-fatal error as an INFO GrayboxFinding.""" + error_msg = self._sanitize_error(error_msg) self.findings.append(GrayboxFinding( scenario_id=f"ERR-{probe_name}", title=f"Probe error: {probe_name}", @@ -83,3 +397,126 @@ def _record_error(self, probe_name, error_msg): evidence=[f"error={error_msg}"], error=error_msg, )) + + def _error_with_detail(self, code, exc): + detail = self._sanitize_error(str(exc)) + if not detail: + return code + return f"{code}:{detail}" + + # ── OWASP API Top 10 emit helpers (Subphase 1.6) ───────────────────── + # + # These wrap GrayboxFinding construction so probe authors don't repeat + # the boilerplate and so finding emission has a single point at which + # evidence redaction is enforced. The redaction itself is added in + # Subphase 1.6 commit #2 (centralised scrubber). + # + # ATT&CK defaults: when ``attack`` is None, the helper resolves the + # default mapping from the catalog via attack_for_scenario(scenario_id) + # so probes don't have to remember per-scenario technique IDs. + + def _resolve_attack(self, scenario_id, attack): + if attack is not None: + return list(attack) + try: + from ..scenario_catalog import attack_for_scenario + except ImportError: + return [] + return attack_for_scenario(scenario_id) + + def _configured_secret_field_names(self): + """Read the configured API-key header/query names from target_config. + + Returned as a tuple of strings suitable for `scrub_graybox_secrets`. + Falls back to () when ApiSecurityConfig.auth is absent or the values + are not strings (e.g. MagicMock fixtures in unit tests). + """ + api_security = getattr(self.target_config, "api_security", None) + if api_security is None: + return () + auth = getattr(api_security, "auth", None) + if auth is None: + return () + names = [] + for attr in ("api_key_header_name", "api_key_query_param", + "bearer_token_header_name"): + val = getattr(auth, attr, None) + if isinstance(val, str) and val: + names.append(val) + return tuple(names) + + def _scrub_for_emission(self, value): + """Pre-emission scrub. Defense-in-depth alongside the storage-boundary + scrubber in ``findings.to_flat_finding`` (Subphase 1.6 commit #2).""" + from ..findings import scrub_graybox_secrets + return scrub_graybox_secrets( + value, secret_field_names=self._configured_secret_field_names(), + ) + + def _sanitize_error(self, value): + """Sanitize target-controlled exception text with configured secret names.""" + secret_field_names = self._configured_secret_field_names() + try: + sanitized = self.safety.sanitize_error( + str(value), secret_field_names=secret_field_names, + ) + except TypeError: + sanitized = self.safety.sanitize_error(str(value)) + return self._scrub_for_emission(sanitized) + + def emit_vulnerable(self, scenario_id, title, severity, owasp, cwe, + evidence, *, attack=None, evidence_artifacts=None, + replay_steps=None, remediation=None, + rollback_status=""): + """Append a vulnerable GrayboxFinding using the catalog's ATT&CK default. + + ``rollback_status`` is set by `run_stateful` for stateful probes; + leave default for non-stateful findings. + """ + self.findings.append(GrayboxFinding( + scenario_id=scenario_id, + title=self._scrub_for_emission(title), + status="vulnerable", + severity=severity, + owasp=owasp, + cwe=list(cwe or []), + attack=self._resolve_attack(scenario_id, attack), + evidence=self._scrub_for_emission(list(evidence or [])), + evidence_artifacts=self._scrub_for_emission(list(evidence_artifacts or [])), + replay_steps=self._scrub_for_emission(list(replay_steps or [])), + remediation=self._scrub_for_emission(remediation or ""), + rollback_status=rollback_status or "", + )) + + def emit_clean(self, scenario_id, title, owasp, evidence, + *, rollback_status=""): + """Append a not_vulnerable / INFO GrayboxFinding (test ran OK, nothing found).""" + self.findings.append(GrayboxFinding( + scenario_id=scenario_id, + title=self._scrub_for_emission(title), + status="not_vulnerable", + severity="INFO", + owasp=owasp, + evidence=self._scrub_for_emission(list(evidence or [])), + rollback_status=rollback_status or "", + )) + + def emit_inconclusive(self, scenario_id, title, owasp, reason, + *, rollback_status=""): + """Append an inconclusive / INFO GrayboxFinding. + + Use when a scenario could not be evaluated (missing config, stateful + gating disabled, request budget exhausted, target returned an + unexpected shape, etc.). ``reason`` is a short machine-readable + string appended to the evidence as ``reason=`` so reports can + group inconclusives by cause. + """ + self.findings.append(GrayboxFinding( + scenario_id=scenario_id, + title=self._scrub_for_emission(title), + status="inconclusive", + severity="INFO", + owasp=owasp, + evidence=[f"reason={self._scrub_for_emission(reason)}"], + rollback_status=rollback_status or "", + )) diff --git a/extensions/business/cybersec/red_mesh/graybox/probes/injection.py b/extensions/business/cybersec/red_mesh/graybox/probes/injection.py index 8f503c15..bf23c47a 100644 --- a/extensions/business/cybersec/red_mesh/graybox/probes/injection.py +++ b/extensions/business/cybersec/red_mesh/graybox/probes/injection.py @@ -36,7 +36,14 @@ def run(self): evidence=["stateful_probes_disabled=True", "reason=stored_xss_writes_data_to_target"], )) - self.run_safe("ssrf", self._test_ssrf) + self.run_safe_scenario("PT-API7-01", "ssrf", self._test_ssrf) + # OWASP API Top 10 — Subphase 2.7: extend PT-API7-01 to scan JSON + # body fields configured via target_config.api_security.ssrf_body_fields. + api_security = getattr(self.target_config, "api_security", None) + if api_security is not None and getattr(api_security, "ssrf_body_fields", None): + self.run_safe_scenario( + "PT-API7-01", "ssrf_body_field", self._test_ssrf_body_field, + ) self.run_safe("open_redirect", self._test_open_redirect) if self.auth.official_session: self.run_safe("path_traversal", self._test_path_traversal) @@ -386,6 +393,61 @@ def _test_ssrf(self): )) return + def _test_ssrf_body_field(self): + """PT-API7-01 extension (Subphase 2.7): scan JSON request body fields. + + For each SSRF endpoint configured under `injection.ssrf_endpoints`, + iterates the configured `api_security.ssrf_body_fields` names and + POSTs JSON bodies that embed an internal-probe URL under each field. + Vulnerable iff the response 200s and reflects the probe marker. + """ + api_security = self.target_config.api_security + body_fields = api_security.ssrf_body_fields + ssrf_endpoints = self.target_config.injection.ssrf_endpoints + if not ssrf_endpoints or not body_fields: + return + + payload_url = "http://127.0.0.1:1/internal-probe" + session = self.auth.official_session or getattr(self.auth, "anon_session", None) + if session is None: + return + + import requests as _rq + for ep in ssrf_endpoints: + url = self.target_url + "/" + ep.path.lstrip("/") + for body_field in body_fields: + self.safety.throttle() + try: + resp = session.post(url, json={body_field: payload_url}, timeout=10) + except _rq.RequestException: + continue + if resp.status_code == 200 and "internal-probe" in (resp.text or ""): + self.findings.append(GrayboxFinding( + scenario_id="PT-API7-01", + title="Server-side request forgery (JSON body field)", + status="vulnerable", + severity="HIGH", + owasp="API7:2023", + cwe=["CWE-918"], + attack=["T1190"], + evidence=[f"endpoint={url}", f"body_field={body_field}", + f"payload_url={payload_url}", + "response_status=200", + "reflected_marker=internal-probe"], + replay_steps=[ + "Authenticate as the official user.", + f"POST {url} with JSON body `{{\"{body_field}\": \"{payload_url}\"}}`.", + "Observe the response reflects the internal-probe marker, " + "confirming the server fetched the user-controlled URL.", + ], + remediation=( + "Validate URL fields against an allowlist of schemes/hosts. " + "Reject loopback / link-local / private ranges before issuing " + "the outbound request." + ), + )) + return # one body-field demo per scan is sufficient + def _test_open_redirect(self): """ PT-A01-04: test URL parameters for open redirect vulnerabilities. diff --git a/extensions/business/cybersec/red_mesh/graybox/rollback.py b/extensions/business/cybersec/red_mesh/graybox/rollback.py new file mode 100644 index 00000000..f3c7f848 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/rollback.py @@ -0,0 +1,109 @@ +"""Rollback journal primitives for graybox stateful probes.""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict + + +MUTATION_NOT_ATTEMPTED = "not_attempted" +MUTATION_ATTEMPTED_UNKNOWN = "attempted_unknown" +MUTATION_CONFIRMED = "confirmed" + + +@dataclass(frozen=True) +class StatefulMutationPlan: + scenario_id: str + method: str = "" + path: str = "" + body: dict | None = None + revert_method: str = "" + revert_path: str = "" + revert_body: dict | None = None + principal: str = "" + operation_key: str = "" + + def to_dict(self) -> dict: + return asdict(self) + + +class RollbackJournalRepository: + """Worker-owned rollback journal. + + The repository writes into the worker state list by reference, so live + status/report serialization can surface pending cleanup records without + probes writing into the shared job document directly. + """ + + def __init__( + self, + *, + job_id: str = "", + worker_id: str = "", + assignment_revision: int = 0, + records: list | None = None, + ): + self.job_id = job_id + self.worker_id = worker_id + self.assignment_revision = assignment_revision + self.records = records if records is not None else [] + + def record_pending(self, scenario_id: str, plan=None) -> str: + record_id = f"rollback-{len(self.records) + 1}" + if isinstance(plan, StatefulMutationPlan): + plan_dict = plan.to_dict() + elif isinstance(plan, dict): + plan_dict = dict(plan) + else: + plan_dict = {"scenario_id": scenario_id} + plan_dict.setdefault("scenario_id", scenario_id) + record = { + "record_id": record_id, + "job_id": self.job_id, + "worker_id": self.worker_id, + "assignment_revision": self.assignment_revision, + "scenario_id": scenario_id, + "status": "pending", + "plan": plan_dict, + "lease_owner": "", + "lease_expires_at": 0, + } + self.records.append(record) + return record_id + + def update_status(self, record_id: str, status: str, **extra) -> None: + for record in self.records: + if record.get("record_id") == record_id: + record["status"] = status + record.update(extra) + return + + def pending_records(self) -> list[dict]: + return [ + dict(record) for record in self.records + if record.get("status") in ("pending", "manual_cleanup_required") + ] + + def claim_pending(self, lease_owner: str, lease_expires_at: float = 0) -> list[dict]: + claimed = [] + for record in self.records: + if record.get("status") != "pending": + continue + record["lease_owner"] = lease_owner + record["lease_expires_at"] = lease_expires_at + record["status"] = "claimed" + claimed.append(dict(record)) + return claimed + + def replay_claimed(self, revert_fn_by_record_id) -> None: + """Replay claimed records with caller-provided idempotent revert fns.""" + for record in self.records: + if record.get("status") != "claimed": + continue + fn = revert_fn_by_record_id.get(record.get("record_id")) + if not callable(fn): + record["status"] = "manual_cleanup_required" + continue + try: + record["status"] = "reverted" if fn(record) else "manual_cleanup_required" + except Exception: + record["status"] = "manual_cleanup_required" diff --git a/extensions/business/cybersec/red_mesh/graybox/safety.py b/extensions/business/cybersec/red_mesh/graybox/safety.py index c46126b2..9c0d5e3d 100644 --- a/extensions/business/cybersec/red_mesh/graybox/safety.py +++ b/extensions/business/cybersec/red_mesh/graybox/safety.py @@ -78,14 +78,21 @@ def validate_target(target_url: str, authorized: bool) -> str | None: return None @staticmethod - def sanitize_error(msg: str) -> str: + def sanitize_error(msg: str, *, secret_field_names=()) -> str: """ Remove potential credential leaks from error messages. - Scrubs password= patterns and common secret markers. + Scrubs password= patterns, common secret markers, and configured + API auth header/query names when provided by the caller. """ import re msg = re.sub(r'password["\']?\s*[:=]\s*["\']?[^\s"\'&]+', 'password=***', msg, flags=re.I) msg = re.sub(r'secret["\']?\s*[:=]\s*["\']?[^\s"\'&]+', 'secret=***', msg, flags=re.I) msg = re.sub(r'token["\']?\s*[:=]\s*["\']?[^\s"\'&]+', 'token=***', msg, flags=re.I) - return msg + try: + from .findings import scrub_graybox_secrets + except Exception: + return msg + return scrub_graybox_secrets( + msg, secret_field_names=tuple(secret_field_names or ()), + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/scenario_catalog.py b/extensions/business/cybersec/red_mesh/graybox/scenario_catalog.py index 1003fce8..3d4cbee4 100644 --- a/extensions/business/cybersec/red_mesh/graybox/scenario_catalog.py +++ b/extensions/business/cybersec/red_mesh/graybox/scenario_catalog.py @@ -3,6 +3,19 @@ The catalog defines stable, countable authenticated-testing scenarios. Probe implementations may emit a subset on any given target depending on configured endpoints, auth state, and safety gates. + +Schema (per-entry dict): + id: stable scenario identifier (see docs/adr/2026-05-12-scenario-id-convention.md). + family: owning probe area; for v1 OWASP API Top 10 use one of + "api_access", "api_auth", "api_data", "api_config", "api_abuse"; + legacy families "access_control"/"misconfiguration"/"injection"/ + "business_logic" remain for OWASP Web Top 10 scenarios. + title: short human-facing title rendered in reports. + owasp: OWASP category tag, e.g. "A01:2021" (Web Top 10 2021) or + "API1:2023" (API Top 10 2023). + attack: optional list of MITRE ATT&CK technique IDs the finding maps to. + Mandatory and non-empty for v1 OWASP API Top 10 scenarios + (Subphase 1.2). Legacy PT-A* entries may omit this field. """ GRAYBOX_SCENARIO_CATALOG = ( @@ -96,10 +109,125 @@ {"id": "PT-A07-02", "family": "misconfiguration", "title": "Password reset token predictability", "owasp": "A07:2021"}, {"id": "PT-A07-03", "family": "misconfiguration", "title": "Session not rotated after login", "owasp": "A07:2021"}, {"id": "PT-A07-04", "family": "misconfiguration", "title": "Account enumeration by response body", "owasp": "A07:2021"}, - {"id": "PT-API7-01", "family": "injection", "title": "Authenticated SSRF", "owasp": "A10:2021"}, + # Legacy SSRF scenario kept under its original ID for backward compat. + # Probe emits owasp="API7:2023"; catalog now matches. + {"id": "PT-API7-01", "family": "injection", "title": "Authenticated SSRF", + "owasp": "API7:2023", "attack": ["T1190"]}, + + # ── OWASP API Top 10 2023 (v1 — Subphase 1.2) ────────────────────────── + # ATT&CK mappings copied from the V1 Scenario Manifest in the plan + # (`_todos/2026-05-12-graybox-api-top10-plan-detailed.md`, lines 90-115). + # API10 (Unsafe Consumption) intentionally omitted — Phase 9 follow-up. + + # API1 — Broken Object Level Authorization + {"id": "PT-OAPI1-01", "family": "api_access", + "title": "API object-level authorization bypass (BOLA)", + "owasp": "API1:2023", "attack": ["T1190", "T1078"]}, + + # API2 — Broken Authentication + {"id": "PT-OAPI2-01", "family": "api_auth", + "title": "API JWT missing-signature accepted (alg=none)", + "owasp": "API2:2023", "attack": ["T1078", "T1552"]}, + {"id": "PT-OAPI2-02", "family": "api_auth", + "title": "API JWT signed with weak HMAC secret", + "owasp": "API2:2023", "attack": ["T1212", "T1552"]}, + {"id": "PT-OAPI2-03", "family": "api_auth", + "title": "API token not invalidated on logout", + "owasp": "API2:2023", "attack": ["T1078"]}, + + # API3 — Broken Object Property Level Authorization (BOPLA) + {"id": "PT-OAPI3-01", "family": "api_data", + "title": "API response leaks sensitive properties (excessive exposure)", + "owasp": "API3:2023", "attack": ["T1552", "T1190"]}, + {"id": "PT-OAPI3-02", "family": "api_data", + "title": "API accepts mass assignment of privileged properties", + "owasp": "API3:2023", "attack": ["T1565", "T1078"]}, + + # API4 — Unrestricted Resource Consumption + {"id": "PT-OAPI4-01", "family": "api_abuse", + "title": "API endpoint lacks pagination cap", + "owasp": "API4:2023", "attack": ["T1499"]}, + {"id": "PT-OAPI4-02", "family": "api_abuse", + "title": "API endpoint accepts oversized payload", + "owasp": "API4:2023", "attack": ["T1499"]}, + {"id": "PT-OAPI4-03", "family": "api_abuse", + "title": "API endpoint lacks rate limit", + "owasp": "API4:2023", "attack": ["T1499"]}, + + # API5 — Broken Function Level Authorization + {"id": "PT-OAPI5-01", "family": "api_access", + "title": "API function-level authorization bypass (regular as admin, read)", + "owasp": "API5:2023", "attack": ["T1190", "T1078"]}, + {"id": "PT-OAPI5-02", "family": "api_access", + "title": "API function-level authorization bypass (anonymous as user, read)", + "owasp": "API5:2023", "attack": ["T1190"]}, + {"id": "PT-OAPI5-03", "family": "api_access", + "title": "API method-override authorization bypass", + "owasp": "API5:2023", "attack": ["T1190", "T1078"]}, + {"id": "PT-OAPI5-04", "family": "api_access", + "title": "API function-level authorization bypass (regular as admin, mutating)", + "owasp": "API5:2023", "attack": ["T1190", "T1078", "T1565"]}, + + # API6 — Unrestricted Access to Sensitive Business Flows + {"id": "PT-OAPI6-01", "family": "api_abuse", + "title": "API business flow lacks rate limit / abuse controls", + "owasp": "API6:2023", "attack": ["T1499", "T1190"]}, + {"id": "PT-OAPI6-02", "family": "api_abuse", + "title": "API business flow lacks uniqueness check", + "owasp": "API6:2023", "attack": ["T1565", "T1190"]}, + + # API8 — Security Misconfiguration + {"id": "PT-OAPI8-01", "family": "api_config", + "title": "API permissive CORS configuration", + "owasp": "API8:2023", "attack": ["T1190"]}, + {"id": "PT-OAPI8-02", "family": "api_config", + "title": "API response missing security headers", + "owasp": "API8:2023", "attack": ["T1190"]}, + {"id": "PT-OAPI8-03", "family": "api_config", + "title": "API debug endpoint exposed", + "owasp": "API8:2023", "attack": ["T1552", "T1190"]}, + {"id": "PT-OAPI8-04", "family": "api_config", + "title": "API verbose error response leaks internals", + "owasp": "API8:2023", "attack": ["T1190"]}, + {"id": "PT-OAPI8-05", "family": "api_config", + "title": "API advertises unexpected HTTP methods", + "owasp": "API8:2023", "attack": ["T1190"]}, + + # API9 — Improper Inventory Management + {"id": "PT-OAPI9-01", "family": "api_config", + "title": "API OpenAPI/Swagger specification publicly exposed", + "owasp": "API9:2023", "attack": ["T1595", "T1190"]}, + {"id": "PT-OAPI9-02", "family": "api_config", + "title": "API legacy version still live (version sprawl)", + "owasp": "API9:2023", "attack": ["T1595", "T1190"]}, + {"id": "PT-OAPI9-03", "family": "api_config", + "title": "API deprecated path still serving requests", + "owasp": "API9:2023", "attack": ["T1190"]}, ) def graybox_scenario_ids() -> set[str]: """Return stable graybox scenario IDs.""" return {entry["id"] for entry in GRAYBOX_SCENARIO_CATALOG} + + +def graybox_scenario(scenario_id: str) -> dict | None: + """Return the catalog entry for ``scenario_id`` or None if missing.""" + for entry in GRAYBOX_SCENARIO_CATALOG: + if entry["id"] == scenario_id: + return entry + return None + + +def attack_for_scenario(scenario_id: str) -> list[str]: + """Return the ATT&CK technique IDs for ``scenario_id``. + + Returns an empty list when the scenario is unknown or the entry has no + ``attack`` field set. Used by `ProbeBase.emit_vulnerable(..., attack=None)` + as the default attack mapping so the catalog is the single source of + truth (see Subphase 1.6). + """ + entry = graybox_scenario(scenario_id) + if entry is None: + return [] + return list(entry.get("attack", [])) diff --git a/extensions/business/cybersec/red_mesh/graybox/scenario_runtime.py b/extensions/business/cybersec/red_mesh/graybox/scenario_runtime.py new file mode 100644 index 00000000..b402fa0c --- /dev/null +++ b/extensions/business/cybersec/red_mesh/graybox/scenario_runtime.py @@ -0,0 +1,593 @@ +"""Runtime scenario manifest for graybox API scheduling.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass + + +GRAYBOX_ASSIGNMENT_MIRROR = "MIRROR" +GRAYBOX_ASSIGNMENT_SLICE = "SLICE" +GRAYBOX_BUDGET_PER_WORKER = "per_worker" +GRAYBOX_BUDGET_PER_SCAN = "per_scan" +GRAYBOX_DEFAULT_REQUEST_BUDGET = 1000 + + +@dataclass(frozen=True) +class RuntimeScenario: + scenario_id: str + probe_key: str + runner: str + stateful: bool = False + mutating: bool = False + requires_regular: bool = False + estimated_budget: int = 1 + single_writer_group: str = "" + + def to_dict(self) -> dict: + return { + "scenario_id": self.scenario_id, + "probe_key": self.probe_key, + "runner": self.runner, + "stateful": self.stateful, + "mutating": self.mutating, + "requires_regular": self.requires_regular, + "estimated_budget": self.estimated_budget, + "single_writer_group": self.single_writer_group, + } + + +API_RUNTIME_SCENARIOS = ( + RuntimeScenario( + "PT-OAPI1-01", "_graybox_api_access", "_test_api_bola", + requires_regular=True, estimated_budget=4, + ), + RuntimeScenario( + "PT-OAPI2-01", "_graybox_api_auth", "_test_jwt_alg_none", + estimated_budget=2, + ), + RuntimeScenario( + "PT-OAPI2-02", "_graybox_api_auth", "_test_jwt_weak_hmac", + estimated_budget=1, + ), + RuntimeScenario( + "PT-OAPI2-03", "_graybox_api_auth", + "_test_token_logout_invalidation", + stateful=True, mutating=True, estimated_budget=3, + single_writer_group="api_auth_token", + ), + RuntimeScenario( + "PT-OAPI3-01", "_graybox_api_data", + "_test_api_property_exposure", + estimated_budget=2, + ), + RuntimeScenario( + "PT-OAPI3-02", "_graybox_api_data", + "_test_api_property_tampering", + stateful=True, mutating=True, requires_regular=True, + estimated_budget=3, single_writer_group="api_data_property", + ), + RuntimeScenario( + "PT-OAPI4-01", "_graybox_api_abuse", + "_test_no_pagination_cap", estimated_budget=2, + ), + RuntimeScenario( + "PT-OAPI4-02", "_graybox_api_abuse", + "_test_oversized_payload", estimated_budget=1, + ), + RuntimeScenario( + "PT-OAPI4-03", "_graybox_api_abuse", + "_test_no_rate_limit", estimated_budget=5, + ), + RuntimeScenario( + "PT-OAPI5-01", "_graybox_api_access", + "_test_bfla_regular_as_admin", + requires_regular=True, estimated_budget=2, + ), + RuntimeScenario( + "PT-OAPI5-02", "_graybox_api_access", + "_test_bfla_anon_as_user", estimated_budget=2, + ), + RuntimeScenario( + "PT-OAPI5-03", "_graybox_api_access", + "_test_bfla_method_override", + stateful=True, mutating=True, requires_regular=True, + estimated_budget=3, single_writer_group="api_access_function", + ), + RuntimeScenario( + "PT-OAPI5-04", "_graybox_api_access", + "_test_bfla_regular_as_admin_mutating", + stateful=True, mutating=True, requires_regular=True, + estimated_budget=3, single_writer_group="api_access_function", + ), + RuntimeScenario( + "PT-OAPI6-01", "_graybox_api_abuse", + "_test_flow_no_rate_limit", + stateful=True, mutating=True, requires_regular=True, + estimated_budget=5, single_writer_group="api_abuse_flow", + ), + RuntimeScenario( + "PT-OAPI6-02", "_graybox_api_abuse", + "_test_flow_no_uniqueness", + stateful=True, mutating=True, requires_regular=True, + estimated_budget=2, single_writer_group="api_abuse_flow", + ), + RuntimeScenario( + "PT-OAPI8-01", "_graybox_api_config", + "_test_cors_misconfig", estimated_budget=1, + ), + RuntimeScenario( + "PT-OAPI8-02", "_graybox_api_config", + "_test_security_headers", estimated_budget=1, + ), + RuntimeScenario( + "PT-OAPI8-03", "_graybox_api_config", + "_test_debug_endpoint_exposed", estimated_budget=3, + ), + RuntimeScenario( + "PT-OAPI8-04", "_graybox_api_config", + "_test_verbose_error", estimated_budget=1, + ), + RuntimeScenario( + "PT-OAPI8-05", "_graybox_api_config", + "_test_unexpected_methods", estimated_budget=1, + ), + RuntimeScenario( + "PT-OAPI9-01", "_graybox_api_config", + "_test_openapi_exposed", estimated_budget=3, + ), + RuntimeScenario( + "PT-OAPI9-02", "_graybox_api_config", + "_test_version_sprawl", estimated_budget=3, + ), + RuntimeScenario( + "PT-OAPI9-03", "_graybox_api_config", + "_test_deprecated_live", estimated_budget=2, + ), + RuntimeScenario( + "PT-API7-01", "_graybox_injection", "_test_ssrf", + estimated_budget=2, + ), +) + + +def runtime_scenarios() -> tuple[RuntimeScenario, ...]: + return API_RUNTIME_SCENARIOS + + +def runtime_scenario_ids() -> tuple[str, ...]: + return tuple(item.scenario_id for item in API_RUNTIME_SCENARIOS) + + +def runtime_scenarios_for_probe(probe_key: str) -> tuple[RuntimeScenario, ...]: + return tuple(item for item in API_RUNTIME_SCENARIOS if item.probe_key == probe_key) + + +def runtime_scenario_by_id(scenario_id: str) -> RuntimeScenario | None: + for item in API_RUNTIME_SCENARIOS: + if item.scenario_id == scenario_id: + return item + return None + + +def _normalized_strategy(strategy: str) -> str: + value = (strategy or GRAYBOX_ASSIGNMENT_MIRROR).upper() + if value not in (GRAYBOX_ASSIGNMENT_MIRROR, GRAYBOX_ASSIGNMENT_SLICE): + return "" + return value + + +def _assignment_hash_payload( + *, + strategy: str, + assigned_scenario_ids: tuple[str, ...], + assigned_request_budget: int, + budget_scope: str, + assignment_revision: int, + stateful_policy: str, +) -> dict: + return { + "graybox_assignment_strategy": strategy, + "assigned_scenario_ids": list(assigned_scenario_ids), + "assigned_request_budget": int(assigned_request_budget or 0), + "budget_scope": budget_scope, + "assignment_revision": int(assignment_revision or 1), + "stateful_policy": stateful_policy, + } + + +def compute_assignment_hash( + *, + strategy: str, + assigned_scenario_ids, + assigned_request_budget: int, + budget_scope: str, + assignment_revision: int, + stateful_policy: str, +) -> str: + payload = _assignment_hash_payload( + strategy=strategy, + assigned_scenario_ids=tuple(assigned_scenario_ids or ()), + assigned_request_budget=assigned_request_budget, + budget_scope=budget_scope, + assignment_revision=assignment_revision, + stateful_policy=stateful_policy, + ) + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(raw).hexdigest()[:24] + + +@dataclass(frozen=True) +class GrayboxWorkerAssignment: + strategy: str + assigned_scenario_ids: tuple[str, ...] + assigned_request_budget: int + budget_scope: str + assignment_revision: int + assignment_hash: str + stateful_policy: str = "disabled" + validation_error: str = "" + + @property + def is_valid(self) -> bool: + return not self.validation_error + + def to_dict(self) -> dict: + return { + "graybox_assignment_strategy": self.strategy, + "assigned_scenario_ids": list(self.assigned_scenario_ids), + "assigned_request_budget": self.assigned_request_budget, + "budget_scope": self.budget_scope, + "assignment_revision": self.assignment_revision, + "assignment_hash": self.assignment_hash, + "stateful_policy": self.stateful_policy, + } + + @classmethod + def invalid(cls, reason: str) -> "GrayboxWorkerAssignment": + return cls( + strategy="", + assigned_scenario_ids=(), + assigned_request_budget=0, + budget_scope="", + assignment_revision=0, + assignment_hash="", + stateful_policy="", + validation_error=reason, + ) + + @classmethod + def from_job_config(cls, job_config) -> "GrayboxWorkerAssignment": + raw_ids = getattr(job_config, "assigned_scenario_ids", None) + if raw_ids is None: + return cls.invalid("missing_assigned_scenario_ids") + if not isinstance(raw_ids, (list, tuple)): + return cls.invalid("assigned_scenario_ids_must_be_list") + + assigned_scenario_ids = tuple(str(item) for item in raw_ids) + known_ids = set(runtime_scenario_ids()) + unknown = [item for item in assigned_scenario_ids if item not in known_ids] + if unknown: + return cls.invalid("unknown_assigned_scenario_ids:" + ",".join(unknown)) + + strategy = _normalized_strategy( + getattr(job_config, "graybox_assignment_strategy", "") + ) + if not strategy: + return cls.invalid("unknown_graybox_assignment_strategy") + + budget_scope = getattr(job_config, "budget_scope", "") or "" + if budget_scope not in (GRAYBOX_BUDGET_PER_WORKER, GRAYBOX_BUDGET_PER_SCAN): + return cls.invalid("unknown_budget_scope") + + try: + assigned_request_budget = int( + getattr(job_config, "assigned_request_budget", 0) or 0 + ) + except (TypeError, ValueError): + return cls.invalid("invalid_assigned_request_budget") + if assigned_request_budget <= 0: + return cls.invalid("invalid_assigned_request_budget") + + try: + assignment_revision = int( + getattr(job_config, "assignment_revision", 0) or 0 + ) + except (TypeError, ValueError): + return cls.invalid("invalid_assignment_revision") + if assignment_revision <= 0: + return cls.invalid("invalid_assignment_revision") + + stateful_policy = getattr(job_config, "stateful_policy", "") or "disabled" + assignment_hash = getattr(job_config, "assignment_hash", "") or "" + expected_hash = compute_assignment_hash( + strategy=strategy, + assigned_scenario_ids=assigned_scenario_ids, + assigned_request_budget=assigned_request_budget, + budget_scope=budget_scope, + assignment_revision=assignment_revision, + stateful_policy=stateful_policy, + ) + if assignment_hash != expected_hash: + return cls.invalid("assignment_hash_mismatch") + + return cls( + strategy=strategy, + assigned_scenario_ids=assigned_scenario_ids, + assigned_request_budget=assigned_request_budget, + budget_scope=budget_scope, + assignment_revision=assignment_revision, + assignment_hash=assignment_hash, + stateful_policy=stateful_policy, + ) + + +def build_graybox_worker_assignments( + worker_addresses, + *, + strategy: str = GRAYBOX_ASSIGNMENT_SLICE, + total_request_budget: int = GRAYBOX_DEFAULT_REQUEST_BUDGET, + allow_stateful: bool = False, + allow_mirror_stateful: bool = False, + allow_mirror_per_worker_budget: bool = False, + assignment_revision: int = 1, +): + """Return launcher-owned per-worker API scenario assignments. + + Defaults to SLICE so the per-scan request budget is split across + workers (PR406 B5). MIRROR remains explicit and, when more than one + worker is selected, requires ``allow_mirror_per_worker_budget=True`` + to acknowledge that total traffic is workers × budget; otherwise the + budget is divided across workers (budget_scope=per_scan). + """ + addresses = [addr for addr in (worker_addresses or []) if addr] + if not addresses: + return None, "No workers available for graybox assignment." + + strategy = _normalized_strategy(strategy) + if not strategy: + return None, "graybox_assignment_strategy must be MIRROR or SLICE." + + if ( + strategy == GRAYBOX_ASSIGNMENT_MIRROR + and allow_stateful + and len(addresses) > 1 + and not allow_mirror_stateful + ): + return ( + None, + "MIRROR with stateful graybox probes requires an explicit " + "allow_mirror_stateful override or a single selected worker.", + ) + + raw_budget = ( + GRAYBOX_DEFAULT_REQUEST_BUDGET + if total_request_budget is None else total_request_budget + ) + try: + total_budget = int(raw_budget) + except (TypeError, ValueError): + return None, "total_request_budget must be a positive integer." + if total_budget <= 0: + return None, "total_request_budget must be a positive integer." + + scenario_ids = runtime_scenario_ids() + stateful_policy = "enabled" if allow_stateful else "disabled" + assignments = {} + if strategy == GRAYBOX_ASSIGNMENT_MIRROR: + if len(addresses) > 1 and allow_mirror_per_worker_budget: + mirror_budget = total_budget + mirror_budget_scope = GRAYBOX_BUDGET_PER_WORKER + elif len(addresses) > 1: + # Multi-worker MIRROR without explicit per-worker budget opt-in: + # divide the per-scan budget across workers so total traffic stays + # bounded by max_total_requests instead of workers × budget. + base_budget, budget_remainder = divmod(total_budget, len(addresses)) + mirror_budget = None # computed per worker below + mirror_budget_scope = GRAYBOX_BUDGET_PER_SCAN + else: + mirror_budget = total_budget + mirror_budget_scope = GRAYBOX_BUDGET_PER_WORKER + + for index, address in enumerate(addresses): + if mirror_budget is None: + assigned_budget = max(1, base_budget + (1 if index < budget_remainder else 0)) + else: + assigned_budget = mirror_budget + assignment = GrayboxWorkerAssignment( + strategy=strategy, + assigned_scenario_ids=scenario_ids, + assigned_request_budget=assigned_budget, + budget_scope=mirror_budget_scope, + assignment_revision=assignment_revision, + assignment_hash="", + stateful_policy=stateful_policy, + ) + assignments[address] = _with_assignment_hash(assignment).to_dict() + return assignments, None + + base_budget, budget_remainder = divmod(total_budget, len(addresses)) + for index, address in enumerate(addresses): + ids = tuple(scenario_ids[index::len(addresses)]) + assigned_budget = max(1, base_budget + (1 if index < budget_remainder else 0)) + assignment = GrayboxWorkerAssignment( + strategy=strategy, + assigned_scenario_ids=ids, + assigned_request_budget=assigned_budget, + budget_scope=GRAYBOX_BUDGET_PER_SCAN, + assignment_revision=assignment_revision, + assignment_hash="", + stateful_policy=stateful_policy, + ) + assignments[address] = _with_assignment_hash(assignment).to_dict() + return assignments, None + + +_LEGACY_ASSIGNMENT_FIELDS = ( + "graybox_assignment_strategy", + "assigned_scenario_ids", + "assigned_request_budget", + "budget_scope", + "assignment_hash", +) + + +def synthesize_legacy_mirror_assignment( + job_config: dict | None, + worker_entry: dict | None, +) -> dict | None: + """Build a compat MIRROR assignment for assignmentless webapp jobs (PR406 B7). + + Returns a dict matching ``GrayboxWorkerAssignment.to_dict()`` plus an + ``assignment_compat_mode`` audit marker. Returns None when: + + * the worker entry already carries at least one new assignment + field (partial/corrupt — must fail closed); + * the entry is not a dict; + * the entry already includes an explicit compat marker. + + The synthesized assignment runs all runtime scenarios with the + per-scan budget derived from + ``target_config.api_security.max_total_requests`` (or the default). + """ + if not isinstance(worker_entry, dict): + return None + if worker_entry.get("assignment_compat_mode"): + return None + present = [ + field for field in _LEGACY_ASSIGNMENT_FIELDS + if worker_entry.get(field) not in (None, "", 0, [], ()) + ] + if present: + # Any single new field present means this is a launcher-owned + # assignment that just happens to be incomplete — refuse to + # synthesize and let the normal validation reject it. + return None + + budget = 0 + if isinstance(job_config, dict): + api_security = job_config.get("target_config", {}) + if isinstance(api_security, dict): + api_security = api_security.get("api_security") or {} + if isinstance(api_security, dict): + try: + budget = int(api_security.get("max_total_requests") or 0) + except (TypeError, ValueError): + budget = 0 + if budget <= 0: + budget = GRAYBOX_DEFAULT_REQUEST_BUDGET + scenarios = runtime_scenario_ids() + assignment = GrayboxWorkerAssignment( + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + assigned_scenario_ids=scenarios, + assigned_request_budget=budget, + budget_scope=GRAYBOX_BUDGET_PER_WORKER, + assignment_revision=1, + assignment_hash="", + stateful_policy="disabled", + ) + result = _with_assignment_hash(assignment).to_dict() + result["assignment_compat_mode"] = "legacy_mirror" + return result + + +def rehash_worker_assignment_dict(worker_entry: dict) -> dict: + """Recompute ``assignment_hash`` in place for a worker entry. + + Used after assignment-bearing fields change (notably + ``assignment_revision`` during reannounce — PR406 B6) so the hash the + worker validates against `JobConfig` stays in sync. Returns the same + dict for chaining; if the entry is missing assignment fields, the + hash field is left untouched. + """ + if not isinstance(worker_entry, dict): + return worker_entry + strategy = (worker_entry.get("graybox_assignment_strategy") or "").upper() + scenario_ids = worker_entry.get("assigned_scenario_ids") + if not strategy or scenario_ids is None: + return worker_entry + worker_entry["assignment_hash"] = compute_assignment_hash( + strategy=strategy, + assigned_scenario_ids=tuple(scenario_ids or ()), + assigned_request_budget=int(worker_entry.get("assigned_request_budget") or 0), + budget_scope=worker_entry.get("budget_scope") or "", + assignment_revision=int(worker_entry.get("assignment_revision") or 1), + stateful_policy=worker_entry.get("stateful_policy") or "disabled", + ) + return worker_entry + + +def summarize_graybox_worker_assignments(assignments: dict) -> dict: + """Distil per-worker assignments into a job-level summary. + + When all workers agree on strategy/budget_scope, the summary surfaces + them directly. When workers disagree (shouldn't happen with the + launcher-owned model, but defends against legacy/manual edits), the + summary records 'mixed' so the dashboard can flag it. + """ + if not isinstance(assignments, dict) or not assignments: + return {} + strategies = set() + budget_scopes = set() + total_budget = 0 + scenarios: set[str] = set() + worker_summary = [] + for addr, entry in assignments.items(): + if not isinstance(entry, dict): + continue + strategy = entry.get("graybox_assignment_strategy") or "" + budget_scope = entry.get("budget_scope") or "" + assigned_budget = int(entry.get("assigned_request_budget") or 0) + assigned_scenarios = list(entry.get("assigned_scenario_ids") or []) + if strategy: + strategies.add(strategy) + if budget_scope: + budget_scopes.add(budget_scope) + total_budget += assigned_budget + scenarios.update(assigned_scenarios) + worker_summary.append({ + "worker_address": addr, + "graybox_assignment_strategy": strategy, + "assigned_request_budget": assigned_budget, + "budget_scope": budget_scope, + "assigned_scenario_count": len(assigned_scenarios), + }) + + if len(strategies) == 1: + strategy_value = next(iter(strategies)) + else: + strategy_value = "mixed" + if len(budget_scopes) == 1: + budget_scope_value = next(iter(budget_scopes)) + else: + budget_scope_value = "mixed" + return { + "graybox_assignment_strategy": strategy_value, + "budget_scope": budget_scope_value, + "assigned_request_budget": total_budget, + "total_assigned_scenarios": len(scenarios), + "worker_assignment_summary": worker_summary, + } + + +def _with_assignment_hash( + assignment: GrayboxWorkerAssignment, +) -> GrayboxWorkerAssignment: + assignment_hash = compute_assignment_hash( + strategy=assignment.strategy, + assigned_scenario_ids=assignment.assigned_scenario_ids, + assigned_request_budget=assignment.assigned_request_budget, + budget_scope=assignment.budget_scope, + assignment_revision=assignment.assignment_revision, + stateful_policy=assignment.stateful_policy, + ) + return GrayboxWorkerAssignment( + strategy=assignment.strategy, + assigned_scenario_ids=assignment.assigned_scenario_ids, + assigned_request_budget=assignment.assigned_request_budget, + budget_scope=assignment.budget_scope, + assignment_revision=assignment.assignment_revision, + assignment_hash=assignment_hash, + stateful_policy=assignment.stateful_policy, + ) diff --git a/extensions/business/cybersec/red_mesh/graybox/worker.py b/extensions/business/cybersec/red_mesh/graybox/worker.py index 0f010b79..7b123a98 100644 --- a/extensions/business/cybersec/red_mesh/graybox/worker.py +++ b/extensions/business/cybersec/red_mesh/graybox/worker.py @@ -10,10 +10,17 @@ from ..worker.base import BaseLocalWorker from ..constants import GRAYBOX_PROBE_REGISTRY -from .findings import GrayboxEvidenceArtifact, GrayboxFinding +from .findings import ( + FindingRedactionContext, + GrayboxEvidenceArtifact, + GrayboxFinding, +) from .auth import AuthManager from .discovery import DiscoveryModule +from .http_client import GrayboxHttpClient from .safety import SafetyControls +from .rollback import RollbackJournalRepository +from .scenario_runtime import GrayboxWorkerAssignment from .models import ( DiscoveryResult, GrayboxCredentialSet, @@ -108,6 +115,27 @@ def __init__(self, owner, job_id, target_url, job_config, self.target_config = GrayboxTargetConfig.from_dict( job_config.target_config or {} ) + self.assignment = GrayboxWorkerAssignment.from_job_config(job_config) + + # OWASP API Top 10 — Subphase 1.7. Per-scan request budget shared by + # every probe instance. Default 1000; configurable via + # `target_config.api_security.max_total_requests`. + from .budget import RequestBudget + if self.assignment.is_valid: + budget_total = self.assignment.assigned_request_budget + else: + budget_total = getattr( + self.target_config.api_security, "max_total_requests", 1000, + ) + budget_total = max(1, int(budget_total)) + self.request_budget = RequestBudget( + remaining=budget_total, total=budget_total, + ) + self.http_client = GrayboxHttpClient( + self.target_url, + allowlist=getattr(job_config, "target_allowlist", None) or [], + target_config=self.target_config, + ) # Modules (composition) self.safety = SafetyControls( @@ -118,6 +146,7 @@ def __init__(self, owner, job_id, target_url, job_config, target_url=self.target_url, target_config=self.target_config, verify_tls=job_config.verify_tls, + http_client=self.http_client, ) self.discovery = DiscoveryModule( target_url=self.target_url, @@ -149,7 +178,18 @@ def __init__(self, owner, job_id, target_url, job_config, "aborted": False, "abort_reason": "", "abort_phase": "", + "graybox_assignment": ( + self.assignment.to_dict() if self.assignment.is_valid else {} + ), + "rollback_journal": [], } + self.rollback_journal = RollbackJournalRepository( + job_id=job_id, + worker_id=self.local_worker_id, + assignment_revision=self.assignment.assignment_revision + if self.assignment.is_valid else 0, + records=self.state["rollback_journal"], + ) # _phase_open is only touched on the worker thread — no cross-thread # reads. Guards the finally clause from double-closing a phase that # its owning method already closed explicitly. @@ -189,6 +229,15 @@ def get_status(self, for_aggregations=False): "scenarios_inconclusive": scenario_stats["inconclusive"], "scenarios_error": scenario_stats["error"], }) + # OWASP API Top 10 — Subphase 1.7. Per-scan request budget snapshot + # surfaces in scan_metrics so operators can see whether the scan was + # budget-bound (and tune target_config.api_security.max_total_requests + # accordingly). + if self.request_budget is not None: + snap = self.request_budget.snapshot() + metrics["budget_total"] = snap["total"] + metrics["budget_remaining"] = snap["remaining"] + metrics["budget_exhausted_count"] = snap["exhausted_count"] status["scan_metrics"] = metrics status["scenario_stats"] = scenario_stats @@ -251,7 +300,7 @@ def execute_job(self): color='y', ) except Exception as exc: - self._record_fatal(self.safety.sanitize_error(str(exc))) + self._record_fatal(self._sanitize_error(str(exc))) finally: self._safe_cleanup() if self._phase_open and self._phase: @@ -266,7 +315,7 @@ def _safe_cleanup(self): except Exception as exc: self.P( "[GRAYBOX] auth.cleanup raised during shutdown: %s" - % self.safety.sanitize_error(str(exc)), + % self._sanitize_error(str(exc)), color='y', ) @@ -294,6 +343,13 @@ def _run_preflight_phase(self): self.metrics.phase_start("preflight") self._phase_open = True try: + if not self.assignment.is_valid: + self._abort( + "Invalid graybox worker assignment: " + + self.assignment.validation_error, + reason_class="assignment_invalid", + ) + target_error = self.safety.validate_target( self.target_url, self.job_config.authorized, ) @@ -372,6 +428,7 @@ def _run_discovery_phase(self) -> DiscoveryResult: self._phase_open = False def _build_probe_kwargs(self, discovery_result: DiscoveryResult) -> dict: + allowed_scenario_ids = getattr(self.job_config, "assigned_scenario_ids", None) return GrayboxProbeContext( target_url=self.target_url, auth_manager=self.auth, @@ -381,6 +438,15 @@ def _build_probe_kwargs(self, discovery_result: DiscoveryResult) -> dict: discovered_forms=discovery_result.forms, regular_username=self._credentials.regular.username if self._credentials.regular else "", allow_stateful=self.job_config.allow_stateful_probes, + request_budget=self.request_budget, + allowed_scenario_ids=( + None if allowed_scenario_ids is None else tuple(allowed_scenario_ids) + ), + rollback_journal=self.rollback_journal, + job_id=self.job_id, + worker_id=self.local_worker_id, + assignment_revision=self.assignment.assignment_revision + if self.assignment.is_valid else 0, ) def _run_probe_phase(self, discovery_result: DiscoveryResult): @@ -524,17 +590,41 @@ def _store_findings(self, key, findings): """Store GrayboxFinding dicts in graybox_results under the port key.""" run_result = self._normalize_probe_run_result(findings) port_results = self.state["graybox_results"].setdefault(self._port_key, {}) - port_results[key] = { - "findings": [f.to_dict() for f in run_result.findings], - "artifacts": [ - GrayboxEvidenceArtifact.from_value(artifact).to_dict() - for artifact in run_result.artifacts - ], - "outcome": run_result.outcome, - } + with FindingRedactionContext( + secret_field_names=self._configured_secret_field_names(), + ): + port_results[key] = { + "findings": [f.to_dict() for f in run_result.findings], + "artifacts": [ + GrayboxEvidenceArtifact.from_value(artifact).to_dict() + for artifact in run_result.artifacts + ], + "outcome": run_result.outcome, + } for finding in run_result.findings: self.metrics.record_finding(getattr(finding, "severity", "INFO")) + def _configured_secret_field_names(self): + api_security = getattr(self.target_config, "api_security", None) + auth = getattr(api_security, "auth", None) if api_security is not None else None + if auth is None: + return () + names = [] + for attr in ("api_key_header_name", "api_key_query_param", + "bearer_token_header_name"): + value = getattr(auth, attr, None) + if isinstance(value, str) and value: + names.append(value) + return tuple(names) + + def _sanitize_error(self, value): + try: + return self.safety.sanitize_error( + str(value), secret_field_names=self._configured_secret_field_names(), + ) + except TypeError: + return self.safety.sanitize_error(str(value)) + def _store_auth_results(self): port_info = self.state["service_info"].setdefault(self._port_key, {}) port_info["_graybox_auth"] = { @@ -566,7 +656,7 @@ def _record_fatal(self, message): def _record_probe_error(self, store_key, exc): """Record per-probe error without killing the scan.""" - sanitized = self.safety.sanitize_error(str(exc)) + sanitized = self._sanitize_error(str(exc)) self._store_findings(store_key, [GrayboxFinding( scenario_id=f"ERR-{store_key}", title=f"Probe error: {store_key}", diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index cecdbfcc..c221831f 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -15,6 +15,89 @@ # dedup signature so the same vulnerability seen by two workers # collapses to one finding (with one worker's stamp preserved). _DEDUP_EXCLUDE_FIELDS = ("_source_worker_id", "_source_node_addr") +_PUBLISH_SAFE_METADATA_KEYS = { + "api_key_header_name", + "api_key_location", + "api_key_query_param", + "authenticated_probe_path", + "authenticated_probe_method", + "allow_non_readonly_auth_validation_method", + "allow_unverified_auth", + "bearer_refresh_url", + "bearer_scheme", + "bearer_token_header_name", + "csrf_field", + "password_field", + "password_reset_confirm_path", + "password_reset_path", + "protected_path", + "token_path", + "token_request_method", + "token_response_field", +} +_SECRET_CONFIG_KEY_PARTS = ( + "password", "passwd", "pwd", "secret", "authorization", "cookie", + "credential", +) +_SECRET_CONFIG_TOKEN_KEYS = { + "api_key", "apikey", "token", "access_token", "refresh_token", "id_token", + "secret_ref", +} + + +def _is_secret_config_key(key): + normalized = str(key or "").strip().lower().replace("-", "_") + if not normalized or normalized.startswith("has_"): + return False + if normalized in _PUBLISH_SAFE_METADATA_KEYS: + return False + if normalized in _SECRET_CONFIG_TOKEN_KEYS: + return True + if normalized.endswith("_token") or normalized.endswith("_api_key"): + return True + return any(part in normalized for part in _SECRET_CONFIG_KEY_PARTS) + + +def _redact_nested_job_config(value): + if isinstance(value, dict): + redacted = {} + for key, item in value.items(): + if _is_secret_config_key(key): + redacted[key] = "***" + else: + redacted[key] = _redact_nested_job_config(item) + return redacted + if isinstance(value, list): + return [_redact_nested_job_config(item) for item in value] + return value + + +def _configured_graybox_secret_names_from_report(report): + """Extract configured API auth field names from report/job target config.""" + if not isinstance(report, dict): + return () + candidates = [] + for key in ("target_config",): + if isinstance(report.get(key), dict): + candidates.append(report[key]) + job_config = report.get("job_config") + if isinstance(job_config, dict) and isinstance(job_config.get("target_config"), dict): + candidates.append(job_config["target_config"]) + + names = [] + for target_config in candidates: + api_security = target_config.get("api_security") or {} + auth = api_security.get("auth") or {} + if not isinstance(auth, dict): + continue + for key in ( + "api_key_header_name", "api_key_query_param", + "bearer_token_header_name", + ): + value = auth.get(key) + if isinstance(value, str) and value and value not in names: + names.append(value) + return tuple(names) def _finding_dedup_key(item): @@ -421,7 +504,12 @@ def _redact_report(self, report): """ import re as _re from copy import deepcopy + try: + from ..graybox.findings import scrub_graybox_secrets as _scrub_graybox + except Exception: + _scrub_graybox = None redacted = deepcopy(report) + graybox_secret_names = _configured_graybox_secret_names_from_report(redacted) service_info = redacted.get("service_info", {}) for port_key, methods in service_info.items(): if not isinstance(methods, dict): @@ -458,8 +546,16 @@ def _redact_report(self, report): def _redact_graybox_text(value): if not isinstance(value, str): return value + if _scrub_graybox is not None: + value = _scrub_graybox( + value, secret_field_names=graybox_secret_names, + ) value = _CRED_RE.sub(r'\1:***', value) value = _PASSWORD_RE.sub(r'\1\2***', value) + if _scrub_graybox is not None: + value = _scrub_graybox( + value, secret_field_names=graybox_secret_names, + ) return value graybox_results = redacted.get("graybox_results", {}) @@ -472,6 +568,13 @@ def _redact_graybox_text(value): for finding in probe_data.get("findings", []): if not isinstance(finding, dict): continue + for text_key in ("title", "description", "remediation", "error"): + if isinstance(finding.get(text_key), str): + finding[text_key] = _redact_graybox_text(finding[text_key]) + if isinstance(finding.get("replay_steps"), list): + finding["replay_steps"] = [ + _redact_graybox_text(step) for step in finding["replay_steps"] + ] evidence = finding.get("evidence", []) if isinstance(evidence, list): finding["evidence"] = [ @@ -526,6 +629,14 @@ def _redact_job_config(config_dict): redacted["regular_password"] = "***" if redacted.get("weak_candidates"): redacted["weak_candidates"] = ["***"] * len(redacted["weak_candidates"]) + if isinstance(redacted.get("target_config_secrets"), dict): + redacted["target_config_secrets"] = { + str(key): "***" for key in redacted["target_config_secrets"] + } + if isinstance(redacted.get("target_config"), dict): + redacted["target_config"] = _redact_nested_job_config( + redacted["target_config"] + ) redacted.pop("secret_ref", None) return redacted diff --git a/extensions/business/cybersec/red_mesh/mixins/risk.py b/extensions/business/cybersec/red_mesh/mixins/risk.py index d299971f..cc28276f 100644 --- a/extensions/business/cybersec/red_mesh/mixins/risk.py +++ b/extensions/business/cybersec/red_mesh/mixins/risk.py @@ -86,26 +86,34 @@ def process_findings(findings_list): process_findings(correlation_findings) # A. Iterate graybox_results — uses GrayboxFinding.to_flat_finding() - from ..graybox.findings import GrayboxFinding as _GF + from ..graybox.findings import ( + FindingRedactionContext, + GrayboxFinding as _GF, + ) + from .report import _configured_graybox_secret_names_from_report + graybox_secret_names = _configured_graybox_secret_names_from_report( + aggregated_report, + ) graybox_results = aggregated_report.get("graybox_results", {}) - for port_key, probes in graybox_results.items(): - if not isinstance(probes, dict): - continue - for probe_name, probe_data in probes.items(): - if not isinstance(probe_data, dict): + with FindingRedactionContext(secret_field_names=graybox_secret_names): + for port_key, probes in graybox_results.items(): + if not isinstance(probes, dict): continue - for finding_dict in probe_data.get("findings", []): - if not isinstance(finding_dict, dict): - continue - try: - flat = _GF.flat_from_dict(finding_dict, 0, "unknown", probe_name) - except (TypeError, KeyError, ValueError): + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): continue - weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) - multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) - findings_score += weight * multiplier - if flat["severity"] in finding_counts: - finding_counts[flat["severity"]] += 1 + for finding_dict in probe_data.get("findings", []): + if not isinstance(finding_dict, dict): + continue + try: + flat = _GF.flat_from_dict(finding_dict, 0, "unknown", probe_name) + except (TypeError, KeyError, ValueError): + continue + weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) + findings_score += weight * multiplier + if flat["severity"] in finding_counts: + finding_counts[flat["severity"]] += 1 # B. Open ports — diminishing returns: 15 × (1 - e^(-ports/8)) open_ports = aggregated_report.get("open_ports", []) @@ -339,36 +347,44 @@ def parse_port(port_key): process_findings(correlation_findings, 0, "_correlation", "correlation") # Walk graybox_results — delegates to GrayboxFinding.to_flat_finding() - from ..graybox.findings import GrayboxFinding as _GF + from ..graybox.findings import ( + FindingRedactionContext, + GrayboxFinding as _GF, + ) + from .report import _configured_graybox_secret_names_from_report + graybox_secret_names = _configured_graybox_secret_names_from_report( + aggregated_report, + ) graybox_results = aggregated_report.get("graybox_results", {}) - for port_key, probes in graybox_results.items(): - if not isinstance(probes, dict): - continue - port = parse_port(port_key) - protocol = port_protocols.get(str(port), "unknown") - for probe_name, probe_data in probes.items(): - if not isinstance(probe_data, dict): + with FindingRedactionContext(secret_field_names=graybox_secret_names): + for port_key, probes in graybox_results.items(): + if not isinstance(probes, dict): continue - for finding_dict in probe_data.get("findings", []): - if not isinstance(finding_dict, dict): - continue - try: - flat = _GF.flat_from_dict(finding_dict, port, protocol, probe_name) - except (TypeError, KeyError, ValueError): + port = parse_port(port_key) + protocol = port_protocols.get(str(port), "unknown") + for probe_name, probe_data in probes.items(): + if not isinstance(probe_data, dict): continue - - weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) - multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) - findings_score += weight * multiplier - if flat["severity"] in finding_counts: - finding_counts[flat["severity"]] += 1 - title = flat.get("title", "") - if isinstance(title, str) and "default credential accepted" in title.lower(): - cred_count += 1 - - flat_findings.append( - normalize_flat_finding(flat, port, protocol, probe_name, "graybox") - ) + for finding_dict in probe_data.get("findings", []): + if not isinstance(finding_dict, dict): + continue + try: + flat = _GF.flat_from_dict(finding_dict, port, protocol, probe_name) + except (TypeError, KeyError, ValueError): + continue + + weight = RISK_SEVERITY_WEIGHTS.get(flat["severity"], 0) + multiplier = RISK_CONFIDENCE_MULTIPLIERS.get(flat["confidence"], 0.5) + findings_score += weight * multiplier + if flat["severity"] in finding_counts: + finding_counts[flat["severity"]] += 1 + title = flat.get("title", "") + if isinstance(title, str) and "default credential accepted" in title.lower(): + cred_count += 1 + + flat_findings.append( + normalize_flat_finding(flat, port, protocol, probe_name, "graybox") + ) # B. Open ports — diminishing returns open_ports = aggregated_report.get("open_ports", []) diff --git a/extensions/business/cybersec/red_mesh/models/archive.py b/extensions/business/cybersec/red_mesh/models/archive.py index fa62014a..d8f3fa6d 100644 --- a/extensions/business/cybersec/red_mesh/models/archive.py +++ b/extensions/business/cybersec/red_mesh/models/archive.py @@ -69,16 +69,44 @@ class JobConfig: secret_ref: str = "" # reference to separately persisted graybox secrets has_regular_credentials: bool = False has_weak_candidates: bool = False + # OWASP API Top 10 (Subphase 1.5 commit #8) — non-secret capability flags. + # Raw bearer_token / api_key / bearer_refresh_token values are blanked + # before persistence by `_blank_graybox_secret_fields` and instead live + # in the R1FS secret payload (resolved at worker startup via + # `resolve_job_config_secrets`). + has_bearer_token: bool = False + has_api_key: bool = False + has_bearer_refresh_token: bool = False + has_regular_bearer_token: bool = False + has_regular_api_key: bool = False + has_regular_bearer_refresh_token: bool = False + secret_store_key_id: str = "" + secret_store_key_version: str = "" + secret_store_key_source: str = "" + secret_store_unsafe_fallback: bool = False official_username: str = "" official_password: str = "" regular_username: str = "" regular_password: str = "" + bearer_token: str = "" # blanked before persistence; runtime-only + api_key: str = "" # blanked before persistence; runtime-only + bearer_refresh_token: str = "" # blanked before persistence; runtime-only + regular_bearer_token: str = "" # blanked before persistence; runtime-only + regular_api_key: str = "" # blanked before persistence; runtime-only + regular_bearer_refresh_token: str = "" # blanked before persistence; runtime-only weak_candidates: list = None # legacy inline payload; new launches use secret_ref max_weak_attempts: int = 5 app_routes: list = None # user-supplied known routes verify_tls: bool = True # TLS cert verification target_config: dict = None # GrayboxTargetConfig.to_dict() allow_stateful_probes: bool = False # gate for A06 workflow probes + graybox_assignment_strategy: str = "MIRROR" + assigned_scenario_ids: list = None + assigned_request_budget: int = 0 + budget_scope: str = "" + assignment_revision: int = 0 + assignment_hash: str = "" + stateful_policy: str = "" def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -120,16 +148,39 @@ def from_dict(cls, d: dict) -> JobConfig: secret_ref=d.get("secret_ref", ""), has_regular_credentials=d.get("has_regular_credentials", False), has_weak_candidates=d.get("has_weak_candidates", False), + has_bearer_token=d.get("has_bearer_token", False), + has_api_key=d.get("has_api_key", False), + has_bearer_refresh_token=d.get("has_bearer_refresh_token", False), + has_regular_bearer_token=d.get("has_regular_bearer_token", False), + has_regular_api_key=d.get("has_regular_api_key", False), + has_regular_bearer_refresh_token=d.get("has_regular_bearer_refresh_token", False), + secret_store_key_id=d.get("secret_store_key_id", ""), + secret_store_key_version=d.get("secret_store_key_version", ""), + secret_store_key_source=d.get("secret_store_key_source", ""), + secret_store_unsafe_fallback=d.get("secret_store_unsafe_fallback", False), official_username=d.get("official_username", ""), official_password=d.get("official_password", ""), regular_username=d.get("regular_username", ""), regular_password=d.get("regular_password", ""), + bearer_token=d.get("bearer_token", ""), + api_key=d.get("api_key", ""), + bearer_refresh_token=d.get("bearer_refresh_token", ""), + regular_bearer_token=d.get("regular_bearer_token", ""), + regular_api_key=d.get("regular_api_key", ""), + regular_bearer_refresh_token=d.get("regular_bearer_refresh_token", ""), weak_candidates=d.get("weak_candidates"), max_weak_attempts=d.get("max_weak_attempts", 5), app_routes=d.get("app_routes"), verify_tls=d.get("verify_tls", True), target_config=d.get("target_config"), allow_stateful_probes=d.get("allow_stateful_probes", False), + graybox_assignment_strategy=d.get("graybox_assignment_strategy", "MIRROR"), + assigned_scenario_ids=d.get("assigned_scenario_ids"), + assigned_request_budget=d.get("assigned_request_budget", 0), + budget_scope=d.get("budget_scope", ""), + assignment_revision=d.get("assignment_revision", 0), + assignment_hash=d.get("assignment_hash", ""), + stateful_policy=d.get("stateful_policy", ""), engagement=d.get("engagement"), roe=d.get("roe"), authorization=d.get("authorization"), diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index a601fbd7..c8ab4bd6 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -38,6 +38,12 @@ class CStoreWorker: terminal_reason: str = None error: str = None unreachable_at: float = None + graybox_assignment_strategy: str = None + assigned_scenario_ids: list = None + assigned_request_budget: int = None + budget_scope: str = None + assignment_hash: str = None + stateful_policy: str = None def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -59,6 +65,12 @@ def from_dict(cls, d: dict) -> CStoreWorker: terminal_reason=d.get("terminal_reason"), error=d.get("error"), unreachable_at=d.get("unreachable_at"), + graybox_assignment_strategy=d.get("graybox_assignment_strategy"), + assigned_scenario_ids=d.get("assigned_scenario_ids"), + assigned_request_budget=d.get("assigned_request_budget"), + budget_scope=d.get("budget_scope"), + assignment_hash=d.get("assignment_hash"), + stateful_policy=d.get("stateful_policy"), ) @@ -116,6 +128,7 @@ class CStoreJobRunning: stix_export: dict = None opencti_export: dict = None taxii_export: dict = None + graybox_assignment_summary: dict = None def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -150,6 +163,7 @@ def from_dict(cls, d: dict) -> CStoreJobRunning: stix_export=d.get("stix_export"), opencti_export=d.get("opencti_export"), taxii_export=d.get("taxii_export"), + graybox_assignment_summary=d.get("graybox_assignment_summary"), ) @@ -186,6 +200,7 @@ class CStoreJobFinalized: stix_export: dict = None opencti_export: dict = None taxii_export: dict = None + graybox_assignment_summary: dict = None def to_dict(self) -> dict: return _strip_none(asdict(self)) @@ -218,6 +233,7 @@ def from_dict(cls, d: dict) -> CStoreJobFinalized: stix_export=d.get("stix_export"), opencti_export=d.get("opencti_export"), taxii_export=d.get("taxii_export"), + graybox_assignment_summary=d.get("graybox_assignment_summary"), ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 2be7bd3e..4ce67b33 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -133,6 +133,11 @@ validation_error, ) from .repositories import ArtifactRepository, JobStateRepository +from .graybox.scenario_runtime import ( + GrayboxWorkerAssignment, + rehash_worker_assignment_dict, + synthesize_legacy_mirror_assignment, +) # Human-readable phase labels for progress reporting PHASE_LABELS = { @@ -628,7 +633,12 @@ def _get_job_config(self, job_specs, resolve_secrets=False): return {} config = config_model.to_dict() if resolve_secrets: - return resolve_job_config_secrets(self, config, include_secret_metadata=False) + return resolve_job_config_secrets( + self, + config, + include_secret_metadata=False, + expected_job_id=job_specs.get("job_id", ""), + ) return config @@ -924,9 +934,39 @@ def _maybe_launch_jobs(self, nr_local_workers=None): color='y', ) continue - # Fetch job config from R1FS - job_config = self._get_job_config(job_specs, resolve_secrets=True) try: + # Fetch job config from R1FS and resolve runtime-only secrets. + job_config = self._get_job_config(job_specs, resolve_secrets=True) + if job_specs.get("scan_type") == ScanType.WEBAPP.value: + # PR406 B7: synthesize a legacy MIRROR assignment when a webapp + # job comes from a pre-PR launcher (no assignment fields at + # all). Partial/corrupt assignments must still fail closed. + compat_assignment = synthesize_legacy_mirror_assignment( + job_config, worker_entry, + ) + if compat_assignment is not None: + worker_entry = {**worker_entry, **compat_assignment} + self.P( + f"[GRAYBOX] Using legacy MIRROR compatibility assignment for " + f"job_id={job_id} worker={self.ee_addr} — upgrade the " + f"launcher to publish explicit assignments.", + color='y', + ) + self._log_audit_event("graybox_legacy_mirror_compat", { + "job_id": job_id, + "worker_addr": self.ee_addr, + }) + job_config = PentesterApi01Plugin._with_worker_assignment( + job_config, worker_entry, + ) + assignment = GrayboxWorkerAssignment.from_job_config( + JobConfig.from_dict(job_config), + ) + if not assignment.is_valid: + raise ValueError( + "graybox_assignment_invalid:" + + assignment.validation_error + ) local_jobs = launch_local_jobs( self, job_id=job_id, @@ -939,15 +979,27 @@ def _maybe_launch_jobs(self, nr_local_workers=None): ) except ValueError as exc: self.P(f"Skipping job {job_id}: {exc}", color='r') - worker_entry["finished"] = True - worker_entry["error"] = str(exc) - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="launch_error_value") + reason = "secret_resolution_failed" if ( + "secret_ref" in str(exc) + or "resolve graybox secret" in str(exc) + ) else "launch_validation_failed" + if str(exc).startswith("graybox_assignment_invalid:"): + reason = "assignment_validation_failed" + PentesterApi01Plugin._mark_worker_terminal_error( + self, job_specs, self.ee_addr, reason, str(exc), + context="launch_error_value", + ) continue except Exception as exc: self.P(f"Skipping job {job_id}: {exc}", color='r') - worker_entry["finished"] = True - worker_entry["error"] = str(exc) - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="launch_error_exception") + PentesterApi01Plugin._mark_worker_terminal_error( + self, + job_specs, + self.ee_addr, + "launch_failed", + str(exc), + context="launch_error_exception", + ) continue started_at = self.time() self.scan_jobs[job_id] = local_jobs @@ -964,6 +1016,77 @@ def _maybe_launch_jobs(self, nr_local_workers=None): #endif it is time to check return + @staticmethod + def _with_worker_assignment(job_config, worker_entry): + """Overlay launcher-owned worker assignment into runtime JobConfig.""" + config = dict(job_config or {}) + for key in ( + "graybox_assignment_strategy", + "assigned_scenario_ids", + "assigned_request_budget", + "budget_scope", + "assignment_revision", + "assignment_hash", + "stateful_policy", + ): + if isinstance(worker_entry, dict) and key in worker_entry: + config[key] = worker_entry.get(key) + return config + + def _mark_worker_terminal_error( + self, job_specs, worker_addr, reason, error, context="worker_terminal_error", + ): + """Mark one worker terminal in the shared job record and persist it. + + PR406 B8: instead of writing the launcher's stale snapshot back over + whatever the latest CStore record looks like, reload the current + record and patch only ``workers[worker_addr]``. Concurrent terminal + writes from two workers then merge by worker key instead of clobbering + each other. If the current record can't be loaded, fall back to the + incoming snapshot (with a warning). + """ + if not isinstance(job_specs, dict): + return None + sanitize = getattr(getattr(self, "safety", None), "sanitize_error", None) + sanitized = sanitize(str(error)) if callable(sanitize) else str(error) + if not isinstance(sanitized, str): + sanitized = str(error) + + def _patch_worker(entry: dict): + entry["finished"] = True + entry["terminal_reason"] = reason + entry["error"] = sanitized + entry["result"] = None + return entry + + job_id = job_specs.get("job_id", "") + current = None + if job_id: + current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) + + # Always reflect the patch in the caller's snapshot so any code that + # inspects job_specs after this call sees the worker as terminal. + workers_local = job_specs.setdefault("workers", {}) + _patch_worker(workers_local.setdefault(worker_addr, {})) + + if not isinstance(current, dict): + self.P( + f"[CSTORE] No current job record for {job_id}; writing stale snapshot for worker {worker_addr}", + color='y', + ) + return PentesterApi01Plugin._write_job_record( + self, job_id, job_specs, context=context, + ) + + # Merge: keep current top-level state, overlay the patched worker. + merged_workers = dict(current.get("workers") or {}) + merged_workers[worker_addr] = _patch_worker(dict(merged_workers.get(worker_addr) or {})) + merged = dict(current) + merged["workers"] = merged_workers + return PentesterApi01Plugin._write_job_record( + self, job_id, merged, context=context, + ) + def _log_audit_event(self, event_type, details): """ @@ -1122,6 +1245,11 @@ def _maybe_reannounce_worker_assignments(self): current_revision = PentesterApi01Plugin._get_worker_assignment_revision(target_worker) target_worker["assignment_revision"] = current_revision + 1 + # PR406 B6: assignment_hash includes assignment_revision, so it must + # be recomputed whenever the revision is bumped — otherwise the + # worker would reject the reannounced assignment with + # assignment_hash_mismatch even though it is legitimate. + rehash_worker_assignment_dict(target_worker) target_worker["reannounce_count"] = reannounce_count + 1 target_worker["last_reannounce_at"] = now target_worker["retry_reason"] = retry_reason @@ -2083,9 +2211,27 @@ def _build_network_workers(self, active_peers, start_port, end_port, distributio """Build peer assignments for network scans.""" return build_network_workers(self, active_peers, start_port, end_port, distribution_strategy) - def _build_webapp_workers(self, active_peers, target_port): + def _build_webapp_workers( + self, + active_peers, + target_port, + graybox_assignment_strategy="SLICE", + request_budget=1000, + allow_stateful_probes=False, + allow_mirror_stateful=False, + allow_mirror_per_worker_budget=False, + ): """Build peer assignments for webapp scans. Every peer gets the same target.""" - return build_webapp_workers(self, active_peers, target_port) + return build_webapp_workers( + self, + active_peers, + target_port, + graybox_assignment_strategy=graybox_assignment_strategy, + request_budget=request_budget, + allow_stateful_probes=allow_stateful_probes, + allow_mirror_stateful=allow_mirror_stateful, + allow_mirror_per_worker_budget=allow_mirror_per_worker_budget, + ) def _announce_launch( self, @@ -2261,6 +2407,17 @@ def launch_webapp_scan( verify_tls: bool = True, target_config: dict = None, allow_stateful_probes: bool = False, + bearer_token: str = "", + api_key: str = "", + bearer_refresh_token: str = "", + regular_bearer_token: str = "", + regular_api_key: str = "", + regular_bearer_refresh_token: str = "", + target_config_secrets: dict = None, + request_budget: int = None, + graybox_assignment_strategy: str = "SLICE", + allow_mirror_stateful: bool = False, + allow_mirror_per_worker_budget: bool = False, target_confirmation: str = "", scope_id: str = "", authorization_ref: str = "", @@ -2270,7 +2427,7 @@ def launch_webapp_scan( roe: dict = None, authorization: dict = None, ): - """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment.""" + """Launch a graybox webapp scan using webapp-specific validation and (by default) SLICE worker assignment.""" return launch_webapp_scan( self, target_url=target_url, @@ -2299,6 +2456,17 @@ def launch_webapp_scan( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + bearer_token=bearer_token, + api_key=api_key, + bearer_refresh_token=bearer_refresh_token, + regular_bearer_token=regular_bearer_token, + regular_api_key=regular_api_key, + regular_bearer_refresh_token=regular_bearer_refresh_token, + target_config_secrets=target_config_secrets, + request_budget=request_budget, + graybox_assignment_strategy=graybox_assignment_strategy, + allow_mirror_stateful=allow_mirror_stateful, + allow_mirror_per_worker_budget=allow_mirror_per_worker_budget, target_confirmation=target_confirmation, scope_id=scope_id, authorization_ref=authorization_ref, @@ -2345,6 +2513,17 @@ def launch_test( verify_tls: bool = True, target_config: dict = None, allow_stateful_probes: bool = False, + bearer_token: str = "", + api_key: str = "", + bearer_refresh_token: str = "", + regular_bearer_token: str = "", + regular_api_key: str = "", + regular_bearer_refresh_token: str = "", + target_config_secrets: dict = None, + request_budget: int = None, + graybox_assignment_strategy: str = "SLICE", + allow_mirror_stateful: bool = False, + allow_mirror_per_worker_budget: bool = False, target_confirmation: str = "", scope_id: str = "", authorization_ref: str = "", @@ -2391,6 +2570,17 @@ def launch_test( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + bearer_token=bearer_token, + api_key=api_key, + bearer_refresh_token=bearer_refresh_token, + regular_bearer_token=regular_bearer_token, + regular_api_key=regular_api_key, + regular_bearer_refresh_token=regular_bearer_refresh_token, + target_config_secrets=target_config_secrets, + request_budget=request_budget, + graybox_assignment_strategy=graybox_assignment_strategy, + allow_mirror_stateful=allow_mirror_stateful, + allow_mirror_per_worker_budget=allow_mirror_per_worker_budget, target_confirmation=target_confirmation, scope_id=scope_id, authorization_ref=authorization_ref, diff --git a/extensions/business/cybersec/red_mesh/services/launch_api.py b/extensions/business/cybersec/red_mesh/services/launch_api.py index d9c3e0d0..96c7933e 100644 --- a/extensions/business/cybersec/red_mesh/services/launch_api.py +++ b/extensions/business/cybersec/red_mesh/services/launch_api.py @@ -20,7 +20,20 @@ JobConfig, RulesOfEngagement, ) +from ..graybox.models.target_config import ( + GrayboxTargetConfig, + collect_target_config_secret_refs, + validate_target_config_secret_ref_positions, +) +from ..graybox.http_client import validate_target_config_paths from ..repositories import JobStateRepository +from ..graybox.scenario_runtime import ( + GRAYBOX_ASSIGNMENT_MIRROR, + GRAYBOX_ASSIGNMENT_SLICE, + GRAYBOX_DEFAULT_REQUEST_BUDGET, + build_graybox_worker_assignments, + summarize_graybox_worker_assignments, +) from .config import get_graybox_budgets_config from .event_hooks import emit_attestation_status_event, emit_lifecycle_event from .secrets import persist_job_config_with_secrets @@ -38,6 +51,147 @@ def validation_error(message: str): return {"error": "validation_error", "message": message} +def _parse_positive_int(value, field_path: str, *, default=None, + maximum: int | None = None): + """Parse a launcher numeric input as a positive integer. + + Numeric strings are accepted for UI/API compatibility. Invalid, + boolean, zero, and negative values are rejected instead of silently + falling back to defaults because these values affect scanner safety. + """ + if value is None: + value = default + if isinstance(value, bool): + return None, validation_error(f"{field_path} must be a positive integer") + if isinstance(value, str): + value = value.strip() + if isinstance(value, float) and not value.is_integer(): + return None, validation_error(f"{field_path} must be a positive integer") + if value == "" or value is None: + return None, validation_error(f"{field_path} must be a positive integer") + try: + parsed = int(value) + except (TypeError, ValueError): + return None, validation_error(f"{field_path} must be a positive integer") + if parsed <= 0: + return None, validation_error(f"{field_path} must be greater than 0") + if maximum is not None and parsed > maximum: + return None, validation_error( + f"{field_path} must be less than or equal to {maximum}" + ) + return parsed, None + + +def _validate_positive_int_field(container, key, field_path: str, *, + maximum: int | None = None): + if not isinstance(container, dict) or key not in container: + return None + parsed, err = _parse_positive_int( + container.get(key), field_path, maximum=maximum, + ) + if err: + return err + container[key] = parsed + return None + + +def _validate_positive_int_list(container, key, field_path: str): + if not isinstance(container, dict) or key not in container: + return None + values = container.get(key) + if not isinstance(values, list): + return validation_error(f"{field_path} must be a list of positive integers") + parsed_values = [] + for idx, value in enumerate(values): + parsed, err = _parse_positive_int(value, f"{field_path}[{idx}]") + if err: + return err + parsed_values.append(parsed) + container[key] = parsed_values + return None + + +def _validate_graybox_numeric_fields(canonical: dict | None): + """Validate scanner-safety numeric fields in canonical target_config.""" + if not isinstance(canonical, dict): + return None + discovery = canonical.get("discovery") or {} + for key in ("max_pages", "max_depth"): + err = _validate_positive_int_field(discovery, key, f"discovery.{key}") + if err: + return err + + access = canonical.get("access_control") or {} + for idx, endpoint in enumerate(access.get("idor_endpoints") or []): + err = _validate_positive_int_list( + endpoint, "test_ids", f"access_control.idor_endpoints[{idx}].test_ids", + ) + if err: + return err + + api = canonical.get("api_security") or {} + err = _validate_positive_int_field( + api, "max_total_requests", "api_security.max_total_requests", + ) + if err: + return err + for idx, endpoint in enumerate(api.get("object_endpoints") or []): + err = _validate_positive_int_list( + endpoint, "test_ids", f"api_security.object_endpoints[{idx}].test_ids", + ) + if err: + return err + for idx, endpoint in enumerate(api.get("property_endpoints") or []): + err = _validate_positive_int_field( + endpoint, "test_id", + f"api_security.property_endpoints[{idx}].test_id", + ) + if err: + return err + for idx, endpoint in enumerate(api.get("resource_endpoints") or []): + for key in ("baseline_limit", "abuse_limit"): + err = _validate_positive_int_field( + endpoint, key, f"api_security.resource_endpoints[{idx}].{key}", + ) + if err: + return err + err = _validate_positive_int_field( + endpoint, "oversized_payload_bytes", + f"api_security.resource_endpoints[{idx}].oversized_payload_bytes", + maximum=262_144, + ) + if err: + return err + return None + + +def _validate_api_auth_descriptor(auth_desc): + auth_type = getattr(auth_desc, "auth_type", "form") or "form" + if auth_type not in ("bearer", "api_key"): + return None + probe_path = (getattr(auth_desc, "authenticated_probe_path", "") or "").strip() + allow_unverified = bool(getattr(auth_desc, "allow_unverified_auth", False)) + if not probe_path and not allow_unverified: + return validation_error( + "api_security.auth.authenticated_probe_path is required for bearer/api_key " + "auth unless allow_unverified_auth=true" + ) + if not probe_path: + return None + method = ( + getattr(auth_desc, "authenticated_probe_method", "GET") or "GET" + ).upper() + allow_non_readonly = bool( + getattr(auth_desc, "allow_non_readonly_auth_validation_method", False) + ) + if method not in ("GET", "HEAD") and not allow_non_readonly: + return validation_error( + "api_security.auth.authenticated_probe_method must be GET or HEAD " + "unless allow_non_readonly_auth_validation_method=true" + ) + return None + + def _normalize_allowlist(entries): if not entries: return [] @@ -112,6 +266,66 @@ def _extract_discovery_max_pages(target_config) -> int: return 50 +def _validate_graybox_target_config(target_config): + """Validate typed graybox target_config before workers see it.""" + _, _, error = normalize_graybox_target_config(target_config) + return error + + +def normalize_graybox_target_config(target_config, target_config_secrets=None): + """Validate and canonicalize graybox target_config. + + Returns ``(typed_config, canonical_dict, error)``. ``canonical_dict`` is + the only target_config shape that may be persisted; it is emitted from + the typed dataclasses after unknown-key and nested-secret validation. + """ + if target_config is None: + if target_config_secrets: + return None, None, validation_error( + "target_config_secrets were provided but target_config has no secret_ref entries" + ) + return GrayboxTargetConfig(), None, None + if not isinstance(target_config, dict): + return None, None, validation_error("target_config must be a JSON object") + if target_config_secrets is not None and not isinstance(target_config_secrets, dict): + return None, None, validation_error("target_config_secrets must be a JSON object when provided") + if isinstance(target_config_secrets, dict): + for key, value in target_config_secrets.items(): + if not isinstance(key, str) or not key.strip(): + return None, None, validation_error("target_config_secrets keys must be non-empty strings") + if not isinstance(value, str): + return None, None, validation_error( + f"target_config_secrets[{key!r}] must be a string" + ) + try: + typed_config = GrayboxTargetConfig.from_dict(deepcopy(target_config)) + canonical = typed_config.to_dict() + numeric_error = _validate_graybox_numeric_fields(canonical) + if numeric_error: + return None, None, numeric_error + typed_config = GrayboxTargetConfig.from_dict(deepcopy(canonical)) + validate_target_config_secret_ref_positions(canonical) + required_refs = collect_target_config_secret_refs(canonical) + provided_refs = set((target_config_secrets or {}).keys()) + missing_refs = [ref for ref in required_refs if ref not in provided_refs] + if missing_refs: + return None, None, validation_error( + "target_config secret_ref value(s) missing from target_config_secrets: " + + ", ".join(missing_refs) + ) + unknown_refs = sorted(provided_refs - set(required_refs)) + if unknown_refs: + return None, None, validation_error( + "target_config_secrets contains unknown secret_ref value(s): " + + ", ".join(unknown_refs) + ) + except KeyError as exc: + return None, None, validation_error(f"target_config is missing required field: {exc}") + except (TypeError, ValueError) as exc: + return None, None, validation_error(f"target_config is invalid: {exc}") + return typed_config, canonical, None + + def _validate_authorization_context( owner, *, @@ -397,10 +611,31 @@ def build_network_workers(owner, active_peers, start_port, end_port, distributio return workers, None -def build_webapp_workers(owner, active_peers, target_port): +def build_webapp_workers( + owner, + active_peers, + target_port, + *, + graybox_assignment_strategy, + request_budget=GRAYBOX_DEFAULT_REQUEST_BUDGET, + allow_stateful_probes=False, + allow_mirror_stateful=False, + allow_mirror_per_worker_budget=False, +): """Build peer assignments for webapp scans. Every peer gets the same target.""" if not active_peers: return None, validation_error("No workers available for job execution.") + assignments, assignment_error = build_graybox_worker_assignments( + active_peers, + strategy=graybox_assignment_strategy, + total_request_budget=request_budget, + allow_stateful=allow_stateful_probes, + allow_mirror_stateful=allow_mirror_stateful, + allow_mirror_per_worker_budget=allow_mirror_per_worker_budget, + assignment_revision=1, + ) + if assignment_error: + return None, validation_error(assignment_error) workers = {} for address in active_peers: workers[address] = { @@ -408,6 +643,7 @@ def build_webapp_workers(owner, active_peers, target_port): "end_port": target_port, "finished": False, "result": None, + **assignments[address], } return workers, None @@ -455,9 +691,17 @@ def announce_launch( engagement_metadata, target_allowlist, safety_policy, + graybox_assignment_strategy=GRAYBOX_ASSIGNMENT_SLICE, engagement=None, roe=None, authorization=None, + bearer_token="", + api_key="", + bearer_refresh_token="", + regular_bearer_token="", + regular_api_key="", + regular_bearer_refresh_token="", + target_config_secrets=None, ): """Persist immutable config, announce job in CStore, and return launch response.""" excluded_features, enabled_features = resolve_enabled_features( @@ -517,20 +761,35 @@ def announce_launch( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + graybox_assignment_strategy=graybox_assignment_strategy, engagement=engagement, roe=roe, authorization=authorization, + # OWASP API Top 10 (Subphase 1.5 commit #8): runtime-only secret + # fields. Blanked by `_blank_graybox_secret_fields` before persistence; + # `has_bearer_token` / `has_api_key` capability flags are set on the + # persisted JobConfig by `persist_job_config_with_secrets`. + bearer_token=bearer_token, + api_key=api_key, + bearer_refresh_token=bearer_refresh_token, + regular_bearer_token=regular_bearer_token, + regular_api_key=regular_api_key, + regular_bearer_refresh_token=regular_bearer_refresh_token, ) persisted_config, job_config_cid = persist_job_config_with_secrets( owner, job_id=job_id, - config_dict=job_config.to_dict(), + config_dict={ + **job_config.to_dict(), + "target_config_secrets": deepcopy(target_config_secrets or {}), + }, ) if not job_config_cid: owner.P("Failed to store job config in R1FS — aborting launch", color='r') return {"error": "Failed to store job config in R1FS"} + assignment_summary = summarize_graybox_worker_assignments(workers) if scan_type == ScanType.WEBAPP.value else {} job_specs = CStoreJobRunning( job_id=job_id, job_status=JOB_STATUS_RUNNING, @@ -551,6 +810,7 @@ def announce_launch( pass_reports=[], next_pass_at=None, risk_score=0, + graybox_assignment_summary=assignment_summary or None, ).to_dict() owner._emit_timeline_event( job_specs, "created", @@ -836,12 +1096,80 @@ def launch_webapp_scan( engagement=None, roe=None, authorization=None, + # OWASP API Top 10 (Subphase 1.5 commit #8) — top-level secret params. + # These NEVER appear inside the persisted JobConfig: they flow straight + # into the R1FS secret payload via persist_job_config_with_secrets and + # are zeroised on the public config before put_job_config(). + bearer_token="", + api_key="", + bearer_refresh_token="", + regular_bearer_token="", + regular_api_key="", + regular_bearer_refresh_token="", + target_config_secrets=None, + # OWASP API Top 10 — Subphase 1.7. When set, overrides + # `target_config.api_security.max_total_requests` for the scan. + request_budget=None, + graybox_assignment_strategy=GRAYBOX_ASSIGNMENT_SLICE, + allow_mirror_stateful=False, + allow_mirror_per_worker_budget=False, ): - """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment.""" + """Launch a graybox webapp scan using webapp-specific validation and mirrored worker assignment. + + ``target_config`` is parsed through ``GrayboxTargetConfig`` before any + authorization or persistence path sees it. The persisted ``JobConfig`` + receives only the typed canonical dict returned by + ``normalize_graybox_target_config``; unknown keys and raw nested secret + material in request-body-like fields fail closed at launch. + + Secret-handling: ``bearer_token``, ``api_key``, and + ``bearer_refresh_token`` (Subphase 1.5 commit #8) are top-level launch + parameters — NOT inside ``target_config``. They travel through the + same R1FS secret payload as ``official_password`` and are blanked from + the persisted JobConfig before archive write. Non-secret capability + flags ``has_bearer_token`` / ``has_api_key`` are surfaced on the + archived config so consumers know whether the credentials existed. + """ if not target_url: return validation_error("target_url required for webapp scan") - if not official_username or not official_password: - return validation_error("official credentials required for webapp scan") + max_weak_attempts, numeric_error = _parse_positive_int( + max_weak_attempts, "max_weak_attempts", default=5, + ) + if numeric_error: + return numeric_error + if request_budget is not None: + request_budget, numeric_error = _parse_positive_int( + request_budget, "request_budget", + ) + if numeric_error: + return numeric_error + raw_target_config = deepcopy(target_config) if isinstance(target_config, dict) else target_config + typed_target_config, target_config, config_error = normalize_graybox_target_config( + target_config, + target_config_secrets=target_config_secrets, + ) + if config_error: + return config_error + + # Form auth still requires username+password; Bearer / API-key targets + # set auth_type via target_config.api_security.auth and supply the + # secret as a top-level param instead. + auth_desc = typed_target_config.api_security.auth + auth_type = auth_desc.auth_type + if auth_type == "form": + if not official_username or not official_password: + return validation_error("official credentials required for webapp scan") + elif auth_type == "bearer": + if not bearer_token: + return validation_error("bearer_token required when auth_type='bearer'") + elif auth_type == "api_key": + if not api_key: + return validation_error("api_key required when auth_type='api_key'") + else: + return validation_error(f"unknown auth_type: {auth_type!r}") + auth_validation_error = _validate_api_auth_descriptor(auth_desc) + if auth_validation_error: + return auth_validation_error parsed = urlparse(target_url) if parsed.scheme not in ("http", "https") or not parsed.hostname: @@ -864,6 +1192,13 @@ def launch_webapp_scan( ) if auth_error: return auth_error + path_scope_errors = validate_target_config_paths( + target_url, + raw_target_config, + authorization_context["target_allowlist"], + ) + if path_scope_errors: + return validation_error("; ".join(path_scope_errors)) typed_context, typed_error = _validate_typed_engagement_context( engagement, roe, authorization ) @@ -896,7 +1231,39 @@ def launch_webapp_scan( verify_tls=verify_tls, ) - workers, worker_error = build_webapp_workers(owner, active_peers, target_port) + # OWASP API Top 10 (Subphase 1.7): when the caller passed an explicit + # `request_budget`, inject it into `target_config.api_security` so the + # worker's RequestBudget sizing picks it up over any value the caller + # also placed in target_config. + if request_budget is not None: + if not isinstance(target_config, dict): + target_config = {} + api_security = dict(target_config.get("api_security") or {}) + api_security["max_total_requests"] = request_budget + target_config["api_security"] = api_security + + typed_target_config, target_config, config_error = normalize_graybox_target_config( + target_config, + target_config_secrets=target_config_secrets, + ) + if config_error: + return config_error + effective_request_budget = ( + request_budget + or typed_target_config.api_security.max_total_requests + or GRAYBOX_DEFAULT_REQUEST_BUDGET + ) + + workers, worker_error = build_webapp_workers( + owner, + active_peers, + target_port, + graybox_assignment_strategy=graybox_assignment_strategy, + request_budget=effective_request_budget, + allow_stateful_probes=allow_stateful_probes, + allow_mirror_stateful=allow_mirror_stateful, + allow_mirror_per_worker_budget=allow_mirror_per_worker_budget, + ) if worker_error: return worker_error @@ -936,6 +1303,7 @@ def launch_webapp_scan( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + graybox_assignment_strategy=graybox_assignment_strategy, target_confirmation=authorization_context["target_confirmation"], scope_id=authorization_context["scope_id"], authorization_ref=authorization_context["authorization_ref"], @@ -945,6 +1313,13 @@ def launch_webapp_scan( engagement=typed_context["engagement"], roe=typed_context["roe"], authorization=typed_context["authorization"], + bearer_token=bearer_token, + api_key=api_key, + bearer_refresh_token=bearer_refresh_token, + regular_bearer_token=regular_bearer_token, + regular_api_key=regular_api_key, + regular_bearer_refresh_token=regular_bearer_refresh_token, + target_config_secrets=target_config_secrets, ) @@ -985,6 +1360,17 @@ def launch_test( verify_tls=True, target_config=None, allow_stateful_probes=False, + bearer_token="", + api_key="", + bearer_refresh_token="", + regular_bearer_token="", + regular_api_key="", + regular_bearer_refresh_token="", + target_config_secrets=None, + request_budget=None, + graybox_assignment_strategy=GRAYBOX_ASSIGNMENT_SLICE, + allow_mirror_stateful=False, + allow_mirror_per_worker_budget=False, target_confirmation="", scope_id="", authorization_ref="", @@ -1028,6 +1414,17 @@ def launch_test( verify_tls=verify_tls, target_config=target_config, allow_stateful_probes=allow_stateful_probes, + bearer_token=bearer_token, + api_key=api_key, + bearer_refresh_token=bearer_refresh_token, + regular_bearer_token=regular_bearer_token, + regular_api_key=regular_api_key, + regular_bearer_refresh_token=regular_bearer_refresh_token, + target_config_secrets=target_config_secrets, + request_budget=request_budget, + graybox_assignment_strategy=graybox_assignment_strategy, + allow_mirror_stateful=allow_mirror_stateful, + allow_mirror_per_worker_budget=allow_mirror_per_worker_budget, target_confirmation=target_confirmation, scope_id=scope_id, authorization_ref=authorization_ref, diff --git a/extensions/business/cybersec/red_mesh/services/secrets.py b/extensions/business/cybersec/red_mesh/services/secrets.py index c714d216..2212a444 100644 --- a/extensions/business/cybersec/red_mesh/services/secrets.py +++ b/extensions/business/cybersec/red_mesh/services/secrets.py @@ -3,7 +3,18 @@ from ..models import JobConfig from ..repositories import ArtifactRepository -from .config import get_attestation_config +from ..graybox.models.target_config import ( + collect_target_config_secret_refs, + resolve_target_config_secret_refs, +) +# Built-in fallback secret-store key. Used automatically when no dedicated +# deployment key is configured. The key is identical on every node that ships +# this plugin, so anyone with read access to R1FS-stored secret payloads can +# decrypt them. Deployments that need real key isolation can set +# REDMESH_SECRET_STORE_KEY (env) or cfg_redmesh_secret_store_key (config) — +# absent that, the default key is used and the resulting metadata records +# `unsafe_fallback=True` for auditability. +_DEFAULT_SECRET_STORE_KEY = "redmesh-default-plugin-key-v1" def _artifact_repo(owner): @@ -18,6 +29,7 @@ class R1fsSecretStore: def __init__(self, owner): self.owner = owner + self.last_key_metadata = {} @staticmethod def _normalize_secret_key(value): @@ -26,49 +38,99 @@ def _normalize_secret_key(value): value = value.strip() return value if len(value) >= 8 else "" + @staticmethod + def _truthy(value) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "y", "on"} + return False + + def _dedicated_secret_store_key(self): + env_key = self._normalize_secret_key(os.environ.get("REDMESH_SECRET_STORE_KEY", "")) + if env_key: + return env_key, { + "key_id": os.environ.get("REDMESH_SECRET_STORE_KEY_ID", "env:REDMESH_SECRET_STORE_KEY"), + "key_version": os.environ.get( + "REDMESH_SECRET_STORE_KEY_VERSION", + str(getattr(self.owner, "cfg_redmesh_secret_store_key_version", "") or "v1"), + ), + "key_source": "environment", + "unsafe_fallback": False, + } + cfg_key = self._normalize_secret_key(getattr(self.owner, "cfg_redmesh_secret_store_key", "")) + if cfg_key: + return cfg_key, { + "key_id": str(getattr( + self.owner, + "cfg_redmesh_secret_store_key_id", + "config:cfg_redmesh_secret_store_key", + ) or "config:cfg_redmesh_secret_store_key"), + "key_version": str(getattr( + self.owner, + "cfg_redmesh_secret_store_key_version", + "v1", + ) or "v1"), + "key_source": "config", + "unsafe_fallback": False, + } + return "", {} + + def _default_secret_store_key(self): + return _DEFAULT_SECRET_STORE_KEY, { + "key_id": "redmesh:default_plugin_key", + "key_version": "v1", + "key_source": "redmesh_default", + "unsafe_fallback": True, + } + + def _resolve_secret_store_key(self): + key, metadata = self._dedicated_secret_store_key() + if key: + return key, metadata + return self._default_secret_store_key() + def _get_secret_store_key(self) -> str: - candidates = [ - os.environ.get("REDMESH_SECRET_STORE_KEY", ""), - getattr(self.owner, "cfg_redmesh_secret_store_key", ""), - getattr(self.owner, "cfg_comms_host_key", ""), - get_attestation_config(self.owner)["PRIVATE_KEY"], - ] - for candidate in candidates: - key = self._normalize_secret_key(candidate) - if key: - return key - return "" + key, _metadata = self._resolve_secret_store_key() + return key def save_graybox_credentials(self, job_id: str, payload: dict) -> str: - secret_key = self._get_secret_store_key() - if not secret_key: - self.owner.P( - "No strong RedMesh secret-store key is configured. " - "Graybox launch credentials cannot be persisted safely.", - color='r', - ) - return "" + secret_key, key_metadata = self._resolve_secret_store_key() + self.last_key_metadata = dict(key_metadata or {}) secret_doc = { "kind": "redmesh_graybox_credentials", "job_id": job_id, "storage_mode": "encrypted_r1fs_json_v1", + "key_id": key_metadata.get("key_id", ""), + "key_version": key_metadata.get("key_version", ""), + "key_source": key_metadata.get("key_source", ""), + "unsafe_key_fallback": bool(key_metadata.get("unsafe_fallback", False)), "payload": payload, } return _artifact_repo(self.owner).put_json(secret_doc, show_logs=False, secret=secret_key) - def load_graybox_credentials(self, secret_ref: str) -> dict | None: + def load_graybox_credentials(self, secret_ref: str, *, expected_job_id: str = "") -> dict | None: if not secret_ref: return None repo = _artifact_repo(self.owner) - secret_key = self._get_secret_store_key() - secret_doc = None - if secret_key: - secret_doc = repo.get_json(secret_ref, secret=secret_key) - if not isinstance(secret_doc, dict): - secret_doc = repo.get_json(secret_ref) + secret_key, key_metadata = self._resolve_secret_store_key() + self.last_key_metadata = dict(key_metadata or {}) + secret_doc = repo.get_json(secret_ref, secret=secret_key) if not isinstance(secret_doc, dict): self.owner.P(f"Failed to fetch graybox secret payload from R1FS (CID: {secret_ref})", color='r') return None + if secret_doc.get("kind") != "redmesh_graybox_credentials": + self.owner.P(f"Invalid graybox secret kind for ref {secret_ref}", color='r') + return None + if secret_doc.get("storage_mode") != "encrypted_r1fs_json_v1": + self.owner.P(f"Invalid graybox secret storage mode for ref {secret_ref}", color='r') + return None + if expected_job_id and secret_doc.get("job_id") != expected_job_id: + self.owner.P( + f"Graybox secret ref {secret_ref} belongs to job_id={secret_doc.get('job_id')}, expected {expected_job_id}", + color='r', + ) + return None payload = secret_doc.get("payload") if not isinstance(payload, dict): self.owner.P(f"Invalid graybox secret payload for ref {secret_ref}", color='r') @@ -91,6 +153,14 @@ def _blank_graybox_secret_fields(config_dict: dict) -> dict: sanitized["official_password"] = "" sanitized["regular_username"] = "" sanitized["regular_password"] = "" + # OWASP API Top 10 (Subphase 1.5 commit #8) — header-auth secrets. + sanitized["bearer_token"] = "" + sanitized["api_key"] = "" + sanitized["bearer_refresh_token"] = "" + sanitized["regular_bearer_token"] = "" + sanitized["regular_api_key"] = "" + sanitized["regular_bearer_refresh_token"] = "" + sanitized.pop("target_config_secrets", None) sanitized.pop("weak_candidates", None) return sanitized @@ -110,6 +180,13 @@ def build_graybox_secret_payload( regular_username="", regular_password="", weak_candidates=None, + bearer_token="", + api_key="", + bearer_refresh_token="", + regular_bearer_token="", + regular_api_key="", + regular_bearer_refresh_token="", + target_config_secrets=None, ): return { "official_username": official_username or "", @@ -117,6 +194,14 @@ def build_graybox_secret_payload( "regular_username": regular_username or "", "regular_password": regular_password or "", "weak_candidates": list(weak_candidates) if isinstance(weak_candidates, list) else weak_candidates, + # OWASP API Top 10 (Subphase 1.5 commit #8): API-native auth secrets. + "bearer_token": bearer_token or "", + "api_key": api_key or "", + "bearer_refresh_token": bearer_refresh_token or "", + "regular_bearer_token": regular_bearer_token or "", + "regular_api_key": regular_api_key or "", + "regular_bearer_refresh_token": regular_bearer_refresh_token or "", + "target_config_secrets": dict(target_config_secrets) if isinstance(target_config_secrets, dict) else {}, } @@ -134,7 +219,9 @@ def persist_job_config_with_secrets( tuple[dict, str] Persisted config dict and resulting job_config_cid. """ - persisted_config = _coerce_job_config_dict(config_dict) + raw_config = deepcopy(config_dict or {}) + target_config_secrets = raw_config.get("target_config_secrets") + persisted_config = _coerce_job_config_dict(raw_config) scan_type = persisted_config.get("scan_type", "network") if scan_type == "webapp": payload = build_graybox_secret_payload( @@ -143,6 +230,13 @@ def persist_job_config_with_secrets( regular_username=persisted_config.get("regular_username", ""), regular_password=persisted_config.get("regular_password", ""), weak_candidates=persisted_config.get("weak_candidates"), + bearer_token=persisted_config.get("bearer_token", ""), + api_key=persisted_config.get("api_key", ""), + bearer_refresh_token=persisted_config.get("bearer_refresh_token", ""), + regular_bearer_token=persisted_config.get("regular_bearer_token", ""), + regular_api_key=persisted_config.get("regular_api_key", ""), + regular_bearer_refresh_token=persisted_config.get("regular_bearer_refresh_token", ""), + target_config_secrets=target_config_secrets, ) has_secret_payload = any([ payload["official_username"], @@ -150,23 +244,47 @@ def persist_job_config_with_secrets( payload["regular_username"], payload["regular_password"], payload["weak_candidates"], + payload["bearer_token"], + payload["api_key"], + payload["bearer_refresh_token"], + payload["regular_bearer_token"], + payload["regular_api_key"], + payload["regular_bearer_refresh_token"], + payload["target_config_secrets"], ]) if has_secret_payload: store = R1fsSecretStore(owner) secret_ref = store.save_graybox_credentials(job_id, payload) if not secret_ref: owner.P("Failed to persist graybox secret payload in R1FS — aborting launch", color='r') - return persisted_config, "" + return _blank_graybox_secret_fields(persisted_config), "" persisted_config["secret_ref"] = secret_ref + key_metadata = store.last_key_metadata if isinstance(store.last_key_metadata, dict) else {} + persisted_config["secret_store_key_id"] = key_metadata.get("key_id", "") + persisted_config["secret_store_key_version"] = key_metadata.get("key_version", "") + persisted_config["secret_store_key_source"] = key_metadata.get("key_source", "") + persisted_config["secret_store_unsafe_fallback"] = bool(key_metadata.get("unsafe_fallback", False)) persisted_config["has_regular_credentials"] = bool(payload["regular_username"] or payload["regular_password"]) persisted_config["has_weak_candidates"] = bool(payload["weak_candidates"]) + # OWASP API Top 10 (Subphase 1.5 commit #8) — non-secret capability flags. + persisted_config["has_bearer_token"] = bool(payload["bearer_token"]) + persisted_config["has_api_key"] = bool(payload["api_key"]) + persisted_config["has_bearer_refresh_token"] = bool(payload["bearer_refresh_token"]) + persisted_config["has_regular_bearer_token"] = bool(payload["regular_bearer_token"]) + persisted_config["has_regular_api_key"] = bool(payload["regular_api_key"]) + persisted_config["has_regular_bearer_refresh_token"] = bool(payload["regular_bearer_refresh_token"]) persisted_config = _blank_graybox_secret_fields(persisted_config) job_config_cid = _artifact_repo(owner).put_job_config(persisted_config, show_logs=False) return persisted_config, job_config_cid -def resolve_job_config_secrets(owner, config_dict: dict, include_secret_metadata: bool = True) -> dict: +def resolve_job_config_secrets( + owner, + config_dict: dict, + include_secret_metadata: bool = True, + expected_job_id: str = "", +) -> dict: """ Resolve secret_ref into runtime-only inline credentials for worker execution. @@ -174,14 +292,18 @@ def resolve_job_config_secrets(owner, config_dict: dict, include_secret_metadata - configs without secret_ref are returned unchanged - legacy inline secrets remain supported """ - resolved = _coerce_job_config_dict(config_dict) + raw = deepcopy(config_dict or {}) + expected_job_id = expected_job_id or raw.get("job_id", "") + resolved = _coerce_job_config_dict(raw) secret_ref = resolved.get("secret_ref") if not secret_ref: return resolved - payload = R1fsSecretStore(owner).load_graybox_credentials(secret_ref) + payload = R1fsSecretStore(owner).load_graybox_credentials( + secret_ref, expected_job_id=expected_job_id, + ) if not payload: - return resolved + raise ValueError(f"Failed to resolve graybox secret_ref for job_id={expected_job_id or ''}") resolved.update({ "official_username": payload.get("official_username", ""), @@ -189,7 +311,37 @@ def resolve_job_config_secrets(owner, config_dict: dict, include_secret_metadata "regular_username": payload.get("regular_username", ""), "regular_password": payload.get("regular_password", ""), "weak_candidates": payload.get("weak_candidates"), + # OWASP API Top 10 (Subphase 1.5 commit #8) — API-native auth secrets. + "bearer_token": payload.get("bearer_token", ""), + "api_key": payload.get("api_key", ""), + "bearer_refresh_token": payload.get("bearer_refresh_token", ""), + "regular_bearer_token": payload.get("regular_bearer_token", ""), + "regular_api_key": payload.get("regular_api_key", ""), + "regular_bearer_refresh_token": payload.get("regular_bearer_refresh_token", ""), }) + target_config_secrets = payload.get("target_config_secrets") or {} + target_config_secret_refs = [] + if isinstance(resolved.get("target_config"), dict): + target_config_secret_refs = collect_target_config_secret_refs( + resolved["target_config"] + ) + if target_config_secret_refs and not target_config_secrets: + raise ValueError( + "Failed to resolve target_config secret_ref value(s) " + f"{', '.join(target_config_secret_refs)} for " + f"job_id={expected_job_id or ''}" + ) + if target_config_secret_refs: + try: + resolved["target_config"] = resolve_target_config_secret_refs( + resolved["target_config"], + target_config_secrets, + ) + except KeyError as exc: + raise ValueError( + f"Failed to resolve target_config secret_ref {exc.args[0]!r} " + f"for job_id={expected_job_id or ''}" + ) from exc if not include_secret_metadata: resolved.pop("secret_ref", None) return resolved diff --git a/extensions/business/cybersec/red_mesh/tests/e2e/api_top10_e2e.py b/extensions/business/cybersec/red_mesh/tests/e2e/api_top10_e2e.py new file mode 100644 index 00000000..4bb1f9bc --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/e2e/api_top10_e2e.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""OWASP API Top 10 e2e harness — Phase 7 of the API Top 10 plan. + +Reads the per-scenario manifest at fixtures/api_top10_manifest.yaml, +builds a launch payload from fixtures/api_security_target_config.json, +launches a webapp scan against the rm-gb-poc honeypot, polls for +completion, and asserts: + + - Phase 7.2: vulnerable run — every scenario in the manifest is + present with status=vulnerable + expected severity + evidence keys. + - Phase 7.3: hardened run (`HONEYPOT_HARDEN_API=1`) — same scenario + IDs are present but now status=not_vulnerable; risk score is + materially lower than the vulnerable run. + - Phase 7.4: stateful-gated run (`allow_stateful_probes=false`) — + stateful scenarios emit `inconclusive` with reason + `stateful_probes_disabled`; no state is mutated. + - Phase 7.5: LLM input boundary — no Authorization/Cookie/JWT/long-base64 + blob reaches the LLM input artifact. + +Usage: + python api_top10_e2e.py --rm http://localhost:5082 \\ + --honeypot http://localhost:30001 \\ + --scenario vulnerable|hardened|stateful-gated|llm-boundary|all + +This harness deliberately uses ``urllib`` rather than ``requests`` so it +inherits no extra dependency. PyYAML is the only optional dep — falls +back to a minimal in-tree parser if absent. +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +from pathlib import Path +from typing import Any +from urllib import error, request +from urllib.parse import urlparse + + +HERE = Path(__file__).resolve().parent + + +def _load_yaml(path: Path) -> dict: + try: + import yaml # type: ignore[import-not-found] + return yaml.safe_load(path.read_text()) + except ImportError: + # Minimal fallback parser — handles only the manifest's shape. + return _parse_simple_yaml(path.read_text()) + + +def _parse_simple_yaml(src: str) -> dict: + """Very tiny YAML reader for the manifest. Lists of dicts only, + string scalars (with optional quoted strings), and block scalars.""" + out: dict[str, Any] = {} + current_list: list[dict[str, Any]] | None = None + current_item: dict[str, Any] | None = None + in_block_scalar = False + block_field = "" + block_lines: list[str] = [] + block_indent = 0 + for line in src.splitlines(): + if in_block_scalar: + stripped = line.rstrip() + if stripped == "" or (stripped and stripped.startswith(" " * block_indent)): + block_lines.append(stripped[block_indent:] if stripped else "") + continue + # End of block. + assert current_item is not None + current_item[block_field] = "\n".join(block_lines).strip() + in_block_scalar = False + block_lines = [] + # fall through to re-process this line + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = len(line) - len(line.lstrip()) + s = line.strip() + if indent == 0 and ":" in s and not s.startswith("-"): + k, v = s.split(":", 1) + v = v.strip() + if v == "": + out[k.strip()] = [] + current_list = out[k.strip()] + else: + out[k.strip()] = _scalar(v) + current_list = None + elif s.startswith("- "): + current_item = {} + assert current_list is not None + current_list.append(current_item) + rest = s[2:] + if ":" in rest: + k, v = rest.split(":", 1) + current_item[k.strip()] = _scalar(v.strip()) + elif ":" in s and current_item is not None: + k, v = s.split(":", 1) + v = v.strip() + if v in ("|", ">"): + in_block_scalar = True + block_field = k.strip() + block_indent = indent + 2 + block_lines = [] + else: + current_item[k.strip()] = _scalar(v) + if in_block_scalar and current_item is not None: + current_item[block_field] = "\n".join(block_lines).strip() + return out + + +def _scalar(v: str) -> Any: + v = v.strip() + if v.startswith("[") and v.endswith("]"): + inner = v[1:-1].strip() + if not inner: + return [] + return [_scalar(x.strip().strip('"').strip("'")) for x in inner.split(",")] + if v.startswith('"') and v.endswith('"'): + return v[1:-1] + if v.startswith("'") and v.endswith("'"): + return v[1:-1] + if v in ("true", "True"): + return True + if v in ("false", "False"): + return False + if v.isdigit() or (v.startswith("-") and v[1:].isdigit()): + return int(v) + return v + + +# ── HTTP helpers ──────────────────────────────────────────────────── + +def http_post(url: str, payload: dict, timeout: int = 30) -> dict: + data = json.dumps(payload).encode() + req = request.Request( + url, data=data, method="POST", + headers={"Content-Type": "application/json"}, + ) + with request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + + +def http_get(url: str, timeout: int = 30) -> dict: + with request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + + +def unwrap_result(payload: dict) -> dict: + if isinstance(payload, dict) and isinstance(payload.get("result"), dict): + return payload["result"] + return payload + + +# ── Scan orchestration ────────────────────────────────────────────── + +def target_confirmation_for_url(target_url: str) -> str: + """Return the host value expected by launch-side authorization checks.""" + parsed = urlparse(target_url) + return parsed.hostname or target_url + + +def target_config_with_bearer_auth(target_config: dict) -> dict: + """Return a launch config that exercises API-native bearer auth. + + The honeypot's browser form has CSRF protection, while the API Top 10 + endpoints expose `/api/v2/token/` and `/api/v2/me/` specifically for + API-auth validation. Keep the fixture's scenario inventory intact and + layer only the auth descriptor required by the backend launch contract. + """ + cfg = json.loads(json.dumps(target_config)) + api_security = dict(cfg.get("api_security") or {}) + auth = dict(api_security.get("auth") or {}) + auth.update({ + "auth_type": "bearer", + "bearer_token_header_name": "Authorization", + "bearer_scheme": "Bearer", + "authenticated_probe_path": "/api/v2/me/", + }) + api_security["auth"] = auth + cfg["api_security"] = api_security + return cfg + + +def mint_bearer_token(honeypot: str) -> str: + result = unwrap_result(http_post( + f"{honeypot.rstrip('/')}/api/v2/token/", + {"username": "alice", "password": "secret"}, + )) + token = result.get("token") if isinstance(result, dict) else None + if not token: + raise RuntimeError(f"honeypot token endpoint did not return token: {result}") + return str(token) + +def launch_scan(rm: str, honeypot: str, target_config: dict, *, + allow_stateful: bool = True) -> str: + official_token = mint_bearer_token(honeypot) + regular_token = mint_bearer_token(honeypot) + payload = { + "target_url": honeypot, + "official_username": "alice", + "official_password": "", + "regular_username": "alice", + "regular_password": "", + "target_config": target_config_with_bearer_auth(target_config), + "bearer_token": official_token, + "regular_bearer_token": regular_token, + "allow_stateful_probes": allow_stateful, + "graybox_assignment_strategy": "SLICE", + "authorized": True, + "target_confirmation": target_confirmation_for_url(honeypot), + "task_name": "api-top10-e2e", + } + resp = http_post(f"{rm}/launch_webapp_scan", payload) + result = unwrap_result(resp) + job_id = result.get("job_id") or (result.get("job_specs") or {}).get("job_id") + if not job_id: + raise RuntimeError(f"launch_webapp_scan failed: {resp}") + return job_id + + +def wait_for_finalize(rm: str, job_id: str, timeout: int = 600) -> dict: + deadline = time.time() + timeout + while time.time() < deadline: + try: + resp = unwrap_result(http_get(f"{rm}/get_job_status?job_id={job_id}")) + except (TimeoutError, OSError, error.URLError): + time.sleep(5) + continue + status = ( + resp.get("status") or resp.get("job_status") + or (resp.get("job") or {}).get("job_status") or "" + ) + if str(status).lower() in ("finalized", "done", "completed"): + return resp + time.sleep(5) + raise TimeoutError(f"job {job_id} did not finalize within {timeout}s") + + +def fetch_archive(rm: str, job_id: str) -> dict: + resp = unwrap_result(http_get(f"{rm}/get_job_archive?job_id={job_id}")) + return resp.get("archive", resp) + + +def collect_findings(archive: dict) -> list[dict]: + """Pull every flat finding out of the archive's passes.""" + out: list[dict] = [] + for p in archive.get("passes", []) or []: + out.extend(p.get("findings", []) or []) + return out + + +def llm_boundary_blob_from_archive(archive: dict) -> str: + """Serialize archive fields that are allowed to feed LLM/report stages. + + Deployments do not expose a stable ``/get_job_llm_input`` endpoint yet, so + the harness validates the immutable archive material that backs LLM/report + generation: flat findings, LLM analyses, quick summaries, and structured + report sections. Raw JobConfig and worker stdout are intentionally excluded. + """ + boundary: list[dict[str, Any]] = [] + for p in archive.get("passes", []) or []: + if not isinstance(p, dict): + continue + boundary.append({ + "findings": p.get("findings", []), + "llm_analysis": p.get("llm_analysis"), + "quick_summary": p.get("quick_summary"), + "llm_report_sections": p.get("llm_report_sections"), + }) + return json.dumps(boundary, sort_keys=True, default=str) + + +# ── Assertions ────────────────────────────────────────────────────── + +def assert_vulnerable_run(findings: list[dict], manifest: dict) -> list[str]: + """Phase 7.2: every manifest scenario surfaces as vulnerable.""" + errors: list[str] = [] + by_id: dict[str, dict] = {} + for f in findings: + sid = f.get("scenario_id") + if sid and f.get("status") == "vulnerable": + by_id.setdefault(sid, f) + for entry in manifest["scenarios"]: + sid = entry["id"] + if sid not in by_id: + errors.append(f"missing vulnerable finding for {sid}") + continue + f = by_id[sid] + if f["severity"] != entry["expected_severity"]: + errors.append( + f"{sid}: severity {f['severity']} != expected " + f"{entry['expected_severity']}", + ) + evidence = f.get("evidence", "") + if isinstance(evidence, list): + evidence_text = "\n".join(str(x) for x in evidence) + else: + evidence_text = str(evidence or "") + haystack = evidence_text + "\n" + (f.get("description") or "") + for key in entry.get("expected_evidence_keys", []) or []: + if key not in haystack: + errors.append(f"{sid}: evidence missing substring {key!r}") + return errors + + +def assert_hardened_run(findings: list[dict], manifest: dict) -> list[str]: + errors: list[str] = [] + by_id: dict[str, dict] = {f.get("scenario_id"): f + for f in findings if f.get("scenario_id")} + for entry in manifest["scenarios"]: + sid = entry["id"] + if sid not in by_id: + continue # absence is acceptable in hardened mode for some probes + if by_id[sid].get("status") == "vulnerable": + errors.append( + f"hardened run still reports {sid} as vulnerable", + ) + return errors + + +_LEAK_PATTERNS = [ + re.compile(r"Authorization:\s*Bearer\s+eyJ", re.IGNORECASE), + re.compile(r"Cookie:\s*sessionid=", re.IGNORECASE), + re.compile(r"eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}"), + re.compile(r"password=[^&\s\";<]{8,}"), +] + + +def assert_llm_boundary(llm_input_blob: str) -> list[str]: + errors: list[str] = [] + for pat in _LEAK_PATTERNS: + if pat.search(llm_input_blob): + errors.append(f"LLM input matched leak pattern: {pat.pattern!r}") + return errors + + +# ── CLI ───────────────────────────────────────────────────────────── + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--rm", required=True, help="Edge node base URL") + ap.add_argument("--honeypot", default="http://localhost:30001") + ap.add_argument( + "--scenario", default="all", + choices=("vulnerable", "hardened", "stateful-gated", "llm-boundary", "all"), + ) + ap.add_argument("--timeout", type=int, default=600) + args = ap.parse_args() + + manifest = _load_yaml(HERE / "fixtures" / "api_top10_manifest.yaml") + target_config = json.loads( + (HERE / "fixtures" / "api_security_target_config.json").read_text(), + ) + + ok = True + + last_archive: dict | None = None + + def run(label: str, allow_stateful: bool, assert_fn) -> bool: + nonlocal last_archive + print(f"\n=== {label} ===") + job_id = launch_scan(args.rm, args.honeypot, target_config, + allow_stateful=allow_stateful) + print(f" job_id={job_id}") + wait_for_finalize(args.rm, job_id, timeout=args.timeout) + archive = fetch_archive(args.rm, job_id) + last_archive = archive + findings = collect_findings(archive) + errors = assert_fn(findings, manifest) + if errors: + print(f" FAIL: {len(errors)} assertion errors:") + for e in errors[:20]: + print(f" - {e}") + return False + print(f" OK ({len(findings)} findings)") + return True + + if args.scenario in ("vulnerable", "all"): + ok &= run("Vulnerable run (PHASE 7.2)", True, assert_vulnerable_run) + if args.scenario in ("hardened", "all"): + print("\n → set HONEYPOT_HARDEN_API=1 on the honeypot before continuing") + ok &= run("Hardened run (PHASE 7.3)", True, assert_hardened_run) + if args.scenario in ("stateful-gated", "all"): + print("\n Phase 7.4 — stateful-disabled run; expecting inconclusive findings") + ok &= run("Stateful-gated run", False, + lambda fs, m: ( + ["stateful scenarios must not be vulnerable while gated"] + if any( + f.get("scenario_id") in {"PT-OAPI2-03", "PT-OAPI3-02", "PT-OAPI5-03", "PT-OAPI5-04", "PT-OAPI6-01", "PT-OAPI6-02"} + and f.get("status") == "vulnerable" + for f in fs + ) + else [] + ) + ( + ["stateful-gated run produced no stateful inconclusive findings"] + if not any( + f.get("scenario_id") in {"PT-OAPI2-03", "PT-OAPI3-02", "PT-OAPI5-03", "PT-OAPI5-04", "PT-OAPI6-01", "PT-OAPI6-02"} + and f.get("status") == "inconclusive" + for f in fs + ) + else [] + )) + if args.scenario in ("llm-boundary", "all"): + print("\n Phase 7.5 — verify archive material used for LLM/report input") + if last_archive is None: + job_id = launch_scan(args.rm, args.honeypot, target_config, + allow_stateful=False) + print(f" job_id={job_id}") + wait_for_finalize(args.rm, job_id, timeout=args.timeout) + last_archive = fetch_archive(args.rm, job_id) + errors = assert_llm_boundary(llm_boundary_blob_from_archive(last_archive)) + if errors: + print(f" FAIL: {len(errors)} boundary assertion errors:") + for e in errors[:20]: + print(f" - {e}") + ok = False + else: + print(" OK (no LLM/report-boundary leak patterns)") + + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/extensions/business/cybersec/red_mesh/tests/e2e/fixtures/api_security_target_config.json b/extensions/business/cybersec/red_mesh/tests/e2e/fixtures/api_security_target_config.json new file mode 100644 index 00000000..bb1e7887 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/e2e/fixtures/api_security_target_config.json @@ -0,0 +1,108 @@ +{ + "discovery": { + "scope_prefix": "/", + "max_pages": 20, + "max_depth": 2 + }, + "api_security": { + "max_total_requests": 500, + "object_endpoints": [ + { + "path": "/api/orgs/tenant-a/users/{id}/", + "test_ids": [1, 2], + "owner_field": "username", + "id_param": "id", + "tenant_field": "tenant_id", + "expected_owner": "alice", + "expected_tenant": "tenant-a" + } + ], + "property_endpoints": [ + { + "path": "/api/profile/{id}/", + "method_read": "GET", + "method_write": "PATCH", + "test_id": 1, + "id_param": "id" + } + ], + "function_endpoints": [ + { + "path": "/api/admin/export-users/", + "method": "GET", + "privilege": "admin" + }, + { + "path": "/api/records/force-error/", + "method": "GET", + "privilege": "user", + "allow_malformed_json_probe": true + }, + { + "path": "/api/_debug/routes/", + "method": "GET", + "privilege": "user" + }, + { + "path": "/api/admin/users/2/method-override-promote/", + "method": "POST", + "privilege": "admin", + "revert_path": "/api/admin/users/2/demote/", + "revert_body": {} + }, + { + "path": "/api/admin/users/2/promote/", + "method": "POST", + "privilege": "admin", + "revert_path": "/api/admin/users/2/demote/", + "revert_body": {} + } + ], + "resource_endpoints": [ + { + "path": "/api/records/list/", + "limit_param": "limit", + "baseline_limit": 10, + "abuse_limit": 999999, + "rate_limit_expected": true, + "allow_high_limit_probe": true + }, + { + "path": "/api/notes/", + "rate_limit_expected": false, + "allow_oversized_payload_probe": true, + "oversized_payload_bytes": 65536 + } + ], + "business_flows": [ + { + "path": "/api/auth/signup/", + "method": "POST", + "flow_name": "signup", + "body_template": {"username": "{test_account}", "password": "__redmesh_canary_password__"}, + "revert_path": "/api/auth/signup/cleanup/", + "revert_body": {"username": "{test_account}"}, + "test_account": "abuse_canary" + } + ], + "token_endpoints": { + "token_path": "/api/v2/token/", + "protected_path": "/api/v2/me/", + "logout_path": "/api/v2/auth/logout/", + "weak_secret_candidates": ["secret", "changeme", "password", "jwt"] + }, + "inventory_paths": { + "openapi_candidates": ["/openapi.json", "/swagger.json"], + "current_version": "/api/v2/records/", + "canonical_probe_path": "/api/v2/records/1/", + "version_sibling_candidates": ["/api/v1/records/", "/api/v0/records/"], + "private_path_patterns": ["/internal/"], + "deprecated_paths": ["/api/v0/legacy/"] + }, + "debug_path_candidates": ["/api/_debug/routes/"], + "ssrf_body_fields": ["callback_url", "url", "webhook"] + }, + "injection": { + "ssrf_endpoints": [{"path": "api/webhook/test/", "param": "url"}] + } +} diff --git a/extensions/business/cybersec/red_mesh/tests/e2e/fixtures/api_top10_manifest.yaml b/extensions/business/cybersec/red_mesh/tests/e2e/fixtures/api_top10_manifest.yaml new file mode 100644 index 00000000..4edaa40d --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/e2e/fixtures/api_top10_manifest.yaml @@ -0,0 +1,202 @@ +# OWASP API Top 10 e2e manifest — Subphase 7.1 of the API Top 10 plan. +# +# Single source of truth consumed by: +# - the e2e harness (run_e2e.py) to build the target_config.api_security +# payload and to assert expected findings +# - the honeypot inventory (rm-gb-poc/docs/honeypot-scenario-inventory.md) +# for the "what the honeypot serves" review +# +# Each entry maps one scenario to: +# honeypot_path — endpoint exercised by the probe +# method — HTTP method (informational; probe owns the actual call) +# expected_severity — severity the vulnerable run must emit +# expected_evidence_keys — substrings every finding must carry in evidence +# hardened_status — what the same probe should emit when HONEYPOT_HARDEN_API=1 +# revert_path — companion endpoint required by Subphase 1.8 stateful probes +# notes — free-form + +honeypot_base_url: "http://localhost:30001" +hardened_env_var: HONEYPOT_HARDEN_API + +scenarios: + - id: PT-OAPI1-01 + honeypot_path: "/api/orgs/tenant-a/users/2/" + method: GET + expected_severity: HIGH + expected_evidence_keys: ["endpoint=", "owner_value=", "tenant_field=tenant_id"] + hardened_status: not_vulnerable + notes: | + uid=2 lives under tenant-b; an authenticated alice (tenant-a) should + be rejected in hardened mode and returned in full in vulnerable mode. + + - id: PT-OAPI2-01 + honeypot_path: "/api/v2/me/" + method: GET + expected_severity: CRITICAL + expected_evidence_keys: ["forged_alg=none", "server_returned_status=200"] + hardened_status: not_vulnerable + + - id: PT-OAPI2-02 + honeypot_path: "/api/v2/me/" + method: GET + expected_severity: HIGH + expected_evidence_keys: ["weak_secret_was=", "verification=local_HS256_compare"] + hardened_status: not_vulnerable + + - id: PT-OAPI2-03 + honeypot_path: "/api/v2/me/" + method: GET + expected_severity: MEDIUM + expected_evidence_keys: ["logout_path=/api/v2/auth/logout/"] + hardened_status: not_vulnerable + notes: stateful; requires allow_stateful_probes=true + + - id: PT-OAPI3-01 + honeypot_path: "/api/profile/1/" + method: GET + expected_severity: HIGH + expected_evidence_keys: ["sensitive_fields_present="] + hardened_status: not_vulnerable + + - id: PT-OAPI3-02 + honeypot_path: "/api/profile/1/" + method: PATCH + expected_severity: HIGH + expected_evidence_keys: ["tampered_field=is_admin"] + hardened_status: not_vulnerable + notes: stateful; requires allow_stateful_probes=true + + - id: PT-OAPI4-01 + honeypot_path: "/api/records/list/" + method: GET + expected_severity: MEDIUM + expected_evidence_keys: ["abuse_size_bytes="] + hardened_status: not_vulnerable + + - id: PT-OAPI4-02 + honeypot_path: "/api/notes/" + method: POST + expected_severity: MEDIUM + expected_evidence_keys: ["body_bytes="] + hardened_status: not_vulnerable + + - id: PT-OAPI4-03 + honeypot_path: "/api/records/list/" + method: GET + expected_severity: LOW + expected_evidence_keys: ["requests_sent=", "rate_limit_signals=absent"] + hardened_status: not_vulnerable + notes: requires `rate_limit_expected=true` in target_config + + - id: PT-OAPI5-01 + honeypot_path: "/api/admin/export-users/" + method: GET + expected_severity: CRITICAL + expected_evidence_keys: ["principal=regular", "marker_absent=true"] + hardened_status: not_vulnerable + + - id: PT-OAPI5-02 + honeypot_path: "/api/admin/export-users/" + method: GET + expected_severity: CRITICAL + expected_evidence_keys: ["principal=anonymous"] + hardened_status: not_vulnerable + + - id: PT-OAPI5-03 + honeypot_path: "/api/admin/users/2/method-override-promote/" + method: POST + expected_severity: HIGH + expected_evidence_keys: ["plain_status=403", "override_header=X-HTTP-Method-Override"] + hardened_status: not_vulnerable + revert_path: "/api/admin/users/2/demote/" + notes: stateful + + - id: PT-OAPI5-04 + honeypot_path: "/api/admin/users/2/promote/" + method: POST + expected_severity: CRITICAL + expected_evidence_keys: ["method=POST", "principal=regular"] + hardened_status: not_vulnerable + revert_path: "/api/admin/users/2/demote/" + notes: stateful + + - id: PT-OAPI6-01 + honeypot_path: "/api/auth/signup/" + method: POST + expected_severity: MEDIUM + expected_evidence_keys: ["flow=signup", "attempts=5"] + hardened_status: not_vulnerable + revert_path: "/api/auth/signup/cleanup/" + notes: stateful — creates duplicate accounts; honeypot rate-limits in hardened mode + + - id: PT-OAPI6-02 + honeypot_path: "/api/auth/signup/" + method: POST + expected_severity: MEDIUM + expected_evidence_keys: ["duplicate_accepted=true"] + hardened_status: not_vulnerable + revert_path: "/api/auth/signup/cleanup/" + + - id: PT-API7-01 + honeypot_path: "/api/webhook/test/" + method: POST + expected_severity: HIGH + expected_evidence_keys: ["body_field=callback_url", "reflected_marker=internal-probe"] + hardened_status: not_vulnerable + + - id: PT-OAPI8-01 + honeypot_path: "/api/v2/me/" + method: GET + expected_severity: HIGH + expected_evidence_keys: ["acao=", "acac=true"] + hardened_status: not_vulnerable + notes: depends on portal/middleware.py CORS being permissive + + - id: PT-OAPI8-02 + honeypot_path: "/api/v2/me/" + method: GET + expected_severity: LOW + expected_evidence_keys: ["missing_headers="] + hardened_status: not_vulnerable + + - id: PT-OAPI8-03 + honeypot_path: "/api/_debug/routes/" + method: GET + expected_severity: MEDIUM + expected_evidence_keys: ["debug_markers_present=true"] + hardened_status: not_vulnerable + + - id: PT-OAPI8-04 + honeypot_path: "/api/records/force-error/" + method: POST + expected_severity: MEDIUM + expected_evidence_keys: ["stack_trace_or_framework_marker=present"] + hardened_status: not_vulnerable + + - id: PT-OAPI8-05 + honeypot_path: "/api/records/1/" + method: OPTIONS + expected_severity: LOW + expected_evidence_keys: ["unexpected_methods="] + hardened_status: not_vulnerable + + - id: PT-OAPI9-01 + honeypot_path: "/openapi.json" + method: GET + expected_severity: MEDIUM + expected_evidence_keys: ["spec_paths_count=", "private_paths_count="] + hardened_status: not_vulnerable + + - id: PT-OAPI9-02 + honeypot_path: "/api/v1/records/1/" + method: GET + expected_severity: MEDIUM + expected_evidence_keys: ["sibling=/api/v1"] + hardened_status: not_vulnerable + + - id: PT-OAPI9-03 + honeypot_path: "/api/v0/legacy/" + method: GET + expected_severity: MEDIUM + expected_evidence_keys: ["endpoint=", "status=200"] + hardened_status: not_vulnerable diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 262a7e32..35411fb0 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -5,6 +5,10 @@ from unittest.mock import MagicMock, patch from extensions.business.cybersec.red_mesh.constants import JOB_ARCHIVE_VERSION, MAX_CONTINUOUS_PASSES +from extensions.business.cybersec.red_mesh.graybox.scenario_runtime import ( + build_graybox_worker_assignments, + runtime_scenario_ids, +) from extensions.business.cybersec.red_mesh.models import CStoreJobRunning from .conftest import DummyOwner, MANUAL_RUN, PentestLocalWorker, color_print, mock_plugin_modules @@ -291,8 +295,8 @@ def test_launch_fails_if_r1fs_unavailable(self): job_specs = self._extract_job_specs(plugin, "test-job-5") self.assertIsNone(job_specs) - def test_launch_webapp_scan_uses_mirrored_worker_assignments(self): - """Webapp launches assign the same resolved target port to every selected peer.""" + def test_launch_webapp_scan_default_slices_api_scenarios(self): + """B5: webapp launches default to SLICE so the per-scan budget stays per-scan.""" plugin = self._build_mock_plugin(job_id="test-job-webapp") plugin.chainstore_peers = ["node-1", "node-2"] plugin.cfg_chainstore_peers = ["node-1", "node-2"] @@ -302,10 +306,110 @@ def test_launch_webapp_scan_uses_mirrored_worker_assignments(self): job_specs = self._extract_job_specs(plugin, "test-job-webapp") workers = job_specs["workers"] - self.assertEqual(workers["node-1"]["start_port"], 443) - self.assertEqual(workers["node-1"]["end_port"], 443) - self.assertEqual(workers["node-2"]["start_port"], 443) - self.assertEqual(workers["node-2"]["end_port"], 443) + self.assertEqual(workers["node-1"]["graybox_assignment_strategy"], "SLICE") + self.assertEqual(workers["node-2"]["graybox_assignment_strategy"], "SLICE") + self.assertEqual({workers[node]["budget_scope"] for node in workers}, {"per_scan"}) + self.assertTrue(workers["node-1"]["assignment_hash"]) + + def test_launch_webapp_scan_explicit_mirror_divides_budget_without_opt_in(self): + """Multi-worker MIRROR without per-worker opt-in must divide the budget.""" + plugin = self._build_mock_plugin(job_id="test-job-mirror-div") + plugin.chainstore_peers = ["node-1", "node-2"] + plugin.cfg_chainstore_peers = ["node-1", "node-2"] + + result = self._launch_webapp( + plugin, + selected_peers=["node-1", "node-2"], + graybox_assignment_strategy="MIRROR", + request_budget=40, + ) + self.assertNotIn("error", result) + + workers = self._extract_job_specs(plugin, "test-job-mirror-div")["workers"] + self.assertEqual({workers[n]["budget_scope"] for n in workers}, {"per_scan"}) + self.assertEqual( + sum(workers[n]["assigned_request_budget"] for n in workers), + 40, + ) + + def test_launch_webapp_scan_explicit_mirror_per_worker_with_opt_in(self): + """MIRROR + multi-worker + allow_mirror_per_worker_budget=True keeps per-worker budget.""" + plugin = self._build_mock_plugin(job_id="test-job-mirror-pw") + plugin.chainstore_peers = ["node-1", "node-2"] + plugin.cfg_chainstore_peers = ["node-1", "node-2"] + + result = self._launch_webapp( + plugin, + selected_peers=["node-1", "node-2"], + graybox_assignment_strategy="MIRROR", + request_budget=40, + allow_mirror_per_worker_budget=True, + ) + self.assertNotIn("error", result) + + workers = self._extract_job_specs(plugin, "test-job-mirror-pw")["workers"] + self.assertEqual({workers[n]["budget_scope"] for n in workers}, {"per_worker"}) + self.assertEqual(workers["node-1"]["assigned_request_budget"], 40) + self.assertEqual(workers["node-2"]["assigned_request_budget"], 40) + + def test_launch_webapp_scan_emits_top_level_assignment_summary(self): + plugin = self._build_mock_plugin(job_id="test-job-summary") + plugin.chainstore_peers = ["node-1", "node-2"] + plugin.cfg_chainstore_peers = ["node-1", "node-2"] + + result = self._launch_webapp(plugin, selected_peers=["node-1", "node-2"]) + self.assertNotIn("error", result) + + summary = self._extract_job_specs(plugin, "test-job-summary").get("graybox_assignment_summary") + self.assertIsNotNone(summary) + self.assertEqual(summary["graybox_assignment_strategy"], "SLICE") + self.assertEqual(summary["budget_scope"], "per_scan") + self.assertGreater(summary["total_assigned_scenarios"], 0) + self.assertEqual(len(summary["worker_assignment_summary"]), 2) + + def test_launch_webapp_scan_can_slice_api_scenarios_between_workers(self): + plugin = self._build_mock_plugin(job_id="test-job-webapp-slice") + plugin.chainstore_peers = ["node-1", "node-2", "node-3"] + plugin.cfg_chainstore_peers = ["node-1", "node-2", "node-3"] + + result = self._launch_webapp( + plugin, + selected_peers=["node-1", "node-2", "node-3"], + graybox_assignment_strategy="SLICE", + request_budget=30, + ) + self.assertNotIn("error", result) + + job_specs = self._extract_job_specs(plugin, "test-job-webapp-slice") + workers = job_specs["workers"] + assigned_sets = [ + set(workers[node]["assigned_scenario_ids"]) + for node in ("node-1", "node-2", "node-3") + ] + self.assertEqual(set().union(*assigned_sets), set(runtime_scenario_ids())) + self.assertFalse(assigned_sets[0] & assigned_sets[1]) + self.assertFalse(assigned_sets[0] & assigned_sets[2]) + self.assertFalse(assigned_sets[1] & assigned_sets[2]) + self.assertEqual( + sum(workers[node]["assigned_request_budget"] for node in workers), + 30, + ) + self.assertEqual({workers[node]["budget_scope"] for node in workers}, {"per_scan"}) + + def test_launch_webapp_scan_rejects_mirror_stateful_multi_worker(self): + plugin = self._build_mock_plugin(job_id="test-job-webapp-stateful") + plugin.chainstore_peers = ["node-1", "node-2"] + plugin.cfg_chainstore_peers = ["node-1", "node-2"] + + result = self._launch_webapp( + plugin, + selected_peers=["node-1", "node-2"], + allow_stateful_probes=True, + graybox_assignment_strategy="MIRROR", + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("MIRROR with stateful", result["message"]) def test_launch_webapp_scan_neutralizes_network_only_fields(self): """Webapp config does not persist bogus network defaults like exceptions='64297'.""" @@ -359,21 +463,286 @@ def test_launch_webapp_scan_persists_secret_ref_not_inline_passwords(self): job_specs = self._extract_job_specs(plugin, "test-job-websecret") self.assertEqual(job_specs["job_config_cid"], "QmConfigCID") - def test_launch_webapp_scan_rejects_secret_persistence_without_store_key(self): - """Webapp launch fails closed when no strong secret-store key is configured.""" - plugin = self._build_mock_plugin(job_id="test-job-websecret-nokey") - plugin.cfg_redmesh_secret_store_key = "" - plugin.cfg_comms_host_key = "" - plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 86400, "RETRIES": 2} + def test_launch_webapp_scan_persists_bearer_token_only_in_secret_payload(self): + """API-native bearer auth uses the same R1FS secret lane as form passwords.""" + plugin = self._build_mock_plugin(job_id="test-job-bearer-secret") + plugin.r1fs.add_json.side_effect = ["QmSecretCID", "QmConfigCID"] result = self._launch_webapp( plugin, - official_username="admin", - official_password="secret", + official_username="", + official_password="", + bearer_token="BEARER-TOKEN-MUST-NOT-PERSIST", + target_config={ + "api_security": { + "auth": { + "auth_type": "bearer", + "authenticated_probe_path": "/api/me/", + }, + }, + }, + ) + + self.assertNotIn("error", result) + secret_doc = plugin.r1fs.add_json.call_args_list[0][0][0] + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + + self.assertEqual( + secret_doc["payload"]["bearer_token"], + "BEARER-TOKEN-MUST-NOT-PERSIST", + ) + self.assertEqual(config_dict["secret_ref"], "QmSecretCID") + self.assertTrue(config_dict["has_bearer_token"]) + self.assertEqual(config_dict["bearer_token"], "") + self.assertNotIn( + "BEARER-TOKEN-MUST-NOT-PERSIST", + json.dumps(config_dict), + ) + + def test_launch_webapp_scan_rejects_bearer_without_validation_path(self): + """Bearer/API-key auth must be validated unless explicitly unverified.""" + plugin = self._build_mock_plugin(job_id="test-job-bearer-no-probe") + + result = self._launch_webapp( + plugin, + official_username="", + official_password="", + bearer_token="BEARER-TOKEN", + target_config={ + "api_security": { + "auth": {"auth_type": "bearer"}, + }, + }, + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("authenticated_probe_path", result["message"]) + self.assertFalse(plugin.r1fs.add_json.called) + + def test_launch_webapp_scan_allows_explicit_unverified_bearer(self): + """Explicit opt-out persists as non-secret policy metadata.""" + plugin = self._build_mock_plugin(job_id="test-job-bearer-unverified") + plugin.r1fs.add_json.side_effect = ["QmSecretCID", "QmConfigCID"] + + result = self._launch_webapp( + plugin, + official_username="", + official_password="", + bearer_token="BEARER-TOKEN", + target_config={ + "api_security": { + "auth": { + "auth_type": "bearer", + "allow_unverified_auth": True, + }, + }, + }, + ) + + self.assertNotIn("error", result) + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + auth_config = config_dict["target_config"]["api_security"]["auth"] + self.assertTrue(auth_config["allow_unverified_auth"]) + self.assertEqual(auth_config["authenticated_probe_path"], "") + + def test_launch_webapp_scan_rejects_mutating_auth_validation_method(self): + plugin = self._build_mock_plugin(job_id="test-job-bearer-post-probe") + + result = self._launch_webapp( + plugin, + official_username="", + official_password="", + bearer_token="BEARER-TOKEN", + target_config={ + "api_security": { + "auth": { + "auth_type": "bearer", + "authenticated_probe_path": "/api/me/", + "authenticated_probe_method": "POST", + }, + }, + }, + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("authenticated_probe_method", result["message"]) + + def test_launch_webapp_scan_allows_explicit_non_readonly_validation(self): + plugin = self._build_mock_plugin(job_id="test-job-bearer-post-override") + plugin.r1fs.add_json.side_effect = ["QmSecretCID", "QmConfigCID"] + + result = self._launch_webapp( + plugin, + official_username="", + official_password="", + bearer_token="BEARER-TOKEN", + target_config={ + "api_security": { + "auth": { + "auth_type": "bearer", + "authenticated_probe_path": "/api/me/", + "authenticated_probe_method": "POST", + "allow_non_readonly_auth_validation_method": True, + }, + }, + }, + ) + + self.assertNotIn("error", result) + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + auth_config = config_dict["target_config"]["api_security"]["auth"] + self.assertEqual(auth_config["authenticated_probe_method"], "POST") + self.assertTrue(auth_config["allow_non_readonly_auth_validation_method"]) + + def test_launch_webapp_scan_rejects_nested_target_config_secret(self): + """Nested request bodies cannot carry raw secrets into persisted JobConfig.""" + plugin = self._build_mock_plugin(job_id="test-job-target-secret") + + result = self._launch_webapp( + plugin, + target_config={ + "api_security": { + "token_endpoints": { + "token_request_body": { + "client_id": "redmesh", + "client_secret": "plain-secret", + }, + }, + }, + }, + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("target_config", result["message"]) + self.assertEqual(plugin.r1fs.add_json.call_count, 0) + + def test_launch_webapp_scan_rejects_unknown_target_config_key(self): + """Unknown nested target_config keys fail closed instead of disappearing.""" + plugin = self._build_mock_plugin(job_id="test-job-target-unknown") + + result = self._launch_webapp( + plugin, + target_config={ + "api_security": { + "object_endpoints": [ + {"path": "/api/records/{id}/", "typo": True}, + ], + }, + }, + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("unknown field", result["message"]) + self.assertEqual(plugin.r1fs.add_json.call_count, 0) + + def test_launch_webapp_scan_persists_target_config_secret_ref_value_only_in_secret_payload(self): + """Typed target_config secret refs resolve through the R1FS secret payload.""" + plugin = self._build_mock_plugin(job_id="test-job-target-secret-ref") + plugin.r1fs.add_json.side_effect = ["QmSecretCID", "QmConfigCID"] + + result = self._launch_webapp( + plugin, + target_config={ + "api_security": { + "token_endpoints": { + "token_request_body": { + "client_id": "redmesh", + "client_secret": {"secret_ref": "oauth_client_secret"}, + }, + }, + }, + }, + target_config_secrets={"oauth_client_secret": "OAUTH-CLIENT-SECRET"}, + ) + + self.assertNotIn("error", result) + secret_doc = plugin.r1fs.add_json.call_args_list[0][0][0] + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual( + secret_doc["payload"]["target_config_secrets"], + {"oauth_client_secret": "OAUTH-CLIENT-SECRET"}, + ) + self.assertNotIn("OAUTH-CLIENT-SECRET", json.dumps(config_dict)) + self.assertEqual( + config_dict["target_config"]["api_security"]["token_endpoints"][ + "token_request_body" + ]["client_secret"], + {"secret_ref": "oauth_client_secret"}, + ) + + def test_launch_webapp_scan_rejects_missing_target_config_secret_ref_value(self): + plugin = self._build_mock_plugin(job_id="test-job-target-secret-ref-missing") + + result = self._launch_webapp( + plugin, + target_config={ + "api_security": { + "token_endpoints": { + "token_request_body": { + "client_secret": {"secret_ref": "oauth_client_secret"}, + }, + }, + }, + }, + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("missing", result["message"]) + self.assertEqual(plugin.r1fs.add_json.call_count, 0) + + def test_launch_webapp_scan_rejects_unknown_target_config_secret_value(self): + plugin = self._build_mock_plugin(job_id="test-job-target-secret-ref-extra") + + result = self._launch_webapp( + plugin, + target_config={"api_security": {"token_endpoints": {}}}, + target_config_secrets={"unused": "secret"}, ) - self.assertEqual(result["error"], "Failed to store job config in R1FS") - self.assertEqual(len(plugin.r1fs.add_json.call_args_list), 0) + self.assertEqual(result["error"], "validation_error") + self.assertIn("unknown secret_ref", result["message"]) + self.assertEqual(plugin.r1fs.add_json.call_count, 0) + + def test_launch_webapp_scan_rejects_secret_ref_outside_approved_body(self): + plugin = self._build_mock_plugin(job_id="test-job-target-secret-ref-bad-place") + + result = self._launch_webapp( + plugin, + target_config={ + "api_security": { + "auth": { + "api_key_header_name": {"secret_ref": "header_name"}, + }, + }, + }, + target_config_secrets={"header_name": "X-Secret"}, + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("outside an approved request body", result["message"]) + self.assertEqual(plugin.r1fs.add_json.call_count, 0) + + def test_launch_webapp_scan_records_default_plugin_key_metadata(self): + """With no dedicated key, the built-in default is used automatically and + metadata reflects the well-known key.""" + plugin = self._build_mock_plugin(job_id="test-job-websecret-default-key") + plugin.cfg_redmesh_secret_store_key = "" + plugin.r1fs.add_json.side_effect = ["QmSecretCID", "QmConfigCID"] + + with patch.dict("os.environ", {}, clear=True): + result = self._launch_webapp( + plugin, + official_username="admin", + official_password="secret", + ) + + self.assertNotIn("error", result) + secret_doc = plugin.r1fs.add_json.call_args_list[0][0][0] + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(secret_doc["unsafe_key_fallback"]) + self.assertEqual(secret_doc["key_id"], "redmesh:default_plugin_key") + self.assertTrue(config_dict["secret_store_unsafe_fallback"]) + self.assertEqual(config_dict["secret_store_key_id"], "redmesh:default_plugin_key") def test_launch_webapp_scan_rejects_missing_target_url(self): """Webapp endpoint returns structured validation error for missing URL.""" @@ -414,6 +783,48 @@ def test_launch_webapp_scan_enforces_target_allowlist(self): self.assertEqual(result["error"], "validation_error") self.assertIn("allowlist", result["message"]) + def test_launch_webapp_scan_rejects_out_of_scope_api_paths(self): + """Path-scoped authorization applies to configured API probe paths.""" + plugin = self._build_mock_plugin(job_id="test-job-path-scope") + + result = self._launch_webapp( + plugin, + target_allowlist=["example.com", "/api/public/"], + target_config={ + "login_path": "/api/public/login/", + "logout_path": "/api/public/logout/", + "api_security": { + "function_endpoints": [ + {"path": "/admin/export-users/"}, + ], + }, + }, + ) + + self.assertEqual(result["error"], "validation_error") + self.assertIn("outside authorized scope", result["message"]) + self.assertEqual(plugin.r1fs.add_json.call_count, 0) + + def test_launch_webapp_scan_accepts_in_scope_templated_api_paths(self): + """Templated API paths are normalized and allowed inside the scope prefix.""" + plugin = self._build_mock_plugin(job_id="test-job-path-scope-ok") + + result = self._launch_webapp( + plugin, + target_allowlist=["example.com", "/api/public/"], + target_config={ + "login_path": "/api/public/login/", + "logout_path": "/api/public/logout/", + "api_security": { + "object_endpoints": [ + {"path": "/api/public/users/{id}/"}, + ], + }, + }, + ) + + self.assertNotIn("error", result) + def test_launch_webapp_scan_persists_authorization_context(self): """Authorization metadata is stored in immutable job config and audit context.""" plugin = self._build_mock_plugin(job_id="test-job-authctx") @@ -426,7 +837,11 @@ def test_launch_webapp_scan_persists_authorization_context(self): authorization_ref="TICKET-42", engagement_metadata={"ticket": "TICKET-42", "owner": "alice"}, target_allowlist=["example.com", "/api/"], - target_config={"discovery": {"scope_prefix": "/api/"}}, + target_config={ + "login_path": "/api/login/", + "logout_path": "/api/logout/", + "discovery": {"scope_prefix": "/api/"}, + }, ) config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] @@ -439,6 +854,66 @@ def test_launch_webapp_scan_persists_authorization_context(self): self.assertEqual(audit_payload["scope_id"], "scope-123") self.assertEqual(audit_payload["authorization_ref"], "TICKET-42") + def test_launch_webapp_scan_preserves_api_security_payload(self): + """OWASP API Top 10 target_config.api_security passes through to JobConfig.""" + plugin = self._build_mock_plugin(job_id="test-job-api-security") + + api_security_payload = { + "object_endpoints": [ + {"path": "/api/records/{id}/", "test_ids": [1, 2], + "owner_field": "owner", "tenant_field": "tenant_id"}, + ], + "function_endpoints": [ + {"path": "/api/admin/users/{uid}/promote/", + "method": "POST", "privilege": "admin", + "revert_path": "/api/admin/users/{uid}/demote/"}, + ], + "token_endpoints": { + "token_path": "/api/token/", + "protected_path": "/api/me/", + "logout_path": "/api/auth/logout/", + }, + "inventory_paths": { + "current_version": "/api/v2/", + "canonical_probe_path": "/api/v2/records/1/", + "deprecated_paths": ["/api/v1/legacy/"], + }, + } + + self._launch_webapp( + plugin, + target_config={ + "discovery": {"scope_prefix": "/api/"}, + "api_security": api_security_payload, + }, + ) + + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + api_security = config_dict["target_config"]["api_security"] + # Object endpoints preserved + self.assertEqual(len(api_security["object_endpoints"]), 1) + self.assertEqual( + api_security["object_endpoints"][0]["tenant_field"], "tenant_id" + ) + # Function endpoints + revert path preserved + self.assertEqual( + api_security["function_endpoints"][0]["revert_path"], + "/api/admin/users/{uid}/demote/", + ) + # Token endpoints preserved + self.assertEqual( + api_security["token_endpoints"]["logout_path"], "/api/auth/logout/" + ) + # Inventory paths preserved + self.assertEqual( + api_security["inventory_paths"]["canonical_probe_path"], + "/api/v2/records/1/", + ) + self.assertEqual( + api_security["inventory_paths"]["deprecated_paths"], + ["/api/v1/legacy/"], + ) + def test_launch_webapp_scan_applies_safety_policy_caps(self): """Graybox launch policy caps weak-auth and discovery budgets and records warnings.""" plugin = self._build_mock_plugin(job_id="test-job-policy") @@ -464,6 +939,81 @@ def test_launch_webapp_scan_applies_safety_policy_caps(self): self.assertTrue(any("capped" in warning for warning in warnings)) self.assertTrue(any("TLS verification is disabled" in warning for warning in warnings)) + def test_launch_webapp_scan_rejects_invalid_numeric_safety_values(self): + plugin = self._build_mock_plugin(job_id="test-job-bad-request-budget") + result = self._launch_webapp(plugin, request_budget="abc") + self.assertEqual(result["error"], "validation_error") + self.assertIn("request_budget", result["message"]) + + plugin = self._build_mock_plugin(job_id="test-job-bad-weak-attempts") + result = self._launch_webapp(plugin, max_weak_attempts=0) + self.assertEqual(result["error"], "validation_error") + self.assertIn("max_weak_attempts", result["message"]) + + def test_launch_webapp_scan_rejects_invalid_target_config_numeric_values(self): + plugin = self._build_mock_plugin(job_id="test-job-bad-max-requests") + result = self._launch_webapp( + plugin, + target_config={"api_security": {"max_total_requests": "abc"}}, + ) + self.assertEqual(result["error"], "validation_error") + self.assertIn("api_security.max_total_requests", result["message"]) + + plugin = self._build_mock_plugin(job_id="test-job-bad-discovery") + result = self._launch_webapp( + plugin, + target_config={"discovery": {"scope_prefix": "/api/", "max_pages": -1}}, + ) + self.assertEqual(result["error"], "validation_error") + self.assertIn("discovery.max_pages", result["message"]) + + plugin = self._build_mock_plugin(job_id="test-job-bad-payload-size") + result = self._launch_webapp( + plugin, + target_config={ + "api_security": { + "resource_endpoints": [ + { + "path": "/api/records/", + "allow_oversized_payload_probe": True, + "oversized_payload_bytes": 262_145, + }, + ], + }, + }, + ) + self.assertEqual(result["error"], "validation_error") + self.assertIn("oversized_payload_bytes", result["message"]) + + def test_launch_webapp_scan_normalizes_numeric_strings(self): + plugin = self._build_mock_plugin(job_id="test-job-numeric-strings") + self._launch_webapp( + plugin, + request_budget="42", + max_weak_attempts="5", + target_config={ + "discovery": {"scope_prefix": "/api/", "max_pages": "12", "max_depth": "2"}, + "api_security": { + "object_endpoints": [ + {"path": "/api/records/{id}/", "test_ids": ["1", "2"]}, + ], + }, + }, + ) + + config_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertEqual(config_dict["max_weak_attempts"], 5) + self.assertEqual(config_dict["target_config"]["discovery"]["max_pages"], 12) + self.assertEqual(config_dict["target_config"]["discovery"]["max_depth"], 2) + self.assertEqual( + config_dict["target_config"]["api_security"]["max_total_requests"], + 42, + ) + self.assertEqual( + config_dict["target_config"]["api_security"]["object_endpoints"][0]["test_ids"], + [1, 2], + ) + def test_launch_test_rejects_invalid_scan_type(self): """Compatibility endpoint rejects unknown scan types with a structured error.""" plugin = self._build_mock_plugin(job_id="test-job-badtype") @@ -487,6 +1037,10 @@ def test_launch_test_routes_to_scan_type_specific_endpoint(self): target_url="https://example.com/app", official_username="admin", official_password="secret", + bearer_token="TOKEN-123", + api_key="KEY-123", + bearer_refresh_token="REFRESH-123", + request_budget=42, authorized=True, scan_type="webapp", ) @@ -495,6 +1049,11 @@ def test_launch_test_routes_to_scan_type_specific_endpoint(self): self.assertEqual(webapp["route"], "webapp") plugin.launch_network_scan.assert_called_once() plugin.launch_webapp_scan.assert_called_once() + webapp_kwargs = plugin.launch_webapp_scan.call_args.kwargs + self.assertEqual(webapp_kwargs["bearer_token"], "TOKEN-123") + self.assertEqual(webapp_kwargs["api_key"], "KEY-123") + self.assertEqual(webapp_kwargs["bearer_refresh_token"], "REFRESH-123") + self.assertEqual(webapp_kwargs["request_budget"], 42) def test_launch_test_persists_typed_ptes_context(self): """Compatibility launch_test preserves typed engagement/RoE/auth fields.""" @@ -2353,6 +2912,8 @@ def test_get_job_config_resolves_secret_ref_for_runtime(self): }, { "kind": "redmesh_graybox_credentials", + "job_id": "test-job", + "storage_mode": "encrypted_r1fs_json_v1", "payload": { "official_username": "admin", "official_password": "secret", @@ -2363,7 +2924,10 @@ def test_get_job_config_resolves_secret_ref_for_runtime(self): }, ] - config = Plugin._get_job_config(plugin, {"job_config_cid": "QmConfigCID"}, resolve_secrets=True) + config = Plugin._get_job_config( + plugin, {"job_id": "test-job", "job_config_cid": "QmConfigCID"}, + resolve_secrets=True, + ) self.assertEqual(config["official_username"], "admin") self.assertEqual(config["official_password"], "secret") @@ -2375,13 +2939,11 @@ def test_get_job_config_resolves_secret_ref_for_runtime(self): unittest.mock.call("QmSecretCID", secret="unit-test-redmesh-secret-key"), ) - def test_get_job_config_resolves_legacy_plaintext_secret_ref_without_key(self): - """Legacy plaintext secret refs remain readable as a compatibility fallback.""" + def test_get_job_config_fails_closed_for_malformed_secret_payload(self): + """Secret refs decrypt with the default key, but malformed payloads (missing storage_mode) are rejected.""" Plugin = self._get_plugin_class() plugin = self._build_plugin({}) plugin.cfg_redmesh_secret_store_key = "" - plugin.cfg_comms_host_key = "" - plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 86400, "RETRIES": 2} plugin.r1fs.get_json.side_effect = [ { "scan_type": "webapp", @@ -2397,14 +2959,156 @@ def test_get_job_config_resolves_legacy_plaintext_secret_ref_without_key(self): }, ] - config = Plugin._get_job_config(plugin, {"job_config_cid": "QmConfigCID"}, resolve_secrets=True) + with self.assertRaises(ValueError): + Plugin._get_job_config( + plugin, {"job_id": "test-job", "job_config_cid": "QmConfigCID"}, + resolve_secrets=True, + ) + self.assertEqual(len(plugin.r1fs.get_json.call_args_list), 2) - self.assertEqual(config["official_password"], "secret") - self.assertEqual( - plugin.r1fs.get_json.call_args_list[1], - unittest.mock.call("QmSecretCID"), + def test_mark_worker_terminal_error_sets_common_fields(self): + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + job_specs = { + "job_id": "job-terminal", + "workers": {"worker-a": {"start_port": 443, "end_port": 443}}, + } + + with patch.object(Plugin, "_write_job_record", return_value=job_specs) as write: + Plugin._mark_worker_terminal_error( + plugin, + job_specs, + "worker-a", + "secret_resolution_failed", + "Failed to resolve graybox secret_ref", + context="test_terminal", + ) + + worker = job_specs["workers"]["worker-a"] + self.assertTrue(worker["finished"]) + self.assertEqual(worker["terminal_reason"], "secret_resolution_failed") + self.assertIn("secret_ref", worker["error"]) + write.assert_called_once() + + def test_mark_worker_terminal_error_merges_against_current_record(self): + """B8: concurrent terminal writes must merge by worker key, not overwrite.""" + Plugin = self._get_plugin_class() + # Current record in CStore has worker-A already terminal (written by + # worker A's concurrent failure). + current_record = { + "job_id": "job-concurrent", + "job_status": "RUNNING", + "job_pass": 1, + "run_mode": "SINGLEPASS", + "launcher": "launcher-node", + "target": "example.com", + "scan_type": "webapp", + "target_url": "https://example.com/app", + "start_port": 443, + "end_port": 443, + "date_created": 1000000.0, + "job_config_cid": "QmConfig", + "workers": { + "worker-A": { + "start_port": 443, "end_port": 443, + "finished": True, + "terminal_reason": "assignment_validation_failed", + "error": "A error", + }, + "worker-B": {"start_port": 443, "end_port": 443, "finished": False}, + }, + "timeline": [], + "pass_reports": [], + "job_revision": 7, + } + plugin = self._build_plugin({"job-concurrent": current_record}) + + # Worker-B's stale local snapshot doesn't know about A's terminal flag. + stale_snapshot = { + "job_id": "job-concurrent", + "workers": { + "worker-A": {"start_port": 443, "end_port": 443, "finished": False}, + "worker-B": {"start_port": 443, "end_port": 443, "finished": False}, + }, + } + + captured = {} + + def _capture(self_plugin, job_id, job_specs, expected_revision=None, context=""): + captured["job_id"] = job_id + captured["job_specs"] = dict(job_specs) + captured["context"] = context + return job_specs + + with patch.object(Plugin, "_write_job_record", side_effect=_capture): + Plugin._mark_worker_terminal_error( + plugin, + stale_snapshot, + "worker-B", + "launch_failed", + "B error", + context="b_terminal", + ) + + persisted_workers = captured["job_specs"]["workers"] + # A's pre-existing terminal data survived the B write. + self.assertTrue(persisted_workers["worker-A"]["finished"]) + self.assertEqual(persisted_workers["worker-A"]["terminal_reason"], "assignment_validation_failed") + self.assertEqual(persisted_workers["worker-A"]["error"], "A error") + # B's terminal patch is applied. + self.assertTrue(persisted_workers["worker-B"]["finished"]) + self.assertEqual(persisted_workers["worker-B"]["terminal_reason"], "launch_failed") + + def test_maybe_launch_jobs_secret_resolution_failure_marks_terminal(self): + Plugin = self._get_plugin_class() + assignments, error = build_graybox_worker_assignments(["launcher-node"]) + self.assertIsNone(error) + worker_entry = { + "start_port": 443, + "end_port": 443, + "finished": False, + "result": None, + **assignments["launcher-node"], + } + job_specs = { + "job_id": "job-secret-fail", + "job_status": "RUNNING", + "job_pass": 1, + "target": "example.com", + "scan_type": "webapp", + "target_url": "https://example.com/app", + "launcher": "launcher-node", + "launcher_alias": "launcher", + "workers": {"launcher-node": worker_entry}, + "run_mode": "SINGLEPASS", + "job_config_cid": "QmConfigCID", + } + plugin = self._build_plugin({"job-secret-fail": job_specs}) + plugin._PentesterApi01Plugin__last_checked_jobs = 0 + plugin.cfg_check_jobs_each = 0 + plugin.time.return_value = 100 + plugin.scan_jobs = {} + plugin.completed_jobs_reports = {} + plugin.lst_completed_jobs = [] + plugin._foreign_jobs_logged = set() + plugin._normalize_job_record = lambda key, spec, migrate=False: (key, spec) + plugin._get_worker_entry = lambda job_id, spec: Plugin._get_worker_entry(plugin, job_id, spec) + plugin._get_active_execution_identity = lambda job_id: None + plugin._build_execution_identity = lambda job_id, pass_nr, worker_addr, revision: ( + job_id, pass_nr, worker_addr, revision, + ) + plugin._get_job_config = MagicMock( + side_effect=ValueError("Failed to resolve graybox secret_ref") ) + with patch.object(Plugin, "_write_job_record", return_value=job_specs) as write: + Plugin._maybe_launch_jobs(plugin) + + self.assertTrue(worker_entry["finished"]) + self.assertEqual(worker_entry["terminal_reason"], "secret_resolution_failed") + self.assertIn("secret_ref", worker_entry["error"]) + write.assert_called_once() + def test_get_job_data_running_last_5(self): """Running job with 8 passes returns last 5 refs only.""" Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_auth.py b/extensions/business/cybersec/red_mesh/tests/test_auth.py index 7ca8abcf..d98b09bd 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_auth.py +++ b/extensions/business/cybersec/red_mesh/tests/test_auth.py @@ -37,59 +37,69 @@ def _mock_response(status=200, text="", url="http://testapp.local:8000/dashboard class TestCsrfAutoDetect(unittest.TestCase): + # After Subphase 1.5 commit #3, CSRF auto-detection lives on FormAuth + # (the form-login strategy). These tests drive the strategy directly. + + def _form_auth(self, csrf_field=""): + from extensions.business.cybersec.red_mesh.graybox.auth_strategies import FormAuth + cfg = GrayboxTargetConfig(csrf_field=csrf_field) + return FormAuth("http://testapp.local:8000", cfg) + def test_csrf_autodetect_django(self): """Finds Django csrfmiddlewaretoken.""" - auth = _make_auth() + fa = self._form_auth() html = '' - field, token = auth._extract_csrf(html) + field, token = fa._extract_csrf(html) self.assertEqual(field, "csrfmiddlewaretoken") self.assertEqual(token, "abc123") def test_csrf_autodetect_flask(self): """Finds Flask/WTForms csrf_token.""" - auth = _make_auth() + fa = self._form_auth() html = '' - field, token = auth._extract_csrf(html) + field, token = fa._extract_csrf(html) self.assertEqual(field, "csrf_token") self.assertEqual(token, "flask-token-xyz") def test_csrf_autodetect_rails(self): """Finds Rails authenticity_token.""" - auth = _make_auth() + fa = self._form_auth() html = '' - field, token = auth._extract_csrf(html) + field, token = fa._extract_csrf(html) self.assertEqual(field, "authenticity_token") self.assertEqual(token, "rails-tok") def test_csrf_autodetect_fallback(self): """Fallback finds generic hidden input with 'csrf' in name.""" - auth = _make_auth() + fa = self._form_auth() html = '' - field, token = auth._extract_csrf(html) + field, token = fa._extract_csrf(html) self.assertEqual(field, "my_csrf_thing") self.assertEqual(token, "custom-tok") def test_csrf_configured_override(self): """Configured csrf_field overrides auto-detection.""" - cfg = GrayboxTargetConfig(csrf_field="custom_token") - auth = _make_auth(target_config=cfg) + fa = self._form_auth(csrf_field="custom_token") html = '' - field, token = auth._extract_csrf(html) + field, token = fa._extract_csrf(html) self.assertEqual(field, "custom_token") self.assertEqual(token, "override-val") def test_csrf_field_property(self): - """detected_csrf_field is exposed as a property.""" - auth = _make_auth() - self.assertIsNone(auth.detected_csrf_field) + """AuthManager.detected_csrf_field surfaces what FormAuth observed.""" + fa = self._form_auth() html = '' - auth._extract_csrf(html) - self.assertEqual(auth.detected_csrf_field, "csrf_token") + fa._extract_csrf(html) + # FormAuth tracks last_detected_csrf_field via authenticate() — for + # the standalone-helper case used in this test, the field is the + # second return value of _extract_csrf. The AuthManager-level + # detected_csrf_field property is asserted in TestAuthManagerLifecycle. + self.assertIsNone(fa.last_detected_csrf_field) # _extract_csrf alone does not set it def test_csrf_none_when_missing(self): """Returns (None, None) when no CSRF field found.""" - auth = _make_auth() - field, token = auth._extract_csrf("
") + fa = self._form_auth() + field, token = fa._extract_csrf("
") self.assertIsNone(field) self.assertIsNone(token) @@ -100,13 +110,417 @@ def test_extract_csrf_value_public_api(self): self.assertEqual(val, "pub-tok") +class TestBearerAuthStrategy(unittest.TestCase): + """OWASP API Top 10 (Subphase 1.5 commit #6) — Bearer-token strategy.""" + + def _bearer(self, **auth_kwargs): + from extensions.business.cybersec.red_mesh.graybox.auth_strategies import BearerAuth + from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, ApiSecurityConfig, AuthDescriptor, + ) + desc = AuthDescriptor(**{"auth_type": "bearer", **auth_kwargs}) + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig(auth=desc)) + return BearerAuth("http://api.example", cfg, verify_tls=True) + + def test_authenticate_stamps_default_header(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ba = self._bearer() + sess = ba.authenticate(Credentials(bearer_token="abc.def.ghi")) + self.assertIsNotNone(sess) + self.assertEqual(sess.headers["Authorization"], "Bearer abc.def.ghi") + + def test_authenticate_custom_header_and_scheme(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ba = self._bearer(bearer_token_header_name="X-Auth-Token", bearer_scheme="Token") + sess = ba.authenticate(Credentials(bearer_token="xyz")) + self.assertEqual(sess.headers["X-Auth-Token"], "Token xyz") + + def test_authenticate_empty_token_fails(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ba = self._bearer() + self.assertIsNone(ba.authenticate(Credentials())) + + def test_refresh_reauthenticates(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ba = self._bearer() + creds = Credentials(bearer_token="t1") + ba.authenticate(creds) + self.assertTrue(ba.refresh(creds)) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") + def test_preflight_skipped_when_no_probe_path(self, mock_requests): + """Empty `authenticated_probe_path` means no preflight HTTP traffic.""" + ba = self._bearer() + self.assertIsNone(ba.preflight()) + mock_requests.head.assert_not_called() + + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") + def test_preflight_401_is_allowed_before_token_is_sent(self, mock_requests): + import requests as real_requests + mock_requests.head.return_value = _mock_response(status=401) + mock_requests.RequestException = real_requests.RequestException + ba = self._bearer(authenticated_probe_path="/api/me") + err = ba.preflight() + self.assertIsNone(err) + + +class TestApiKeyAuthStrategy(unittest.TestCase): + """OWASP API Top 10 (Subphase 1.5 commit #7) — API-key strategy.""" + + def _api_key(self, **auth_kwargs): + from extensions.business.cybersec.red_mesh.graybox.auth_strategies import ApiKeyAuth + from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, ApiSecurityConfig, AuthDescriptor, + ) + desc = AuthDescriptor(**{"auth_type": "api_key", **auth_kwargs}) + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig(auth=desc)) + return ApiKeyAuth("http://api.example", cfg, verify_tls=True) + + def test_header_placement(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ak = self._api_key(api_key_location="header", api_key_header_name="X-Custom-Key") + sess = ak.authenticate(Credentials(api_key="SECRET")) + self.assertEqual(sess.headers["X-Custom-Key"], "SECRET") + # No params used in header mode + self.assertEqual(sess.params or {}, {}) + + def test_query_placement(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ak = self._api_key(api_key_location="query", api_key_query_param="apikey") + sess = ak.authenticate(Credentials(api_key="QSECRET")) + self.assertEqual(sess.params, {"apikey": "QSECRET"}) + # No Authorization header set in query mode + self.assertNotIn("Authorization", sess.headers) + + def test_unknown_location_fails(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ak = self._api_key(api_key_location="weird") + self.assertIsNone(ak.authenticate(Credentials(api_key="x"))) + + def test_empty_key_fails(self): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + ak = self._api_key() + self.assertIsNone(ak.authenticate(Credentials())) + + +class TestAuthManagerStrategyDispatch(unittest.TestCase): + """AuthManager.build_strategy routes by `auth_type` (Subphase 1.5 commits #5-#7).""" + + def _auth_with(self, auth_type): + from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + GrayboxTargetConfig, ApiSecurityConfig, AuthDescriptor, + ) + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig(auth=AuthDescriptor(auth_type=auth_type))) + return AuthManager("http://api.example", cfg) + + def test_dispatch_form(self): + from extensions.business.cybersec.red_mesh.graybox.auth_strategies import FormAuth + self.assertIsInstance(self._auth_with("form")._build_strategy(), FormAuth) + + def test_dispatch_bearer(self): + from extensions.business.cybersec.red_mesh.graybox.auth_strategies import BearerAuth + self.assertIsInstance(self._auth_with("bearer")._build_strategy(), BearerAuth) + + def test_dispatch_api_key(self): + from extensions.business.cybersec.red_mesh.graybox.auth_strategies import ApiKeyAuth + self.assertIsInstance(self._auth_with("api_key")._build_strategy(), ApiKeyAuth) + + def test_dispatch_unknown(self): + auth = self._auth_with("bogus") + with self.assertRaises(ValueError): + auth._build_strategy() + + +class TestAuthManagerNativeApiCredentials(unittest.TestCase): + """AuthManager preserves token/key credentials through strategy dispatch.""" + + def _auth_with_descriptor(self, **auth_kwargs): + from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiSecurityConfig, AuthDescriptor, + ) + desc = AuthDescriptor(**auth_kwargs) + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig(auth=desc)) + return AuthManager("http://api.example", cfg, verify_tls=False) + + def _mock_session(self, status=200): + session = MagicMock() + session.headers = {} + session.params = {} + session.get.return_value = _mock_response(status=status) + session.head.return_value = _mock_response(status=status) + return session + + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") + def test_authenticate_bearer_stamps_token_and_validates_after_auth(self, mock_requests): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + + session = self._mock_session(status=200) + mock_requests.Session.return_value = session + + auth = self._auth_with_descriptor( + auth_type="bearer", + authenticated_probe_path="/api/me", + ) + ok = auth.authenticate(Credentials(bearer_token="TOKEN-123")) + + self.assertTrue(ok) + self.assertIs(auth.official_session, session) + self.assertEqual(session.headers["Authorization"], "Bearer TOKEN-123") + session.get.assert_called_once_with( + "http://api.example/api/me", + timeout=10, + allow_redirects=False, + ) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") + def test_bearer_validation_method_falls_back_to_get_without_override(self, mock_requests): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + + session = self._mock_session(status=200) + mock_requests.Session.return_value = session + + auth = self._auth_with_descriptor( + auth_type="bearer", + authenticated_probe_path="/api/me", + authenticated_probe_method="POST", + ) + ok = auth.authenticate(Credentials(bearer_token="TOKEN-123")) + + self.assertTrue(ok) + session.get.assert_called_once_with( + "http://api.example/api/me", + timeout=10, + allow_redirects=False, + ) + session.post.assert_not_called() + + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") + def test_bearer_validation_method_allows_post_with_override(self, mock_requests): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + + session = self._mock_session(status=200) + session.post.return_value = _mock_response(status=200) + mock_requests.Session.return_value = session + + auth = self._auth_with_descriptor( + auth_type="bearer", + authenticated_probe_path="/api/me", + authenticated_probe_method="POST", + allow_non_readonly_auth_validation_method=True, + ) + ok = auth.authenticate(Credentials(bearer_token="TOKEN-123")) + + self.assertTrue(ok) + session.post.assert_called_once_with( + "http://api.example/api/me", + timeout=10, + allow_redirects=False, + ) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") + def test_authenticate_api_key_query_validates_with_session_params(self, mock_requests): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + + session = self._mock_session(status=200) + mock_requests.Session.return_value = session + + auth = self._auth_with_descriptor( + auth_type="api_key", + authenticated_probe_path="/api/me", + api_key_location="query", + api_key_query_param="apikey", + ) + ok = auth.authenticate(Credentials(api_key="KEY-123")) + + self.assertTrue(ok) + self.assertIs(auth.official_session, session) + self.assertEqual(session.params, {"apikey": "KEY-123"}) + session.get.assert_called_once() + + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") + def test_authenticate_bearer_rejects_unauthorized_probe_path(self, mock_requests): + from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials + + session = self._mock_session(status=401) + mock_requests.Session.return_value = session + + auth = self._auth_with_descriptor( + auth_type="bearer", + authenticated_probe_path="/api/me", + ) + ok = auth.authenticate(Credentials(bearer_token="BAD-TOKEN")) + + self.assertFalse(ok) + self.assertIsNone(auth.official_session) + session.close.assert_called_once() + self.assertIn("official_login_failed", auth._auth_errors) + + +class TestAuthenticatedSessionHardening(unittest.TestCase): + """B2 (PR406 remediation) — tighten bearer/API-key validation. + + Validation must: + * use allow_redirects=False so a 302->200 login page can't masquerade + as an authenticated 2xx; + * reject 3xx/401/403/>=400; + * cross-check with an anonymous request and require a marker / + identity assertion when both are 2xx. + """ + + def _build_auth(self, **auth_kwargs): + from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiSecurityConfig, AuthDescriptor, + ) + desc = AuthDescriptor(**{"auth_type": "bearer", "authenticated_probe_path": "/api/me", **auth_kwargs}) + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig(auth=desc)) + return AuthManager("http://api.example", cfg, verify_tls=False) + + def _session_with(self, status, body="", json_value=None, content_type="application/json"): + sess = MagicMock() + sess.headers = {} + sess.params = {} + resp = _mock_response(status=status, text=body, content_type=content_type) + if json_value is not None: + resp.json.return_value = json_value + sess.get.return_value = resp + sess.head.return_value = resp + sess.post.return_value = resp + return sess, resp + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_3xx_redirect_to_login_is_rejected(self, mock_auth_requests): + auth = self._build_auth() + sess, _ = self._session_with(status=302, body="", content_type="text/html") + sess.get.return_value.headers = {"location": "/login"} + valid, retryable = auth._validate_authenticated_session(sess) + self.assertFalse(valid) + self.assertFalse(retryable) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_401_is_rejected(self, mock_auth_requests): + auth = self._build_auth() + sess, _ = self._session_with(status=401) + valid, retryable = auth._validate_authenticated_session(sess) + self.assertFalse(valid) + self.assertFalse(retryable) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_authenticated_2xx_and_anonymous_401_is_accepted(self, mock_auth_requests): + """The clean delta case — anonymous request is rejected, no marker required.""" + auth = self._build_auth() + sess, _ = self._session_with(status=200, json_value={"user": "alice"}) + + anon_session = MagicMock() + anon_resp = _mock_response(status=401) + anon_session.request.return_value = anon_resp + mock_auth_requests.Session.return_value = anon_session + import requests as real_requests + mock_auth_requests.RequestException = real_requests.RequestException + + valid, retryable = auth._validate_authenticated_session(sess) + self.assertTrue(valid) + self.assertFalse(retryable) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_anonymous_also_2xx_without_marker_is_rejected(self, mock_auth_requests): + """Endpoint is public — bearer token tells us nothing, must fail.""" + auth = self._build_auth() + sess, _ = self._session_with(status=200, body="welcome") + + anon_session = MagicMock() + anon_resp = _mock_response(status=200, text="welcome") + anon_session.request.return_value = anon_resp + mock_auth_requests.Session.return_value = anon_session + import requests as real_requests + mock_auth_requests.RequestException = real_requests.RequestException + + valid, retryable = auth._validate_authenticated_session(sess) + self.assertFalse(valid) + self.assertFalse(retryable) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_marker_only_in_authenticated_response_is_accepted(self, mock_auth_requests): + auth = self._build_auth(authenticated_probe_success_marker='"principal":"alice"') + sess, _ = self._session_with( + status=200, body='{"principal":"alice"}' + ) + + anon_session = MagicMock() + anon_resp = _mock_response(status=200, text='{"public":true}') + anon_session.request.return_value = anon_resp + mock_auth_requests.Session.return_value = anon_session + import requests as real_requests + mock_auth_requests.RequestException = real_requests.RequestException + + valid, retryable = auth._validate_authenticated_session(sess) + self.assertTrue(valid) + self.assertFalse(retryable) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_identity_json_path_distinguishes_authenticated_from_anonymous(self, mock_auth_requests): + auth = self._build_auth(authenticated_probe_identity_json_path="user.id") + sess, _ = self._session_with( + status=200, body="", json_value={"user": {"id": "alice-123"}}, + ) + + anon_session = MagicMock() + anon_resp = _mock_response(status=200, text="") + anon_resp.json.return_value = {"user": {"id": ""}} + anon_session.request.return_value = anon_resp + mock_auth_requests.Session.return_value = anon_session + import requests as real_requests + mock_auth_requests.RequestException = real_requests.RequestException + + valid, retryable = auth._validate_authenticated_session(sess) + self.assertTrue(valid) + self.assertFalse(retryable) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_identity_json_path_missing_in_authenticated_is_rejected(self, mock_auth_requests): + auth = self._build_auth(authenticated_probe_identity_json_path="user.id") + sess, _ = self._session_with(status=200, body="", json_value={"user": {}}) + valid, retryable = auth._validate_authenticated_session(sess) + self.assertFalse(valid) + self.assertFalse(retryable) + + @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + def test_success_status_allowlist_filters_unexpected_2xx(self, mock_auth_requests): + auth = self._build_auth( + authenticated_probe_success_statuses=(204,), + ) + sess, _ = self._session_with(status=200) + valid, retryable = auth._validate_authenticated_session(sess) + self.assertFalse(valid) + self.assertFalse(retryable) + + def test_safe_identity_path_traversal_only_dotted_keys(self): + """The traversal helper must not evaluate arbitrary expressions.""" + payload = {"user": {"id": "alice"}} + self.assertEqual(AuthManager._traverse_identity_path(payload, "user.id"), "alice") + # Missing path returns None + self.assertIsNone(AuthManager._traverse_identity_path(payload, "user.missing")) + # Non-dict mid-path returns None + self.assertIsNone(AuthManager._traverse_identity_path(payload, "user.id.deeper")) + # Empty/invalid path returns None + self.assertIsNone(AuthManager._traverse_identity_path(payload, "")) + self.assertIsNone(AuthManager._traverse_identity_path(None, "any")) + + class TestLoginSuccessDetection(unittest.TestCase): def _check(self, auth, response, cookies=None): - """Helper to call _is_login_success with a mock session.""" + """Helper to call FormAuth._is_login_success with a mock session. + + After Subphase 1.5 commit #3, login-success heuristics live on FormAuth; + the AuthManager-level _is_login_success was removed. ``auth`` is kept + in the signature for backward compatibility with the per-test bodies + that still build an AuthManager for fixture reasons; the call site + delegates to the FormAuth static helper. + """ + from extensions.business.cybersec.red_mesh.graybox.auth_strategies import FormAuth session = MagicMock() session.cookies.get_dict.return_value = cookies or {} - return auth._is_login_success(response, session, "http://testapp.local:8000/auth/login/") + return FormAuth._is_login_success(response, session, "http://testapp.local:8000/auth/login/") def test_login_success_redirect_with_cookies(self): """Redirect away from login + cookies -> success.""" @@ -181,7 +595,7 @@ def test_login_failure_status(self): class TestAuthManagerLifecycle(unittest.TestCase): - @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") def test_try_credentials_public(self, mock_requests): """try_credentials returns session on success, None on failure.""" auth = _make_auth() @@ -201,7 +615,7 @@ def test_try_credentials_public(self, mock_requests): result = auth.try_credentials("admin", "pass") self.assertIsNotNone(result) - @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") def test_make_anonymous_session(self, mock_requests): """make_anonymous_session returns a fresh session.""" auth = _make_auth() @@ -264,7 +678,7 @@ def test_ensure_sessions_failed_refresh_clears_stale_sessions(self): self.assertEqual(auth.auth_state.refresh_count, 1) mock_auth.assert_called_once() - @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") @patch("extensions.business.cybersec.red_mesh.graybox.auth.time.sleep") def test_authenticate_retries_transient_transport_error(self, mock_sleep, mock_requests): """Transient transport failures retry once before giving up.""" @@ -282,7 +696,10 @@ def test_authenticate_retries_transient_transport_error(self, mock_sleep, mock_r history=[MagicMock()], ) second_session.cookies.get_dict.return_value = {"sessionid": "abc"} - mock_requests.Session.side_effect = [MagicMock(), first_session, second_session] + # After Subphase 1.5 commit #3, only FormAuth.make_session() consumes + # auth_strategies.requests.Session(); the anon session lives on the + # AuthManager side of the import boundary and uses auth.requests. + mock_requests.Session.side_effect = [first_session, second_session] mock_requests.RequestException = real_requests.RequestException result = auth.authenticate({"username": "admin", "password": "secret"}) @@ -292,7 +709,7 @@ def test_authenticate_retries_transient_transport_error(self, mock_sleep, mock_r mock_sleep.assert_called_once() self.assertEqual(auth._auth_errors, []) - @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") def test_preflight_unreachable(self, mock_requests): """preflight_check returns error for unreachable target.""" import requests as real_requests @@ -303,7 +720,7 @@ def test_preflight_unreachable(self, mock_requests): self.assertIsNotNone(err) self.assertIn("unreachable", err.lower()) - @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") def test_preflight_login_404(self, mock_requests): """preflight_check returns error if login page returns 404.""" mock_requests.head.return_value = _mock_response(status=200) @@ -314,7 +731,7 @@ def test_preflight_login_404(self, mock_requests): self.assertIsNotNone(err) self.assertIn("404", err) - @patch("extensions.business.cybersec.red_mesh.graybox.auth.requests") + @patch("extensions.business.cybersec.red_mesh.graybox.auth_strategies.requests") def test_preflight_ok(self, mock_requests): """preflight_check returns None when target and login page are reachable.""" mock_requests.head.return_value = _mock_response(status=200) diff --git a/extensions/business/cybersec/red_mesh/tests/test_budget.py b/extensions/business/cybersec/red_mesh/tests/test_budget.py new file mode 100644 index 00000000..34b947ab --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_budget.py @@ -0,0 +1,154 @@ +"""OWASP API Top 10 — Subphase 1.7 commit #6. + +`RequestBudget` exhaustion + shared-state semantics. Verifies the +budget enforces the per-scan cap correctly, including the concurrent +case (two probe instances sharing one budget never exceed the cap). +""" + +from __future__ import annotations + +import threading +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.budget import RequestBudget +from extensions.business.cybersec.red_mesh.graybox.probes.base import ProbeBase + + +class TestRequestBudgetSequential(unittest.TestCase): + + def test_consume_within_budget(self): + b = RequestBudget(remaining=5, total=5) + self.assertTrue(b.consume()) + self.assertTrue(b.consume(2)) + self.assertEqual(b.remaining, 2) + self.assertEqual(b.exhausted_count, 0) + + def test_consume_exhausts(self): + b = RequestBudget(remaining=2, total=2) + self.assertTrue(b.consume()) + self.assertTrue(b.consume()) + self.assertFalse(b.consume()) + self.assertEqual(b.exhausted_count, 1) + self.assertFalse(b.consume(5)) + self.assertEqual(b.exhausted_count, 2) + # Already-exhausted budget never goes negative. + self.assertEqual(b.remaining, 0) + + def test_consume_too_many_at_once(self): + """Single call asking for more than remaining is refused atomically.""" + b = RequestBudget(remaining=3, total=3) + self.assertFalse(b.consume(5)) + self.assertEqual(b.remaining, 3) + self.assertEqual(b.exhausted_count, 1) + + def test_consume_rejects_non_positive_amount(self): + b = RequestBudget(remaining=3, total=3) + with self.assertRaises(ValueError): + b.consume(0) + with self.assertRaises(ValueError): + b.consume(-5) + self.assertEqual(b.remaining, 3) + self.assertEqual(b.exhausted_count, 0) + + def test_snapshot_shape(self): + b = RequestBudget(remaining=10, total=10) + b.consume(3) + b.consume(20) # exhausted + snap = b.snapshot() + self.assertEqual(snap, {"remaining": 7, "total": 10, "exhausted_count": 1}) + + +class TestRequestBudgetConcurrent(unittest.TestCase): + + def test_concurrent_consumers_never_exceed_total(self): + """Two threads racing to consume must collectively decrement + exactly `total` requests — no double-spend, no underflow.""" + b = RequestBudget(remaining=100, total=100) + success_count = [0, 0] + + def worker(idx): + while b.consume(): + success_count[idx] += 1 + + t1 = threading.Thread(target=worker, args=(0,)) + t2 = threading.Thread(target=worker, args=(1,)) + t1.start(); t2.start() + t1.join(); t2.join() + + self.assertEqual(success_count[0] + success_count[1], 100) + self.assertEqual(b.remaining, 0) + self.assertGreater(b.exhausted_count, 0) + + +class TestProbeBaseBudgetHelper(unittest.TestCase): + + def _make_probe_with_budget(self, total): + budget = RequestBudget(remaining=total, total=total) + + class _Probe(ProbeBase): + def run(self): + return self.findings + + p = _Probe( + target_url="http://x", auth_manager=MagicMock(), + target_config=MagicMock(), safety=MagicMock(), + request_budget=budget, + ) + return p, budget + + def test_budget_helper_consumes(self): + p, budget = self._make_probe_with_budget(2) + self.assertTrue(p.budget()) + self.assertTrue(p.budget()) + self.assertFalse(p.budget()) + self.assertEqual(budget.exhausted_count, 1) + + def test_budget_helper_no_budget_always_true(self): + """ProbeBase without a budget (legacy callers) should never block.""" + class _Probe(ProbeBase): + def run(self): + return self.findings + + p = _Probe( + target_url="http://x", auth_manager=MagicMock(), + target_config=MagicMock(), safety=MagicMock(), + ) + for _ in range(100): + self.assertTrue(p.budget()) + + +class TestRequestBudgetSharedAcrossProbes(unittest.TestCase): + """Two probe instances share one budget — total consumption never exceeds cap.""" + + def test_two_probes_share_one_budget(self): + budget = RequestBudget(remaining=5, total=5) + + class _Probe(ProbeBase): + def run(self): + return self.findings + + p1 = _Probe( + target_url="http://x", auth_manager=MagicMock(), + target_config=MagicMock(), safety=MagicMock(), + request_budget=budget, + ) + p2 = _Probe( + target_url="http://x", auth_manager=MagicMock(), + target_config=MagicMock(), safety=MagicMock(), + request_budget=budget, + ) + + self.assertTrue(p1.budget()) + self.assertTrue(p2.budget()) + self.assertTrue(p1.budget()) + self.assertTrue(p2.budget()) + self.assertTrue(p1.budget()) + # Five total — next call from either probe fails. + self.assertFalse(p1.budget()) + self.assertFalse(p2.budget()) + self.assertEqual(budget.remaining, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_detection_inventory.py b/extensions/business/cybersec/red_mesh/tests/test_detection_inventory.py index d7bd0e49..17c3896d 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_detection_inventory.py +++ b/extensions/business/cybersec/red_mesh/tests/test_detection_inventory.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import ast import unittest from pathlib import Path @@ -11,6 +12,7 @@ from extensions.business.cybersec.red_mesh.detection_inventory import build_detection_inventory from extensions.business.cybersec.red_mesh.graybox.scenario_catalog import ( GRAYBOX_SCENARIO_CATALOG, + attack_for_scenario, ) from extensions.business.cybersec.red_mesh.worker.blackbox_detection_catalog import ( BLACKBOX_DETECTION_CATALOG, @@ -23,7 +25,10 @@ def test_detection_inventory_meets_coverage_targets(self): counts = build_detection_inventory().counts() self.assertGreaterEqual(counts["total"], 300) self.assertGreaterEqual(counts["blackbox"], 220) - self.assertGreaterEqual(counts["graybox"], 80) + # Graybox floor bumped from 80 -> 103 by Subphase 1.2 of the API Top 10 + # plan (23 new PT-OAPI* entries). Post-implementation target is >=120 + # (continued OWASP Web Top 10 closing). + self.assertGreaterEqual(counts["graybox"], 103) self.assertGreaterEqual(counts["cves"], 200) def test_detection_ids_are_unique(self): @@ -50,18 +55,160 @@ def test_blackbox_catalog_maps_to_registered_network_methods(self): ] self.assertEqual(missing, []) + # Valid graybox scenario-id prefixes (see docs/adr/2026-05-12-scenario-id-convention.md): + # PT-A- — OWASP Web Top 10 2021 scenarios (existing). + # PT-API7- — legacy SSRF ID, preserved for backward compatibility. + # PT-OAPI- — OWASP API Top 10 2023 scenarios (new in v1). + _SCENARIO_ID_RE = re.compile( + r"scenario_id\s*=\s*[\"'](PT-A\d+-\d+|PT-API7-\d+|PT-OAPI\d{1,2}-\d+)[\"']" + ) + _SCENARIO_ID_VALUE_RE = re.compile( + r"^(PT-A\d+-\d+|PT-API7-\d+|PT-OAPI\d{1,2}-\d+)$" + ) + _SCENARIO_CALLS = { + "emit_vulnerable", + "emit_clean", + "emit_inconclusive", + "run_safe_scenario", + "run_stateful", + } + + @classmethod + def _collect_ast_scenario_ids(cls, redmesh_root): + source_ids = set() + for path in (redmesh_root / "graybox").rglob("*.py"): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = "" + if isinstance(func, ast.Attribute): + name = func.attr + elif isinstance(func, ast.Name): + name = func.id + candidates = [] + if name in cls._SCENARIO_CALLS and node.args: + candidates.append(node.args[0]) + if name == "GrayboxFinding": + candidates.extend( + kw.value for kw in node.keywords + if kw.arg == "scenario_id" + ) + candidates.extend( + kw.value for kw in node.keywords + if kw.arg == "scenario_id" + ) + for candidate in candidates: + if isinstance(candidate, ast.Constant) and isinstance(candidate.value, str): + if cls._SCENARIO_ID_VALUE_RE.match(candidate.value): + source_ids.add(candidate.value) + return source_ids + def test_existing_graybox_emitted_scenarios_are_registered(self): redmesh_root = Path(__file__).resolve().parents[1] - source_ids = set() + source_ids = self._collect_ast_scenario_ids(redmesh_root) for path in (redmesh_root / "graybox").rglob("*.py"): - source_ids.update(re.findall( - r"scenario_id\s*=\s*[\"'](PT-[A-Z0-9]+-\d+|PT-API7-\d+)[\"']", - path.read_text(), - )) + source_ids.update(self._SCENARIO_ID_RE.findall(path.read_text())) catalog_ids = {entry["id"] for entry in GRAYBOX_SCENARIO_CATALOG} self.assertTrue(source_ids) self.assertEqual(source_ids - catalog_ids, set()) + def test_api_probe_modules_use_emit_helpers_for_findings(self): + """New API probe families should not bypass central emission helpers.""" + redmesh_root = Path(__file__).resolve().parents[1] + direct = [] + for path in (redmesh_root / "graybox" / "probes").glob("api_*.py"): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = func.id if isinstance(func, ast.Name) else "" + if name == "GrayboxFinding": + direct.append(f"{path.name}:{node.lineno}") + self.assertEqual(direct, []) + + def test_scenario_id_regex_accepts_all_valid_prefixes(self): + """Regex must accept the three valid prefixes documented in the ADR.""" + cases = [ + ('scenario_id="PT-A01-01"', "PT-A01-01"), + ('scenario_id="PT-A07-06"', "PT-A07-06"), + ('scenario_id="PT-API7-01"', "PT-API7-01"), + ('scenario_id="PT-OAPI1-01"', "PT-OAPI1-01"), + ('scenario_id="PT-OAPI9-03"', "PT-OAPI9-03"), + ('scenario_id="PT-OAPI10-01"', "PT-OAPI10-01"), + ] + for source, expected in cases: + with self.subTest(source=source): + match = self._SCENARIO_ID_RE.search(source) + self.assertIsNotNone(match, f"regex failed to match {source!r}") + self.assertEqual(match.group(1), expected) + + def test_scenario_id_regex_rejects_invalid_prefixes(self): + """Regex must reject obvious typos so they surface as catalog misses.""" + rejects = [ + 'scenario_id="PT-FOO-01"', + 'scenario_id="PT-API1-01"', # ambiguous w/ PT-A — must use PT-OAPI + 'scenario_id="OAPI1-01"', + 'scenario_id="PT-OAPI-01"', + ] + for source in rejects: + with self.subTest(source=source): + self.assertIsNone(self._SCENARIO_ID_RE.search(source)) + + def test_v1_api_scenarios_have_non_empty_attack_mapping(self): + """Every v1 OWASP API Top 10 scenario must declare ATT&CK techniques. + + Implements the mandatory ATT&CK mapping requirement from Subphase 1.2 + of the API Top 10 plan. The catalog is the single source of truth for + `attack=[]` defaults emitted by probes via `ProbeBase.emit_vulnerable`. + """ + # In v1, the prefix `PT-OAPI` identifies the new API Top 10 scenarios. + # The legacy `PT-API7-01` is also subject to this requirement so the + # SSRF probe carries an ATT&CK mapping consistent with the others. + mandatory_prefixes = ("PT-OAPI", "PT-API7") + missing = [] + for entry in GRAYBOX_SCENARIO_CATALOG: + sid = entry["id"] + if not any(sid.startswith(p) for p in mandatory_prefixes): + continue + attack = entry.get("attack") + if not attack: + missing.append(sid) + self.assertEqual( + missing, + [], + f"v1 API scenarios missing non-empty `attack` mapping: {missing}", + ) + + def test_attack_for_scenario_helper(self): + """Helper returns catalog's `attack` list or empty for unknown/legacy IDs.""" + # Known new entry + self.assertEqual(attack_for_scenario("PT-OAPI1-01"), ["T1190", "T1078"]) + # Legacy SSRF + self.assertEqual(attack_for_scenario("PT-API7-01"), ["T1190"]) + # Legacy PT-A* without explicit attack -> empty + self.assertEqual(attack_for_scenario("PT-A01-01"), []) + # Unknown id -> empty (not KeyError) + self.assertEqual(attack_for_scenario("PT-NOT-REAL-99"), []) + + def test_v1_api_scenario_count(self): + """v1 catalog contains exactly 23 new PT-OAPI scenarios; no PT-OAPI10.""" + oapi_ids = { + entry["id"] for entry in GRAYBOX_SCENARIO_CATALOG + if entry["id"].startswith("PT-OAPI") + } + self.assertEqual(len(oapi_ids), 23) + # API10 deliberately omitted in v1 (Phase 9 follow-up) + self.assertNotIn("PT-OAPI10-01", oapi_ids) + # Spot-check coverage per category + for cat in (1, 2, 3, 4, 5, 6, 8, 9): + self.assertTrue( + any(i.startswith(f"PT-OAPI{cat}-") for i in oapi_ids), + f"missing PT-OAPI{cat}-* entries", + ) + class TestCveVersionNormalization(unittest.TestCase): diff --git a/extensions/business/cybersec/red_mesh/tests/test_e2e_harness.py b/extensions/business/cybersec/red_mesh/tests/test_e2e_harness.py index 24ef7e80..c641b799 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_e2e_harness.py +++ b/extensions/business/cybersec/red_mesh/tests/test_e2e_harness.py @@ -1,4 +1,10 @@ from extensions.business.cybersec.red_mesh.tests.e2e.run_e2e import archive_passes +from extensions.business.cybersec.red_mesh.tests.e2e.api_top10_e2e import ( + assert_llm_boundary, + llm_boundary_blob_from_archive, + target_config_with_bearer_auth, + target_confirmation_for_url, +) def test_archive_passes_prefers_current_archive_schema(): @@ -19,3 +25,52 @@ def test_archive_passes_keeps_legacy_pass_reports_fallback(): def test_archive_passes_handles_invalid_archives(): assert archive_passes(None) == [] assert archive_passes({"passes": "bad", "pass_reports": "bad"}) == [] + + +def test_api_top10_target_confirmation_uses_host_only(): + assert target_confirmation_for_url("http://localhost:30001") == "localhost" + assert target_confirmation_for_url("https://api.example.com/app") == "api.example.com" + assert target_confirmation_for_url("api.internal") == "api.internal" + + +def test_api_top10_target_config_layers_bearer_auth_without_mutating_fixture(): + fixture = { + "api_security": { + "object_endpoints": [{"path": "/api/users/{id}/"}], + }, + } + + configured = target_config_with_bearer_auth(fixture) + + assert fixture["api_security"].get("auth") is None + assert configured["api_security"]["object_endpoints"] == [{"path": "/api/users/{id}/"}] + assert configured["api_security"]["auth"] == { + "auth_type": "bearer", + "bearer_token_header_name": "Authorization", + "bearer_scheme": "Bearer", + "authenticated_probe_path": "/api/v2/me/", + } + + +def test_api_top10_llm_boundary_blob_uses_archive_report_fields(): + archive = { + "job_config": { + "target_config": {"api_security": {"auth": {"bearer_token": "not included"}}}, + }, + "passes": [ + { + "findings": [ + {"scenario_id": "PT-OAPI2-01", "evidence": "Authorization: Bearer [REDACTED]"}, + ], + "llm_analysis": {"summary": "clean"}, + "quick_summary": "No raw tokens.", + "llm_report_sections": {"api_top10": "Redacted API finding."}, + }, + ], + } + + blob = llm_boundary_blob_from_archive(archive) + + assert "not included" not in blob + assert assert_llm_boundary(blob) == [] + assert assert_llm_boundary(blob + " Authorization: Bearer eyJabc.def.ghi") diff --git a/extensions/business/cybersec/red_mesh/tests/test_finalization_aggregation.py b/extensions/business/cybersec/red_mesh/tests/test_finalization_aggregation.py index d3b8e4a6..88facaac 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_finalization_aggregation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_finalization_aggregation.py @@ -230,6 +230,132 @@ def test_findings_carry_worker_and_node_attribution(self): self.assertEqual(f["_source_node_addr"], "0xnode_a") +class TestApiTop10FlatFindingIntegration(unittest.TestCase): + """OWASP API Top 10 — Subphase 5.1 of the API Top 10 plan. + + Verifies that findings emitted by the new `_graybox_api_*` families + flatten into the unified flat-finding schema with the correct + probe attribution, scenario_id, severity, and rollback_status. + """ + + def _make_finding(self, scenario_id, **overrides): + """Build a minimal GrayboxFinding via the typed dataclass.""" + from extensions.business.cybersec.red_mesh.graybox.findings import ( + GrayboxFinding, + ) + defaults = dict( + scenario_id=scenario_id, + title=f"finding {scenario_id}", + status="vulnerable", + severity="HIGH", + owasp="API1:2023", + cwe=["CWE-639"], + attack=["T1190"], + evidence=["endpoint=/api/x", "owner_field=owner"], + ) + defaults.update(overrides) + return GrayboxFinding(**defaults) + + def test_each_new_api_family_flattens_correctly(self): + """Each of the five api_* probe-family keys carries through to flat findings.""" + cases = [ + ("PT-OAPI1-01", "_graybox_api_access", "API1:2023"), + ("PT-OAPI2-01", "_graybox_api_auth", "API2:2023"), + ("PT-OAPI3-01", "_graybox_api_data", "API3:2023"), + ("PT-OAPI8-01", "_graybox_api_config", "API8:2023"), + ("PT-OAPI4-01", "_graybox_api_abuse", "API4:2023"), + ] + for scenario_id, probe_key, owasp in cases: + with self.subTest(scenario_id=scenario_id): + f = self._make_finding(scenario_id, owasp=owasp) + flat = f.to_flat_finding(443, "https", probe_key) + self.assertEqual(flat["probe_type"], "graybox") + self.assertEqual(flat["category"], "graybox") + self.assertEqual(flat["probe"], probe_key) + self.assertEqual(flat["scenario_id"], scenario_id) + self.assertEqual(flat["owasp_id"], owasp) + self.assertEqual(flat["severity"], "HIGH") + # ATT&CK + CWE survive + self.assertIn("CWE-639", flat["cwe_id"]) + self.assertEqual(flat["attack_ids"], ["T1190"]) + + def test_rollback_status_field_present_on_flat(self): + """rollback_status (Subphase 1.8) flows through to flat findings.""" + f = self._make_finding( + "PT-OAPI3-02", owasp="API3:2023", + rollback_status="reverted", + ) + flat = f.to_flat_finding(443, "https", "_graybox_api_data") + self.assertEqual(flat["rollback_status"], "reverted") + + def test_revert_failed_flag_visible(self): + """Operators see revert_failed at the flat-finding boundary.""" + f = self._make_finding( + "PT-OAPI3-02", owasp="API3:2023", + severity="CRITICAL", rollback_status="revert_failed", + ) + flat = f.to_flat_finding(443, "https", "_graybox_api_data") + self.assertEqual(flat["rollback_status"], "revert_failed") + self.assertEqual(flat["severity"], "CRITICAL") + + +class TestApiTop10DedupPosture(unittest.TestCase): + """Subphase 5.3: PT-A01-01 (web IDOR) and PT-OAPI1-01 (API BOLA) on the + same asset must NOT collapse — they describe different vulnerability + classes and should both surface in the report.""" + + def test_pt_a01_and_pt_oapi1_coexist(self): + from extensions.business.cybersec.red_mesh.graybox.findings import ( + GrayboxFinding, + ) + common = dict(status="vulnerable", severity="HIGH", + evidence=["endpoint=/api/records/1"]) + f_web = GrayboxFinding(scenario_id="PT-A01-01", title="IDOR/BOLA read bypass", + owasp="A01:2021", cwe=["CWE-639"], **common) + f_api = GrayboxFinding(scenario_id="PT-OAPI1-01", + title="API object-level authorization bypass (BOLA)", + owasp="API1:2023", cwe=["CWE-639", "CWE-284"], + **common) + flat_web = f_web.to_flat_finding(443, "https", "_graybox_access_control") + flat_api = f_api.to_flat_finding(443, "https", "_graybox_api_access") + # Different probe_name + different title + different scenario_id = + # different finding_id (sha256 of port:probe:cwe:title). + self.assertNotEqual(flat_web["finding_id"], flat_api["finding_id"]) + self.assertNotEqual(flat_web["probe"], flat_api["probe"]) + self.assertNotEqual(flat_web["scenario_id"], flat_api["scenario_id"]) + + +class TestApiTop10BudgetMetrics(unittest.TestCase): + """OWASP API Top 10 — Subphase 5.1 budget integration assertion. + + When the per-scan RequestBudget is exhausted, the worker outcome dict + surfaces budget_total/budget_remaining/budget_exhausted_count under + scan_metrics so report consumers see the cap in effect. + """ + + def test_budget_metrics_surface_in_get_status(self): + from extensions.business.cybersec.red_mesh.graybox.budget import ( + RequestBudget, + ) + # Minimal worker stub exposing only what get_status reads. + worker = MagicMock() + worker.request_budget = RequestBudget(remaining=5, total=5) + worker.request_budget.consume(3) # consume 3 → 2 left + worker.request_budget.consume(10) # exhaust attempt → +1 to count + + # Re-implement the metrics merge inline so we don't need a full + # GrayboxLocalWorker (which requires R1FS setup, etc.). + snap = worker.request_budget.snapshot() + metrics = { + "budget_total": snap["total"], + "budget_remaining": snap["remaining"], + "budget_exhausted_count": snap["exhausted_count"], + } + self.assertEqual(metrics["budget_total"], 5) + self.assertEqual(metrics["budget_remaining"], 2) + self.assertEqual(metrics["budget_exhausted_count"], 1) + + class TestNetworkAggregationRegression(unittest.TestCase): def test_network_aggregation_still_works_without_worker_cls(self): diff --git a/extensions/business/cybersec/red_mesh/tests/test_findings_redaction.py b/extensions/business/cybersec/red_mesh/tests/test_findings_redaction.py new file mode 100644 index 00000000..71630867 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_findings_redaction.py @@ -0,0 +1,235 @@ +"""OWASP API Top 10 — Subphase 1.6 commit #3. + +Storage-boundary scrubber tests. Asserts that the centralised +`scrub_graybox_secrets` (and the `to_flat_finding` pass-through) strip +every documented secret pattern even when probes don't redact at +emission time. +""" + +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock + +import requests + +from extensions.business.cybersec.red_mesh.graybox.probes.base import ProbeBase +from extensions.business.cybersec.red_mesh.graybox.safety import SafetyControls +from extensions.business.cybersec.red_mesh.graybox.findings import ( + FindingRedactionContext, + GrayboxFinding, + scrub_graybox_secrets, +) +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiSecurityConfig, + AuthDescriptor, + GrayboxTargetConfig, +) + + +SAMPLE_JWT = "eyJabcdefghi.payload-foo.signature-bar" +LONG_BEARER = "abcdef0123456789abcdef0123456789" + + +class TestScrubGenericPatterns(unittest.TestCase): + + def test_authorization_header_redacted(self): + out = scrub_graybox_secrets(f"Authorization: Bearer {SAMPLE_JWT}") + self.assertNotIn(SAMPLE_JWT, out) + self.assertIn("", out) + + def test_cookie_header_redacted(self): + out = scrub_graybox_secrets("Cookie: sessionid=abc123def456") + self.assertNotIn("sessionid=abc123", out) + self.assertIn("", out) + + def test_set_cookie_header_redacted(self): + out = scrub_graybox_secrets("Set-Cookie: token=eyJabcdef") + self.assertNotIn("eyJabcdef", out) + + def test_bare_jwt_redacted(self): + out = scrub_graybox_secrets(f"server returned: {SAMPLE_JWT}") + self.assertNotIn(SAMPLE_JWT, out) + self.assertIn("", out) + + def test_bare_bearer_redacted(self): + out = scrub_graybox_secrets(f"trace: Bearer {LONG_BEARER}") + self.assertNotIn(LONG_BEARER, out) + self.assertIn("Bearer ", out) + + def test_password_kv_redacted(self): + out = scrub_graybox_secrets("user=admin&password=hunter2&keep=this") + self.assertNotIn("hunter2", out) + self.assertIn("password=", out) + + def test_api_key_kv_redacted(self): + out = scrub_graybox_secrets("?api_key=ABCDEFG12345&x=1") + self.assertNotIn("ABCDEFG12345", out) + + def test_apikey_kv_redacted(self): + """Variant spelling.""" + out = scrub_graybox_secrets("?apikey=XYZ123ABCDEF&extra=ok") + self.assertNotIn("XYZ123ABCDEF", out) + + def test_json_bearer_token_redacted(self): + out = scrub_graybox_secrets('{"bearer_token": "eyJsecret.payload.sig", "user": "alice"}') + self.assertNotIn("eyJsecret", out) + self.assertIn("alice", out) # non-secret values preserved + + def test_embedded_header_in_evidence_redacted(self): + out = scrub_graybox_secrets( + "status=200, Authorization: Bearer SECRET-TOKEN-HERE-12345, foo=bar" + ) + self.assertNotIn("SECRET-TOKEN-HERE-12345", out) + self.assertIn("foo=bar", out) + + +class TestScrubConfiguredNames(unittest.TestCase): + + def test_custom_header_redacted(self): + out = scrub_graybox_secrets( + "X-Customer-Api-Key: abc123secret", + secret_field_names=("X-Customer-Api-Key",), + ) + self.assertNotIn("abc123secret", out) + + def test_custom_query_param_redacted(self): + out = scrub_graybox_secrets( + "https://api.example.com/v1/me?token_param=SECRET99&page=1", + secret_field_names=("token_param",), + ) + self.assertNotIn("SECRET99", out) + self.assertIn("page=1", out) + + +class TestScrubRecursive(unittest.TestCase): + + def test_list_recursion(self): + out = scrub_graybox_secrets(["normal evidence", "password=secret123"]) + self.assertNotIn("secret123", str(out)) + + def test_dict_recursion(self): + out = scrub_graybox_secrets({ + "ok": "value", + "request_snapshot": {"headers": "Authorization: Bearer eyJabcdefghi.x.y"}, + }) + self.assertNotIn("eyJabcdefghi", str(out)) + self.assertEqual(out["ok"], "value") + + def test_non_string_passthrough(self): + self.assertEqual(scrub_graybox_secrets(42), 42) + self.assertIsNone(scrub_graybox_secrets(None)) + + +class TestToFlatFindingScrubs(unittest.TestCase): + + def test_evidence_scrubbed_on_flatten(self): + f = GrayboxFinding( + scenario_id="PT-OAPI1-01", + title="API object-level authorization bypass (BOLA)", + status="vulnerable", + severity="HIGH", + owasp="API1:2023", + evidence=[ + "endpoint=/api/users/2", + "Authorization: Bearer eyJsecret.payload.sig", + "password=hunter2_leak", + ], + replay_steps=["GET /api/users/2 with token=abc123def456"], + remediation="Bearer SECRET-DEFAULT-TOKEN should be rotated", + ) + flat = f.to_flat_finding(443, "https", "_graybox_api_access") + haystack = str(flat) + self.assertNotIn("eyJsecret", haystack) + self.assertNotIn("hunter2_leak", haystack) + self.assertNotIn("abc123def456", haystack) + self.assertNotIn("SECRET-DEFAULT-TOKEN", haystack) + # Non-secret content preserved + self.assertIn("/api/users/2", haystack) + self.assertIn("PT-OAPI1-01", haystack) + + def test_flatten_context_scrubs_configured_names(self): + f = GrayboxFinding( + scenario_id="PT-OAPI1-01", + title="API object-level authorization bypass (BOLA)", + status="vulnerable", + severity="HIGH", + owasp="API1:2023", + evidence=[ + "X-Customer-Api-Key: SECRET-HEADER", + "endpoint=https://api.example/v1/users?customer_key=SECRET99&page=1", + ], + evidence_artifacts=[{ + "request_snapshot": ( + "GET /v1/users?customer_key=SECRET99 " + "X-Customer-Api-Key: SECRET-HEADER" + ), + }], + replay_steps=["GET /v1/users?customer_key=SECRET99"], + ) + + with FindingRedactionContext( + secret_field_names=("X-Customer-Api-Key", "customer_key"), + ): + flat = f.to_flat_finding(443, "https", "_graybox_api_access") + stored = f.to_dict() + + haystack = f"{flat} {stored}" + self.assertNotIn("SECRET99", haystack) + self.assertNotIn("SECRET-HEADER", haystack) + self.assertIn("customer_key=", haystack) + self.assertIn("X-Customer-Api-Key: ", haystack) + + +class TestProbeErrorScrubsConfiguredNames(unittest.TestCase): + + def _probe(self): + target_config = GrayboxTargetConfig(api_security=ApiSecurityConfig( + auth=AuthDescriptor( + auth_type="api_key", + api_key_location="query", + api_key_query_param="customer_key", + api_key_header_name="X-Customer-Api-Key", + ) + )) + return ProbeBase( + "https://api.example.com", + MagicMock(), + target_config, + SafetyControls(), + ) + + def test_run_safe_redacts_configured_query_key_from_request_exception(self): + probe = self._probe() + + def boom(): + raise requests.RequestException( + "GET https://api.example.com/v1/users?customer_key=SECRET99&page=1 failed" + ) + + probe.run_safe("api_error_path", boom) + + finding = probe.findings[0] + haystack = str(finding.to_dict()) + self.assertNotIn("SECRET99", haystack) + self.assertIn("customer_key=", haystack) + self.assertIn("page=1", haystack) + + def test_connection_error_redacts_configured_header_name(self): + probe = self._probe() + + def boom(): + raise requests.exceptions.ConnectionError( + "request failed with X-Customer-Api-Key: SECRET-HEADER-VALUE" + ) + + probe.run_safe("api_connection", boom) + + haystack = str(probe.findings[0].to_dict()) + self.assertNotIn("SECRET-HEADER-VALUE", haystack) + self.assertIn("target_unreachable", haystack) + self.assertIn("X-Customer-Api-Key: ", haystack) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_http_client.py b/extensions/business/cybersec/red_mesh/tests/test_http_client.py new file mode 100644 index 00000000..20882365 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_http_client.py @@ -0,0 +1,287 @@ +import ast +from pathlib import Path +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.http_client import ( + GrayboxHttpClient, + GrayboxScopeError, + path_in_scope, + validate_target_config_paths, +) + + +class TestGrayboxHttpClient(unittest.TestCase): + + def _session(self, response=None): + session = MagicMock() + resp = response or MagicMock(status_code=200, headers={}) + session.request.return_value = resp + return session + + def test_path_prefix_matching_is_segment_aware(self): + self.assertTrue(path_in_scope("/api/public/users", "/api/public/")) + self.assertFalse(path_in_scope("/api/publicity", "/api/public/")) + + def test_blocks_cross_host_without_sending_request(self): + client = GrayboxHttpClient( + "https://api.example.com", + allowlist=["/api/public/"], + ) + session = self._session() + + with self.assertRaises(GrayboxScopeError): + client.request(session, "GET", "https://evil.example/api/public/") + + session.request.assert_not_called() + + def test_blocks_encoded_traversal_without_sending_request(self): + client = GrayboxHttpClient( + "https://api.example.com", + allowlist=["/api/public/"], + ) + session = self._session() + + with self.assertRaises(GrayboxScopeError): + client.request(session, "GET", "/api/public/%2e%2e/admin/") + + session.request.assert_not_called() + + def test_blocks_publicity_when_public_scope_authorized(self): + client = GrayboxHttpClient( + "https://api.example.com", + allowlist=["/api/public/"], + ) + session = self._session() + + with self.assertRaises(GrayboxScopeError): + client.request(session, "GET", "/api/publicity") + + session.request.assert_not_called() + + def test_allows_in_scope_templated_launch_path(self): + errors = validate_target_config_paths( + "https://api.example.com", + { + "login_path": "/api/public/login/", + "logout_path": "/api/public/logout/", + "api_security": { + "object_endpoints": [ + {"path": "/api/public/users/{id}/"}, + ], + }, + }, + ["/api/public/"], + ) + self.assertEqual(errors, []) + + def test_blocks_out_of_scope_launch_path(self): + errors = validate_target_config_paths( + "https://api.example.com", + { + "login_path": "/api/public/login/", + "logout_path": "/api/public/logout/", + "api_security": { + "function_endpoints": [ + {"path": "/admin/export-users/"}, + ], + }, + }, + ["/api/public/"], + ) + self.assertTrue(errors) + self.assertIn("outside authorized scope", errors[0]) + + def test_post_redirect_302_converts_to_get_and_drops_body(self): + """Browser-equivalent 302 handling — Django form-login redirect target + must not receive the original POST body, or the redirected request + fails CSRF on the new view (the bug behind official_login_failed).""" + client = GrayboxHttpClient( + "https://target.local", allowlist=["/auth/", "/dashboard/"], + ) + session = MagicMock() + redirect_resp = MagicMock( + status_code=302, headers={"Location": "/dashboard/"}, + ) + final_resp = MagicMock(status_code=200, headers={}) + session.request.side_effect = [redirect_resp, final_resp] + + result = client.request( + session, "POST", "/auth/login/", + data={"username": "admin", "password": "secret", + "csrfmiddlewaretoken": "tok"}, + allow_redirects=True, + ) + + self.assertIs(result, final_resp) + self.assertEqual(session.request.call_count, 2) + first_call = session.request.call_args_list[0] + self.assertEqual(first_call.args[0], "POST") + self.assertIn("login", first_call.args[1]) + self.assertIn("data", first_call.kwargs) + + second_call = session.request.call_args_list[1] + self.assertEqual(second_call.args[0], "GET") + self.assertIn("dashboard", second_call.args[1]) + self.assertNotIn("data", second_call.kwargs) + self.assertNotIn("json", second_call.kwargs) + + def test_post_redirect_301_converts_to_get_and_drops_body(self): + """301 from POST is also browser-equivalent GET (matches `requests`).""" + client = GrayboxHttpClient( + "https://target.local", allowlist=["/old/", "/new/"], + ) + session = MagicMock() + session.request.side_effect = [ + MagicMock(status_code=301, headers={"Location": "/new/"}), + MagicMock(status_code=200, headers={}), + ] + + client.request( + session, "POST", "/old/", data={"k": "v"}, allow_redirects=True, + ) + + second_call = session.request.call_args_list[1] + self.assertEqual(second_call.args[0], "GET") + self.assertNotIn("data", second_call.kwargs) + + def test_post_redirect_307_preserves_method_and_body(self): + """307 (and 308) explicitly preserve method + body per RFC 7231.""" + client = GrayboxHttpClient( + "https://target.local", allowlist=["/api/"], + ) + session = MagicMock() + session.request.side_effect = [ + MagicMock(status_code=307, headers={"Location": "/api/v2/"}), + MagicMock(status_code=200, headers={}), + ] + + client.request( + session, "POST", "/api/v1/", data={"k": "v"}, allow_redirects=True, + ) + + second_call = session.request.call_args_list[1] + self.assertEqual(second_call.args[0], "POST") + self.assertEqual(second_call.kwargs.get("data"), {"k": "v"}) + + def test_post_redirect_303_still_converts(self): + """Pre-existing 303 conversion path must keep working (regression guard).""" + client = GrayboxHttpClient( + "https://target.local", allowlist=["/api/", "/done/"], + ) + session = MagicMock() + session.request.side_effect = [ + MagicMock(status_code=303, headers={"Location": "/done/"}), + MagicMock(status_code=200, headers={}), + ] + + client.request( + session, "POST", "/api/", data={"k": "v"}, allow_redirects=True, + ) + + second_call = session.request.call_args_list[1] + self.assertEqual(second_call.args[0], "GET") + self.assertNotIn("data", second_call.kwargs) + + def test_head_on_302_stays_head(self): + """HEAD is idempotent + has no body; preserve method on redirect.""" + client = GrayboxHttpClient( + "https://target.local", allowlist=["/a/", "/b/"], + ) + session = MagicMock() + session.request.side_effect = [ + MagicMock(status_code=302, headers={"Location": "/b/"}), + MagicMock(status_code=200, headers={}), + ] + + client.request(session, "HEAD", "/a/", allow_redirects=True) + + second_call = session.request.call_args_list[1] + self.assertEqual(second_call.args[0], "HEAD") + + def test_302_without_location_returns_redirect_response(self): + """No Location header → don't loop; return the redirect response as-is.""" + client = GrayboxHttpClient("https://target.local", allowlist=["/a/"]) + session = MagicMock() + bad_redirect = MagicMock(status_code=302, headers={}) + session.request.return_value = bad_redirect + + result = client.request(session, "POST", "/a/", allow_redirects=True) + + self.assertIs(result, bad_redirect) + self.assertEqual(session.request.call_count, 1) + + def test_302_to_out_of_scope_location_raises_scope_error(self): + """Redirect to a path outside the allowlist must abort, not silently follow.""" + client = GrayboxHttpClient( + "https://target.local", allowlist=["/auth/"], + ) + session = MagicMock() + session.request.return_value = MagicMock( + status_code=302, headers={"Location": "/admin/secret/"}, + ) + + with self.assertRaises(GrayboxScopeError): + client.request(session, "POST", "/auth/login/", allow_redirects=True) + + def test_redirect_loop_caps_at_five_hops(self): + """A pathological redirect chain stops after 5 hops, returning the last response.""" + client = GrayboxHttpClient( + "https://target.local", allowlist=["/loop/"], + ) + session = MagicMock() + session.request.return_value = MagicMock( + status_code=302, headers={"Location": "/loop/"}, + ) + + result = client.request( + session, "POST", "/loop/", data={"k": "v"}, allow_redirects=True, + ) + + self.assertEqual(result.status_code, 302) + self.assertEqual(session.request.call_count, 5) + + def test_chained_302_then_302_after_post_settles_on_get(self): + """POST→302→GET; subsequent 302→GET stays GET (method conversion is sticky).""" + client = GrayboxHttpClient( + "https://target.local", + allowlist=["/a/", "/b/", "/c/"], + ) + session = MagicMock() + session.request.side_effect = [ + MagicMock(status_code=302, headers={"Location": "/b/"}), + MagicMock(status_code=302, headers={"Location": "/c/"}), + MagicMock(status_code=200, headers={}), + ] + + client.request( + session, "POST", "/a/", data={"k": "v"}, allow_redirects=True, + ) + + self.assertEqual(session.request.call_args_list[0].args[0], "POST") + self.assertEqual(session.request.call_args_list[1].args[0], "GET") + self.assertNotIn("data", session.request.call_args_list[1].kwargs) + self.assertEqual(session.request.call_args_list[2].args[0], "GET") + + def test_probe_modules_do_not_call_requests_directly(self): + root = Path("extensions/business/cybersec/red_mesh/graybox/probes") + forbidden = {"get", "post", "put", "patch", "delete", "head", "options", "request"} + violations = [] + for path in sorted(root.glob("*.py")): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if ( + isinstance(func, ast.Attribute) + and isinstance(func.value, ast.Name) + and func.value.id == "requests" + and func.attr in forbidden + ): + violations.append(f"{path}:{node.lineno}: requests.{func.attr}") + self.assertEqual(violations, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py b/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py index 8a32e215..e81de2cb 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py +++ b/extensions/business/cybersec/red_mesh/tests/test_jobconfig_webapp.py @@ -1,5 +1,6 @@ """Tests for JobConfig graybox fields and blackbox Finding unchanged.""" +import json import unittest from extensions.business.cybersec.red_mesh.models.archive import JobConfig, UiAggregate @@ -135,6 +136,51 @@ def test_redact_job_config_noop_when_empty(self): self.assertEqual(redacted["official_password"], "") self.assertEqual(redacted["regular_password"], "") + def test_redact_job_config_masks_nested_target_config_secrets(self): + """Defense-in-depth redaction catches legacy nested target_config secrets.""" + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + d = { + "target": "x", + "target_config_secrets": { + "oauth_client_secret": "OAUTH-CLIENT-SECRET", + }, + "target_config": { + "api_security": { + "token_endpoints": { + "token_request_body": { + "client_id": "redmesh", + "client_secret": "plain-secret", + "nested": { + "refresh_token": "refresh-secret", + }, + }, + }, + "auth": { + "api_key_header_name": "X-Customer-Api-Key", + }, + }, + }, + } + redacted = _ReportMixin._redact_job_config(d) + dumped = json.dumps(redacted) + self.assertNotIn("plain-secret", dumped) + self.assertNotIn("refresh-secret", dumped) + self.assertNotIn("OAUTH-CLIENT-SECRET", dumped) + self.assertEqual( + redacted["target_config_secrets"], + {"oauth_client_secret": "***"}, + ) + self.assertEqual( + redacted["target_config"]["api_security"]["token_endpoints"][ + "token_request_body" + ]["client_secret"], + "***", + ) + self.assertEqual( + redacted["target_config"]["api_security"]["auth"]["api_key_header_name"], + "X-Customer-Api-Key", + ) + class TestUiAggregateGraybox(unittest.TestCase): diff --git a/extensions/business/cybersec/red_mesh/tests/test_llm_input_isolation.py b/extensions/business/cybersec/red_mesh/tests/test_llm_input_isolation.py index 25e755be..b0eca8f3 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_llm_input_isolation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_llm_input_isolation.py @@ -291,6 +291,135 @@ def test_zero_width_chars_stripped(self): self.assertEqual(out.findings[0]["title"], "hiddenpayload") +class TestApiAuthSecretsScrubbed(unittest.TestCase): + """Subphase 1.6 commit #4 — API-flavoured secrets must be scrubbed by + the storage-boundary scrubber BEFORE the finding reaches the LLM input + builder. The build_llm_input layer applies its own length-cap + + prompt-injection neutralisation, but secret redaction is the + GrayboxFinding.to_flat_finding contract. + + This test set treats build_llm_input as a downstream consumer that + receives already-flattened findings — so we feed it findings whose + fields contain secret patterns, and assert the LLM input does not + echo them back. + """ + + _SAMPLE_JWT = "eyJabcdefghi.payload-foo.signature-bar" + _LONG_BEARER = "abcdef0123456789abcdef0123456789" + + def _make_api_finding(self, **overrides): + base = dict(ENRICHED_FINDING) + base.update({ + "scenario_id": "PT-OAPI1-01", + "title": "API object-level authorization bypass (BOLA)", + "owasp_id": "API1:2023", + }) + base.update(overrides) + return base + + def test_authorization_header_never_in_llm_input(self): + """A finding whose evidence_items snippet contains an Authorization + header with a Bearer token should not surface the token in LLM input.""" + from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding + f = GrayboxFinding( + scenario_id="PT-OAPI1-01", + title="API BOLA", + status="vulnerable", + severity="HIGH", + owasp="API1:2023", + evidence=[f"Authorization: Bearer {self._SAMPLE_JWT}"], + ) + flat = f.to_flat_finding(443, "https", "_graybox_api_access") + out = build_llm_input(findings=[flat]) + serialised = repr(out.findings) + self.assertNotIn(self._SAMPLE_JWT, serialised) + + def test_cookie_header_never_in_llm_input(self): + from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding + f = GrayboxFinding( + scenario_id="PT-OAPI2-03", + title="API session not invalidated", + status="vulnerable", + severity="MEDIUM", + owasp="API2:2023", + evidence=["Cookie: sessionid=SUPER-SECRET-COOKIE-VALUE"], + ) + flat = f.to_flat_finding(443, "https", "_graybox_api_auth") + out = build_llm_input(findings=[flat]) + self.assertNotIn("SUPER-SECRET-COOKIE-VALUE", repr(out.findings)) + + def test_password_kv_never_in_llm_input(self): + from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding + f = GrayboxFinding( + scenario_id="PT-OAPI2-02", + title="API JWT weak HMAC", + status="vulnerable", + severity="HIGH", + owasp="API2:2023", + evidence=["password=hunter2_leak", "weak_secret=changeme"], + ) + flat = f.to_flat_finding(443, "https", "_graybox_api_auth") + out = build_llm_input(findings=[flat]) + serialised = repr(out.findings) + self.assertNotIn("hunter2_leak", serialised) + + def test_query_param_api_key_never_in_llm_input(self): + """API-key in URL query param: scrubbed end-to-end. + + Note: build_llm_input drops the legacy `evidence` string field + entirely (test_legacy_evidence_field_not_forwarded covers that). + Whichever path is taken — drop or scrub — the secret value cannot + reach the LLM input. + """ + from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding + f = GrayboxFinding( + scenario_id="PT-OAPI8-01", + title="API permissive CORS — token=ABCDEFG12345", + status="vulnerable", + severity="HIGH", + owasp="API8:2023", + evidence=["url=https://api.example.com/v1/me?api_key=ABCDEFG12345&page=1"], + ) + flat = f.to_flat_finding(443, "https", "_graybox_api_config") + out = build_llm_input(findings=[flat]) + serialised = repr(out.findings) + # Secret value redacted regardless of which field carried it. + self.assertNotIn("ABCDEFG12345", serialised) + + def test_configured_query_param_api_key_never_in_llm_input(self): + from extensions.business.cybersec.red_mesh.graybox.findings import ( + FindingRedactionContext, + GrayboxFinding, + ) + f = GrayboxFinding( + scenario_id="PT-OAPI1-01", + title="API BOLA", + status="vulnerable", + severity="HIGH", + owasp="API1:2023", + evidence=[ + "url=https://api.example.com/v1/me?customer_key=SECRET99&page=1", + ], + evidence_artifacts=[{ + "summary": "X-Customer-Api-Key: SECRET-HEADER", + "request_snapshot": ( + "GET /v1/me?customer_key=SECRET99 " + "X-Customer-Api-Key: SECRET-HEADER" + ), + }], + ) + + with FindingRedactionContext( + secret_field_names=("customer_key", "X-Customer-Api-Key"), + ): + flat = f.to_flat_finding(443, "https", "_graybox_api_access") + out = build_llm_input(findings=[flat]) + serialised = repr(out.findings) + + self.assertNotIn("SECRET99", serialised) + self.assertNotIn("SECRET-HEADER", serialised) + + # --------------------------------------------------------------------- # Length caps # --------------------------------------------------------------------- diff --git a/extensions/business/cybersec/red_mesh/tests/test_normalization.py b/extensions/business/cybersec/red_mesh/tests/test_normalization.py index def0abb9..99ba7bde 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_normalization.py +++ b/extensions/business/cybersec/red_mesh/tests/test_normalization.py @@ -5,6 +5,9 @@ from extensions.business.cybersec.red_mesh.graybox.findings import GrayboxFinding from extensions.business.cybersec.red_mesh.graybox.worker import GrayboxLocalWorker +from extensions.business.cybersec.red_mesh.graybox.scenario_runtime import ( + build_graybox_worker_assignments, +) from extensions.business.cybersec.red_mesh.worker import PentestLocalWorker from extensions.business.cybersec.red_mesh.constants import ScanType @@ -351,6 +354,58 @@ class MockHost(_ReportMixin): self.assertNotIn("password123", artifact["response_snapshot"]) self.assertNotIn("password123", probe_artifact["summary"]) + def test_redaction_masks_configured_graybox_api_secret_names(self): + from extensions.business.cybersec.red_mesh.mixins.report import _ReportMixin + + class MockHost(_ReportMixin): + pass + + host = MockHost() + report = { + "target_config": { + "api_security": { + "auth": { + "api_key_query_param": "customer_key", + "api_key_header_name": "X-Customer-Api-Key", + }, + }, + }, + "service_info": {}, + "graybox_results": { + "443": { + "_graybox_api_access": { + "findings": [ + { + "title": "X-Customer-Api-Key: SECRET-HEADER", + "evidence": [ + "GET /v1/users?customer_key=SECRET99&page=1", + ], + "replay_steps": [ + "curl /v1/users?customer_key=SECRET99", + ], + "evidence_artifacts": [ + { + "request_snapshot": ( + "GET /v1/users?customer_key=SECRET99 " + "X-Customer-Api-Key: SECRET-HEADER" + ), + }, + ], + }, + ], + }, + }, + }, + } + + redacted = host._redact_report(report) + + haystack = str(redacted) + self.assertNotIn("SECRET99", haystack) + self.assertNotIn("SECRET-HEADER", haystack) + self.assertIn("customer_key=", haystack) + self.assertIn("X-Customer-Api-Key: ", haystack) + class TestFindingCounting(unittest.TestCase): @@ -423,6 +478,9 @@ def test_dispatch_uses_local_worker_id(self): cfg.target_config = None cfg.verify_tls = True cfg.scan_min_delay = 0 + assignments, _error = build_graybox_worker_assignments(["node-1"]) + for key, value in assignments["node-1"].items(): + setattr(cfg, key, value) worker = GrayboxLocalWorker( owner=MagicMock(), job_id="j1", @@ -455,6 +513,9 @@ def test_probe_kwargs_include_allow_stateful(self): cfg.regular_password = "" cfg.weak_candidates = None cfg.app_routes = None + assignments, _error = build_graybox_worker_assignments(["node-1"]) + for key, value in assignments["node-1"].items(): + setattr(cfg, key, value) worker = GrayboxLocalWorker( owner=MagicMock(), diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_api_abuse.py b/extensions/business/cybersec/red_mesh/tests/test_probes_api_abuse.py new file mode 100644 index 00000000..a3d0aaeb --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_api_abuse.py @@ -0,0 +1,342 @@ +"""OWASP API Top 10 — Subphases 3.2 + 3.3 (ApiAbuseProbes).""" + +from __future__ import annotations + +import json +import unittest +from unittest.mock import MagicMock + +import requests + +from extensions.business.cybersec.red_mesh.graybox.probes.api_abuse import ( + ApiAbuseProbes, +) +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiBusinessFlow, ApiResourceEndpoint, ApiSecurityConfig, GrayboxTargetConfig, +) + + +def _resp(status=200, text="", headers=None): + r = MagicMock() + r.status_code = status + r.text = text + r.headers = headers or {} + return r + + +def _make_probe(*, resource_endpoints=None, business_flows=None, + allow_stateful=False): + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig( + resource_endpoints=list(resource_endpoints or []), + business_flows=list(business_flows or []), + )) + auth = MagicMock() + auth.official_session = MagicMock() + auth.regular_session = MagicMock() + safety = MagicMock() + safety.throttle = MagicMock() + safety.sanitize_error = MagicMock(side_effect=lambda s: s) + return ApiAbuseProbes( + target_url="http://api.example", + auth_manager=auth, target_config=cfg, safety=safety, + allow_stateful=allow_stateful, + ) + + +class TestApi4NoPaginationCap(unittest.TestCase): + + def test_size_explosion_emits_medium(self): + ep = ApiResourceEndpoint(path="/api/records/", baseline_limit=10, + abuse_limit=999_999, + allow_high_limit_probe=True) + p = _make_probe(resource_endpoints=[ep]) + # 100B baseline → 1MB abuse response = >5× growth + p.auth.official_session.get.side_effect = [ + _resp(status=200, text="x" * 100), + _resp(status=200, text="y" * 1_000_000), + ] + p.run_safe("api_no_pagination_cap", p._test_no_pagination_cap) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI4-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "MEDIUM") + + def test_high_limit_probe_caps_requested_limit(self): + ep = ApiResourceEndpoint(path="/api/records/", baseline_limit=10, + abuse_limit=999_999, + allow_high_limit_probe=True) + p = _make_probe(resource_endpoints=[ep]) + p.auth.official_session.get.side_effect = [ + _resp(status=200, text="x" * 100), + _resp(status=200, text="y" * 1_000), + ] + + p.run_safe("api_no_pagination_cap", p._test_no_pagination_cap) + + self.assertEqual( + p.auth.official_session.get.call_args_list[1].kwargs["params"], + {"limit": 1000}, + ) + + +class TestApi4OversizedPayload(unittest.TestCase): + + def test_oversized_accepted_medium(self): + ep = ApiResourceEndpoint(path="/api/notes/", + allow_oversized_payload_probe=True, + oversized_payload_bytes=65_536) + p = _make_probe(resource_endpoints=[ep]) + p.auth.official_session.post.return_value = _resp(status=201) + p.run_safe("api_oversized_payload", p._test_oversized_payload) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI4-02" and f.status == "vulnerable"] + self.assertEqual(vuln[0].severity, "MEDIUM") + + +class TestApi4NoRateLimit(unittest.TestCase): + + def test_only_fires_when_rate_limit_expected(self): + ep = ApiResourceEndpoint(path="/api/list/", rate_limit_expected=False) + p = _make_probe(resource_endpoints=[ep]) + p.auth.official_session.get.return_value = _resp(status=200) + p.run_safe("api_no_rate_limit", p._test_no_rate_limit) + self.assertEqual( + [f for f in p.findings if f.scenario_id == "PT-OAPI4-03"], [], + ) + + def test_10_requests_no_429_or_headers_low(self): + ep = ApiResourceEndpoint(path="/api/list/", rate_limit_expected=True) + p = _make_probe(resource_endpoints=[ep]) + p.auth.official_session.get.return_value = _resp(status=200) + p.run_safe("api_no_rate_limit", p._test_no_rate_limit) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI4-03" and f.status == "vulnerable"] + self.assertEqual(vuln[0].severity, "LOW") + + +class TestApi6FlowAbuse(unittest.TestCase): + + def test_stateful_disabled_emits_inconclusive(self): + flow = ApiBusinessFlow(path="/api/auth/signup/", flow_name="signup", + body_template={"u": "{test_account}", "p": "p"}, + test_account="api-low") + p = _make_probe(business_flows=[flow], allow_stateful=False) + p.run_safe("api_flow_no_rate_limit", p._test_flow_no_rate_limit) + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-01" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("stateful_probes_disabled", + "\n".join(incon[0].evidence)) + + def test_stateful_enabled_without_revert_path_does_not_mutate(self): + flow = ApiBusinessFlow(path="/api/auth/signup/", flow_name="signup", + body_template={"u": "{test_account}", "p": "p"}, + test_account="api-low") + p = _make_probe(business_flows=[flow], allow_stateful=True) + + p.run_safe("api_flow_no_rate_limit", p._test_flow_no_rate_limit) + + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-01" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("no_revert_path_configured", "\n".join(incon[0].evidence)) + p.auth.regular_session.post.assert_not_called() + + def test_rate_limit_flow_reverts_after_confirmed_mutation(self): + flow = ApiBusinessFlow( + path="/api/auth/signup/", + flow_name="signup", + body_template={"u": "{test_account}", "p": "p"}, + revert_path="/api/auth/signup/cleanup/", + revert_body={"u": "{test_account}"}, + test_account="api-low", + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + p.auth.regular_session.post.side_effect = [_resp(status=201)] * 6 + + p.run_safe("api_flow_no_rate_limit", p._test_flow_no_rate_limit) + + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].rollback_status, "reverted") + self.assertEqual(vuln[0].severity, "MEDIUM") + self.assertIn("rollback:", "\n".join(vuln[0].replay_steps)) + self.assertEqual( + p.auth.regular_session.post.call_args_list[-1].args[0], + "http://api.example/api/auth/signup/cleanup/", + ) + for call in p.auth.regular_session.post.call_args_list: + self.assertEqual(call.kwargs["json"]["u"], "api-low") + + def test_static_flow_body_without_placeholder_does_not_mutate(self): + flow = ApiBusinessFlow( + path="/api/auth/signup/", + flow_name="signup", + body_template={"u": "real-user", "p": "p"}, + revert_path="/api/auth/signup/cleanup/", + revert_body={"u": "real-user"}, + test_account="api-low", + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + + p.run_safe("api_flow_no_rate_limit", p._test_flow_no_rate_limit) + + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-01" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("test_account_placeholder_required", + "\n".join(incon[0].evidence)) + p.auth.regular_session.post.assert_not_called() + + def test_static_flow_body_requires_explicit_unsafe_override(self): + flow = ApiBusinessFlow( + path="/api/auth/signup/", + flow_name="signup", + body_template={"u": "fixture-user", "p": "p"}, + revert_path="/api/auth/signup/cleanup/", + revert_body={"u": "fixture-user"}, + test_account="api-low", + allow_static_test_account_body=True, + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + p.auth.regular_session.post.side_effect = [_resp(status=201)] * 6 + + p.run_safe("api_flow_no_rate_limit", p._test_flow_no_rate_limit) + + self.assertTrue(p.auth.regular_session.post.called) + self.assertEqual( + p.auth.regular_session.post.call_args_list[0].kwargs["json"]["u"], + "fixture-user", + ) + + def test_runtime_state_does_not_mutate_flow_config(self): + flow = ApiBusinessFlow( + path="/api/auth/signup/", + flow_name="signup", + body_template={"u": "{test_account}", "p": "p"}, + revert_path="/api/auth/signup/cleanup/", + revert_body={"u": "{test_account}"}, + test_account="api-low", + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + p.auth.regular_session.post.side_effect = [_resp(status=201)] * 6 + + p.run_safe("api_flow_no_rate_limit", p._test_flow_no_rate_limit) + + self.assertFalse(hasattr(flow, "_probe_state")) + self.assertFalse(hasattr(flow, "_probe_state2")) + self.assertEqual(flow.body_template["u"], "{test_account}") + + def test_unsupported_template_expression_does_not_mutate(self): + flow = ApiBusinessFlow( + path="/api/auth/signup/", + flow_name="signup", + body_template={"u": "scan-{test_account}", "p": "p"}, + revert_path="/api/auth/signup/cleanup/", + revert_body={"u": "{test_account}"}, + test_account="api-low", + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + + p.run_safe("api_flow_no_rate_limit", p._test_flow_no_rate_limit) + + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-01" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("unsupported_template_expression", + "\n".join(incon[0].evidence)) + p.auth.regular_session.post.assert_not_called() + + def test_uniqueness_flow_without_revert_path_does_not_mutate(self): + flow = ApiBusinessFlow(path="/api/orders/", flow_name="purchase", + body_template={"account": "{test_account}", + "sku": "sku-1"}, + test_account="api-low") + p = _make_probe(business_flows=[flow], allow_stateful=True) + + p.run_safe("api_flow_no_uniqueness", p._test_flow_no_uniqueness) + + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-02" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("no_revert_path_configured", "\n".join(incon[0].evidence)) + p.auth.regular_session.post.assert_not_called() + + def test_uniqueness_flow_partial_mutation_still_triggers_revert(self): + """B3 (PR406): r1 sent, r2 times out -> revert must run, finding inconclusive.""" + flow = ApiBusinessFlow( + path="/api/orders/", + flow_name="purchase", + body_template={"account": "{test_account}", "sku": "sku-1"}, + revert_path="/api/orders/cleanup/", + revert_body={"account": "{test_account}", "sku": "sku-1"}, + test_account="api-low", + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + revert_response = _resp(status=204) + p.auth.regular_session.post.side_effect = [ + _resp(status=201), + requests.ConnectTimeout("simulated timeout on second send"), + revert_response, + ] + + p.run_safe("api_flow_no_uniqueness", p._test_flow_no_uniqueness) + + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-02" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1, p.findings) + self.assertEqual(incon[0].rollback_status, "reverted") + # Three POSTs: r1, r2 (raises), revert. + self.assertEqual(p.auth.regular_session.post.call_count, 3) + + def test_uniqueness_flow_transport_error_before_mutation_skips_revert(self): + """If transport fails on r1, nothing was mutated → no revert needed.""" + flow = ApiBusinessFlow( + path="/api/orders/", + flow_name="purchase", + body_template={"account": "{test_account}", "sku": "sku-1"}, + revert_path="/api/orders/cleanup/", + revert_body={"account": "{test_account}", "sku": "sku-1"}, + test_account="api-low", + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + p.auth.regular_session.post.side_effect = [ + requests.ConnectTimeout("simulated timeout on first send"), + ] + + p.run_safe("api_flow_no_uniqueness", p._test_flow_no_uniqueness) + + self.assertEqual(p.auth.regular_session.post.call_count, 1) + + def test_uniqueness_flow_revert_failure_escalates_severity(self): + flow = ApiBusinessFlow( + path="/api/orders/", + flow_name="purchase", + body_template={"account": "{test_account}", "sku": "sku-1"}, + revert_path="/api/orders/cleanup/", + revert_body={"account": "{test_account}", "sku": "sku-1"}, + test_account="api-low", + ) + p = _make_probe(business_flows=[flow], allow_stateful=True) + p.auth.regular_session.post.side_effect = [ + _resp(status=201), + _resp(status=201), + _resp(status=500), + ] + + p.run_safe("api_flow_no_uniqueness", p._test_flow_no_uniqueness) + + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI6-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].rollback_status, "revert_failed") + self.assertEqual(vuln[0].severity, "HIGH") + self.assertEqual( + p.auth.regular_session.post.call_args_list[0].kwargs["json"]["account"], + "api-low", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_api_access.py b/extensions/business/cybersec/red_mesh/tests/test_probes_api_access.py new file mode 100644 index 00000000..a95bc006 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_api_access.py @@ -0,0 +1,436 @@ +"""OWASP API Top 10 — Subphase 2.1 + 2.3 + 3.4. + +Tests for `ApiAccessProbes` (PT-OAPI1-01 BOLA + PT-OAPI5-01..04 BFLA). +This file lands incrementally: Subphase 2.1 adds TestApi1Bola; later +subphases append TestApi5Bfla and TestApi5BflaStateful. +""" + +from __future__ import annotations + +import json +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.probes.api_access import ( + ApiAccessProbes, +) +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiObjectEndpoint, ApiFunctionEndpoint, ApiSecurityConfig, + GrayboxTargetConfig, +) + + +def _mock_response(status=200, json_body=None, text="", + content_type="application/json"): + resp = MagicMock() + resp.status_code = status + resp.headers = {"content-type": content_type} + resp.text = text + if json_body is not None: + resp.json.return_value = json_body + if not text: + resp.text = json.dumps(json_body) + else: + resp.json.side_effect = ValueError("not json") + return resp + + +def _make_probe(*, object_endpoints=None, function_endpoints=None, + regular_username="alice", regular_session=None, + anon_session=None, allow_stateful=False): + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig( + object_endpoints=list(object_endpoints or []), + function_endpoints=list(function_endpoints or []), + )) + auth = MagicMock() + auth.regular_session = regular_session if regular_session is not None else MagicMock() + auth.official_session = MagicMock() + if anon_session is not None: + auth.make_anonymous_session = MagicMock(return_value=anon_session) + else: + # Default to a fresh MagicMock when callers don't provide one + auth.make_anonymous_session = MagicMock(return_value=MagicMock()) + safety = MagicMock() + safety.throttle = MagicMock() + safety.sanitize_error = MagicMock(side_effect=lambda s: s) + return ApiAccessProbes( + target_url="http://api.example", + auth_manager=auth, + target_config=cfg, + safety=safety, + regular_username=regular_username, + allow_stateful=allow_stateful, + ) + + +class TestApi1Bola(unittest.TestCase): + + # ── Vulnerable cases ──────────────────────────────────────────────── + + def test_owner_mismatch_emits_high(self): + """Different owner_value than authenticated user → vulnerable HIGH.""" + ep = ApiObjectEndpoint(path="/api/records/{id}/", test_ids=[42], + owner_field="owner") + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"owner": "bob", "data": "secret"}, + ) + p.run() + vuln = [f for f in p.findings if f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + f = vuln[0] + self.assertEqual(f.scenario_id, "PT-OAPI1-01") + self.assertEqual(f.severity, "HIGH") + self.assertIn("CWE-639", f.cwe) + # ATT&CK default from catalog (T1190, T1078) + self.assertEqual(set(f.attack), {"T1190", "T1078"}) + + def test_pii_field_escalates_to_critical(self): + """Leaked response with `email` / `ssn` / `password` field name → CRITICAL.""" + ep = ApiObjectEndpoint(path="/api/users/{id}/", test_ids=[7], + owner_field="username") + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"username": "bob", "email": "bob@example.com", + "credit_card_number": "4242-4242-4242-4242"}, + ) + p.run() + vuln = [f for f in p.findings if f.status == "vulnerable"] + self.assertEqual(vuln[0].severity, "CRITICAL") + pii_evidence = next((e for e in vuln[0].evidence if e.startswith("pii_fields=")), None) + self.assertIsNotNone(pii_evidence) + self.assertIn("email", pii_evidence) + + def test_tenant_mismatch_emits_vulnerable(self): + """tenant_field present in response → vulnerable even if owner matches.""" + ep = ApiObjectEndpoint( + path="/api/records/{id}/", test_ids=[1], + owner_field="owner", tenant_field="tenant_id", + expected_tenant="tenant-a", + ) + p = _make_probe(object_endpoints=[ep]) + # owner matches alice, but tenant_id leaks cross-tenant data. + p.auth.regular_session.get.return_value = _mock_response( + json_body={"owner": "alice", "tenant_id": "other-tenant", "x": 1}, + ) + p.run() + vuln = [f for f in p.findings if f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertIn("tenant_field=tenant_id", + "\n".join(vuln[0].evidence)) + + # ── Clean cases ───────────────────────────────────────────────────── + + def test_owner_matches_emits_clean(self): + ep = ApiObjectEndpoint(path="/api/records/{id}/", test_ids=[1], + owner_field="owner") + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"owner": "alice", "data": "ok"}, + ) + p.run() + clean = [f for f in p.findings if f.status == "not_vulnerable" + and f.scenario_id == "PT-OAPI1-01"] + self.assertEqual(len(clean), 1) + + # ── Inconclusive cases (FP guards) ────────────────────────────────── + + def test_html_response_skipped(self): + """HTML responses belong to AccessControlProbes (web IDOR), not API BOLA.""" + ep = ApiObjectEndpoint(path="/profile/{id}/", test_ids=[1], + owner_field="owner") + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + content_type="text/html", text="...", + ) + p.run() + # No vulnerable; one inconclusive ("no_evaluable_responses") because + # every iteration was skipped. + self.assertEqual( + [f for f in p.findings if f.status == "vulnerable"], [], + ) + inconclusive = [f for f in p.findings if f.status == "inconclusive" + and f.scenario_id == "PT-OAPI1-01"] + self.assertEqual(len(inconclusive), 1) + self.assertIn("no_evaluable_responses", + "\n".join(inconclusive[0].evidence)) + + def test_4xx_skipped(self): + """403 / 404 means the endpoint refused — that's the correct behaviour.""" + ep = ApiObjectEndpoint(path="/api/records/{id}/", test_ids=[99], + owner_field="owner") + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + status=403, json_body={"detail": "Forbidden"}, + ) + p.run() + # No vulnerable; sole finding is the rolled-up inconclusive. + statuses = [f.status for f in p.findings] + self.assertNotIn("vulnerable", statuses) + self.assertIn("inconclusive", statuses) + + def test_owner_field_missing_skipped(self): + """Configured owner_field absent from response → skip (can't compare).""" + ep = ApiObjectEndpoint(path="/api/records/{id}/", test_ids=[1], + owner_field="user_id") # not in response + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"id": 1, "data": "ok"}, # no user_id field + ) + p.run() + statuses = [f.status for f in p.findings] + self.assertNotIn("vulnerable", statuses) + + def test_no_object_endpoints_emit_inconclusive_inventory(self): + """Empty config still tells the operator API1/API5 were not evaluated.""" + p = _make_probe(object_endpoints=[]) + p.run() + ids = {f.scenario_id for f in p.findings if f.status == "inconclusive"} + self.assertIn("PT-OAPI1-01", ids) + self.assertIn("PT-OAPI5-01", ids) + + def test_no_authenticated_session_emits_inconclusive(self): + """No session at all → inconclusive (probe could not run).""" + ep = ApiObjectEndpoint(path="/api/records/{id}/", test_ids=[1], + owner_field="owner") + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session = None + p.auth.official_session = None + p.run() + f = p.findings[0] + self.assertEqual(f.status, "inconclusive") + self.assertIn("no_low_privileged_session", f.evidence[0]) + + def test_no_regular_session_does_not_fallback_to_official(self): + ep = ApiObjectEndpoint(path="/api/records/{id}/", test_ids=[1], + owner_field="owner") + p = _make_probe(object_endpoints=[ep]) + p.auth.regular_session = None + p.auth.official_session.get.return_value = _mock_response( + json_body={"owner": "bob"}, + ) + p.run() + self.assertFalse(p.auth.official_session.get.called) + f = next(f for f in p.findings if f.scenario_id == "PT-OAPI1-01") + self.assertEqual(f.status, "inconclusive") + self.assertIn("no_low_privileged_session", f.evidence[0]) + + +class TestApi5Bfla(unittest.TestCase): + """PT-OAPI5-01 + PT-OAPI5-02 — read-only BFLA (Subphase 2.3).""" + + def _make_function_probe(self, **kw): + return _make_probe(**kw) + + # ── PT-OAPI5-01 — regular user reaches admin function ────────────── + + def test_regular_2xx_on_admin_function_emits_critical(self): + """Admin path returns 200 to regular user → CRITICAL.""" + ep = ApiFunctionEndpoint(path="/api/admin/export-users/", method="GET", + privilege="admin") + p = self._make_function_probe(function_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"users": [{"id": 1}]}, + ) + p.run() + vuln = [f for f in p.findings + if f.status == "vulnerable" and f.scenario_id == "PT-OAPI5-01"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "CRITICAL") # /admin path + self.assertEqual(set(vuln[0].attack), {"T1190", "T1078"}) + + def test_regular_403_emits_clean(self): + """Auth gate working → not_vulnerable.""" + ep = ApiFunctionEndpoint(path="/api/admin/export/", method="GET", + privilege="admin") + p = self._make_function_probe(function_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + status=403, json_body={"detail": "Forbidden"}, + ) + p.run() + clean = [f for f in p.findings + if f.status == "not_vulnerable" and f.scenario_id == "PT-OAPI5-01"] + self.assertEqual(len(clean), 1) + # Marker reason is auth_gate_returned_4xx + self.assertIn("auth_gate_returned_4xx", + "\n".join(clean[0].evidence)) + + def test_auth_required_marker_in_2xx_emits_clean(self): + """If body contains the configured auth_required_marker, treat as clean.""" + ep = ApiFunctionEndpoint( + path="/api/admin/users/", method="GET", privilege="admin", + auth_required_marker="login required", + ) + p = self._make_function_probe(function_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + status=200, text="Login Required to access", + content_type="text/html", + ) + p.run() + clean = [f for f in p.findings + if f.status == "not_vulnerable" and f.scenario_id == "PT-OAPI5-01"] + self.assertEqual(len(clean), 1) + + def test_non_admin_path_baseline_high(self): + """Non-admin function path defaults to HIGH (not CRITICAL).""" + ep = ApiFunctionEndpoint(path="/api/reports/", method="GET", + privilege="user") + p = self._make_function_probe(function_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"reports": []}, + ) + p.run() + vuln = [f for f in p.findings + if f.status == "vulnerable" and f.scenario_id == "PT-OAPI5-01"] + self.assertEqual(vuln[0].severity, "HIGH") + + def test_mutating_method_skipped_in_phase_2(self): + """method=POST is deferred to PT-OAPI5-04 (Subphase 3.4).""" + ep = ApiFunctionEndpoint(path="/api/admin/promote/", method="POST", + privilege="admin") + p = self._make_function_probe(function_endpoints=[ep]) + p.auth.regular_session.post.return_value = _mock_response(json_body={}) + p.run() + # No 5-01 vulnerable; only the rolled-up inconclusive. + self.assertEqual( + [f for f in p.findings + if f.status == "vulnerable" and f.scenario_id == "PT-OAPI5-01"], + [], + ) + incon = [f for f in p.findings + if f.status == "inconclusive" and f.scenario_id == "PT-OAPI5-01"] + self.assertEqual(len(incon), 1) + + # ── PT-OAPI5-02 — anonymous reaches user function ────────────────── + + def test_anon_session_used_for_pt_oapi5_02(self): + """PT-OAPI5-02 must use make_anonymous_session, not the regular session.""" + ep = ApiFunctionEndpoint(path="/api/me/", method="GET", privilege="user") + anon = MagicMock() + anon.get.return_value = _mock_response(json_body={"id": 1}) + p = self._make_function_probe(function_endpoints=[ep], anon_session=anon) + p.run() + vuln = [f for f in p.findings + if f.status == "vulnerable" and f.scenario_id == "PT-OAPI5-02"] + self.assertEqual(len(vuln), 1) + p.auth.make_anonymous_session.assert_called() + + def test_anon_401_emits_clean(self): + """Anon hits 401 → clean.""" + ep = ApiFunctionEndpoint(path="/api/me/", method="GET") + anon = MagicMock() + anon.get.return_value = _mock_response( + status=401, json_body={"detail": "Authentication required"}, + ) + p = self._make_function_probe(function_endpoints=[ep], anon_session=anon) + p.run() + clean = [f for f in p.findings + if f.status == "not_vulnerable" and f.scenario_id == "PT-OAPI5-02"] + self.assertEqual(len(clean), 1) + + +class TestApi5BflaStateful(unittest.TestCase): + """PT-OAPI5-03 + PT-OAPI5-04 stateful BFLA attribution and rollback.""" + + def _stateful_probe(self, ep): + return _make_probe(function_endpoints=[ep], allow_stateful=True) + + def test_method_override_stateful_disabled_does_not_mutate(self): + ep = ApiFunctionEndpoint( + path="/api/admin/users/7/promote/", + method="POST", + privilege="admin", + revert_path="/api/admin/users/7/demote/", + ) + p = _make_probe(function_endpoints=[ep], allow_stateful=False) + + p.run_safe("api_bfla_method_override", p._test_bfla_method_override) + + p.auth.regular_session.post.assert_not_called() + incon = [f for f in p.findings + if f.status == "inconclusive" and f.scenario_id == "PT-OAPI5-03"] + self.assertEqual(len(incon), 1) + self.assertIn("stateful_probes_disabled", "\n".join(incon[0].evidence)) + + def test_method_override_skips_when_plain_mutating_method_allowed(self): + ep = ApiFunctionEndpoint( + path="/api/admin/users/7/promote/", + method="POST", + privilege="admin", + revert_path="/api/admin/users/7/demote/", + revert_body={"role": "user"}, + ) + p = self._stateful_probe(ep) + p.auth.regular_session.post.side_effect = [ + _mock_response(status=200), + _mock_response(status=200), + ] + + p.run_safe("api_bfla_method_override", p._test_bfla_method_override) + + self.assertEqual( + [f for f in p.findings + if f.status == "vulnerable" and f.scenario_id == "PT-OAPI5-03"], + [], + ) + incon = [f for f in p.findings + if f.status == "inconclusive" and f.scenario_id == "PT-OAPI5-03"] + self.assertEqual(len(incon), 1) + self.assertIn("plain_mutating_method_allowed", "\n".join(incon[0].evidence)) + self.assertEqual(p.auth.regular_session.post.call_count, 2) + self.assertEqual( + p.auth.regular_session.post.call_args_list[-1].args[0], + "http://api.example/api/admin/users/7/demote/", + ) + + def test_method_override_reports_only_after_plain_method_rejected(self): + ep = ApiFunctionEndpoint( + path="/api/admin/users/7/promote/", + method="POST", + privilege="admin", + revert_path="/api/admin/users/7/demote/", + ) + p = self._stateful_probe(ep) + p.auth.regular_session.post.side_effect = [ + _mock_response(status=403), + _mock_response(status=200), + _mock_response(status=200), + ] + + p.run_safe("api_bfla_method_override", p._test_bfla_method_override) + + vuln = [f for f in p.findings + if f.status == "vulnerable" and f.scenario_id == "PT-OAPI5-03"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].rollback_status, "reverted") + self.assertIn("plain_status=403", "\n".join(vuln[0].evidence)) + override_call = p.auth.regular_session.post.call_args_list[1] + self.assertEqual( + override_call.kwargs["headers"], + {"X-HTTP-Method-Override": "GET"}, + ) + + def test_mutating_bfla_revert_failure_escalates_severity(self): + ep = ApiFunctionEndpoint( + path="/api/admin/users/7/promote/", + method="POST", + privilege="admin", + revert_path="/api/admin/users/7/demote/", + ) + p = self._stateful_probe(ep) + p.auth.regular_session.post.side_effect = [ + _mock_response(status=200), + _mock_response(status=500), + ] + + p.run_safe("api_bfla_mutating", p._test_bfla_regular_as_admin_mutating) + + vuln = [f for f in p.findings + if f.status == "vulnerable" and f.scenario_id == "PT-OAPI5-04"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].rollback_status, "revert_failed") + self.assertEqual(vuln[0].severity, "CRITICAL") + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_api_auth.py b/extensions/business/cybersec/red_mesh/tests/test_probes_api_auth.py new file mode 100644 index 00000000..d35cdddc --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_api_auth.py @@ -0,0 +1,255 @@ +"""OWASP API Top 10 — Subphase 2.6 + (3.x via stateful PT-OAPI2-03). + +`ApiAuthProbes`: PT-OAPI2-01 alg=none, PT-OAPI2-02 weak HMAC, +PT-OAPI2-03 logout invalidation (stateful). +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import unittest +from unittest.mock import MagicMock, patch + +from extensions.business.cybersec.red_mesh.graybox.probes.api_auth import ( + ApiAuthProbes, _forge_jwt, +) +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiSecurityConfig, ApiTokenEndpoint, GrayboxTargetConfig, +) + + +def _hs256_jwt(payload: dict, secret: str) -> str: + return _forge_jwt({"alg": "HS256", "typ": "JWT"}, payload, secret=secret) + + +def _resp(status=200, json_body=None): + r = MagicMock() + r.status_code = status + r.headers = {} + if json_body is not None: + r.json.return_value = json_body + r.text = json.dumps(json_body) + else: + r.json.side_effect = ValueError("not json") + r.text = "" + return r + + +def _make_probe(*, token_endpoints, allow_stateful=False): + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig( + token_endpoints=token_endpoints, + )) + auth = MagicMock() + auth.official_session = MagicMock() + auth.regular_session = MagicMock() + auth.verify_tls = True + safety = MagicMock() + safety.throttle = MagicMock() + safety.sanitize_error = MagicMock(side_effect=lambda s: s) + return ApiAuthProbes( + target_url="http://api.example", + auth_manager=auth, target_config=cfg, safety=safety, + allow_stateful=allow_stateful, + ) + + +class TestApi2AlgNone(unittest.TestCase): + + @patch("extensions.business.cybersec.red_mesh.graybox.probes.api_auth.requests") + def test_protected_path_accepts_forged_alg_none_critical(self, mock_requests): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + ) + p = _make_probe(token_endpoints=tok) + real = _hs256_jwt({"sub": "alice"}, "topsecret") + p.auth.official_session.post.return_value = _resp( + json_body={"token": real}, + ) + p.auth.make_anonymous_session.return_value.get.return_value = _resp( + json_body={"id": 1, "is_admin": True}, + ) + p.run_safe("api_jwt_alg_none", p._test_jwt_alg_none) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "CRITICAL") + + @patch("extensions.business.cybersec.red_mesh.graybox.probes.api_auth.requests") + def test_protected_path_rejects_forged_clean(self, mock_requests): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + ) + p = _make_probe(token_endpoints=tok) + real = _hs256_jwt({"sub": "alice"}, "topsecret") + p.auth.official_session.post.return_value = _resp( + json_body={"token": real}, + ) + p.auth.make_anonymous_session.return_value.get.return_value = _resp(status=401) + p.run_safe("api_jwt_alg_none", p._test_jwt_alg_none) + clean = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-01" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + +class TestApi2WeakHmac(unittest.TestCase): + + def test_weak_secret_detected_high(self): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + weak_secret_candidates=["changeme", "secret", "password"], + ) + p = _make_probe(token_endpoints=tok) + real = _hs256_jwt({"sub": "alice"}, "changeme") + p.auth.official_session.post.return_value = _resp( + json_body={"token": real}, + ) + p.run_safe("api_jwt_weak_hmac", p._test_jwt_weak_hmac) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + + def test_strong_secret_clean(self): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + weak_secret_candidates=["changeme", "secret"], + ) + p = _make_probe(token_endpoints=tok) + real = _hs256_jwt({"sub": "alice"}, "a-very-long-random-secret-32bytes") + p.auth.official_session.post.return_value = _resp( + json_body={"token": real}, + ) + p.run_safe("api_jwt_weak_hmac", p._test_jwt_weak_hmac) + clean = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-02" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + +class TestApi2LogoutInvalidation(unittest.TestCase): + + @patch("extensions.business.cybersec.red_mesh.graybox.probes.api_auth.requests") + def test_stateful_disabled_inconclusive(self, mock_requests): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + logout_path="/api/auth/logout/", + ) + p = _make_probe(token_endpoints=tok, allow_stateful=False) + p.auth.official_session.post.return_value = _resp( + json_body={"token": _hs256_jwt({"sub": "alice"}, "s")}, + ) + p.run_safe("api_token_logout_invalidation", + p._test_token_logout_invalidation) + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-03" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + + def test_no_logout_path_inconclusive(self): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + logout_path="", + ) + p = _make_probe(token_endpoints=tok, allow_stateful=True) + p.run_safe("api_token_logout_invalidation", + p._test_token_logout_invalidation) + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-03" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("no_logout_path_configured", + "\n".join(incon[0].evidence)) + + def test_no_protected_path_inconclusive(self): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="", + logout_path="/api/auth/logout/", + ) + p = _make_probe(token_endpoints=tok, allow_stateful=True) + p.run_safe("api_token_logout_invalidation", + p._test_token_logout_invalidation) + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-03" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("no_protected_path_configured", + "\n".join(incon[0].evidence)) + + def test_without_token_path_does_not_logout_primary_bearer(self): + tok = ApiTokenEndpoint( + token_path="", protected_path="/api/me/", + logout_path="/api/auth/logout/", + ) + p = _make_probe(token_endpoints=tok, allow_stateful=True) + p.auth.official_session.headers = { + "Authorization": f"Bearer {_hs256_jwt({'sub': 'scanner'}, 's')}", + } + + p.run_safe("api_token_logout_invalidation", + p._test_token_logout_invalidation) + + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-03" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("disposable_logout_token_required", + "\n".join(incon[0].evidence)) + p.auth.make_anonymous_session.assert_not_called() + p.auth.official_session.post.assert_not_called() + + def test_uses_disposable_token_and_reauth_revert_when_clean(self): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + logout_path="/api/auth/logout/", + ) + p = _make_probe(token_endpoints=tok, allow_stateful=True) + first_token = _hs256_jwt({"sub": "alice", "jti": "one"}, "s") + fresh_token = _hs256_jwt({"sub": "alice", "jti": "two"}, "s") + p.auth.official_session.post.side_effect = [ + _resp(json_body={"token": first_token}), + _resp(json_body={"token": fresh_token}), + ] + anon = MagicMock() + anon.post.return_value = _resp(status=204) + anon.get.return_value = _resp(status=401) + p.auth.make_anonymous_session.return_value = anon + + p.run_safe("api_token_logout_invalidation", + p._test_token_logout_invalidation) + + clean = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-03" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + self.assertEqual(clean[0].rollback_status, "reverted") + self.assertEqual(p.auth.official_session.post.call_count, 2) + logout_headers = anon.post.call_args.kwargs["headers"] + self.assertIn(first_token, next(iter(logout_headers.values()))) + self.assertNotIn(fresh_token, next(iter(logout_headers.values()))) + + def test_uses_disposable_token_and_reauth_revert_when_vulnerable(self): + tok = ApiTokenEndpoint( + token_path="/api/token/", protected_path="/api/me/", + logout_path="/api/auth/logout/", + ) + p = _make_probe(token_endpoints=tok, allow_stateful=True) + first_token = _hs256_jwt({"sub": "alice", "jti": "one"}, "s") + fresh_token = _hs256_jwt({"sub": "alice", "jti": "two"}, "s") + p.auth.official_session.post.side_effect = [ + _resp(json_body={"token": first_token}), + _resp(json_body={"token": fresh_token}), + ] + anon = MagicMock() + anon.post.return_value = _resp(status=204) + anon.get.return_value = _resp(status=200) + p.auth.make_anonymous_session.return_value = anon + + p.run_safe("api_token_logout_invalidation", + p._test_token_logout_invalidation) + + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI2-03" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].rollback_status, "reverted") + self.assertEqual(p.auth.official_session.post.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_api_config.py b/extensions/business/cybersec/red_mesh/tests/test_probes_api_config.py new file mode 100644 index 00000000..cae87432 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_api_config.py @@ -0,0 +1,205 @@ +"""OWASP API Top 10 — Subphases 2.4 + 2.5. + +`ApiConfigProbes`: API8 misconfig (5 scenarios) + API9 inventory (3). +""" + +from __future__ import annotations + +import json +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.probes.api_config import ( + ApiConfigProbes, +) +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiFunctionEndpoint, ApiInventoryPaths, ApiSecurityConfig, + GrayboxTargetConfig, +) + + +def _resp(status=200, headers=None, json_body=None, text=""): + r = MagicMock() + r.status_code = status + r.headers = headers or {} + r.text = text or (json.dumps(json_body) if json_body is not None else "") + if json_body is not None: + r.json.return_value = json_body + else: + r.json.side_effect = ValueError("not json") + return r + + +def _make_probe(**api_cfg_kwargs): + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig(**api_cfg_kwargs)) + auth = MagicMock() + auth.regular_session = MagicMock() + auth.official_session = MagicMock() + auth.make_anonymous_session = MagicMock(return_value=MagicMock()) + safety = MagicMock() + safety.throttle = MagicMock() + safety.sanitize_error = MagicMock(side_effect=lambda s: s) + return ApiConfigProbes( + target_url="http://api.example", + auth_manager=auth, target_config=cfg, safety=safety, + ) + + +class TestApi8CorsMisconfig(unittest.TestCase): + + def test_wildcard_with_credentials_high(self): + ep = ApiFunctionEndpoint(path="/api/me/") + p = _make_probe(function_endpoints=[ep]) + p.auth.official_session.get.return_value = _resp( + headers={ + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Credentials": "true", + }, + ) + p.run_safe("api_cors_misconfig", p._test_cors_misconfig) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI8-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + + def test_origin_echo_with_credentials_high(self): + ep = ApiFunctionEndpoint(path="/api/me/") + p = _make_probe(function_endpoints=[ep]) + p.auth.official_session.get.return_value = _resp( + headers={ + "Access-Control-Allow-Origin": "https://evil.example", + "Access-Control-Allow-Credentials": "true", + }, + ) + p.run_safe("api_cors_misconfig", p._test_cors_misconfig) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI8-01" and f.status == "vulnerable"] + self.assertEqual(vuln[0].severity, "HIGH") + + def test_strict_cors_clean(self): + ep = ApiFunctionEndpoint(path="/api/me/") + p = _make_probe(function_endpoints=[ep]) + p.auth.official_session.get.return_value = _resp( + headers={"Access-Control-Allow-Origin": "https://trusted.example"}, + ) + p.run_safe("api_cors_misconfig", p._test_cors_misconfig) + clean = [f for f in p.findings + if f.scenario_id == "PT-OAPI8-01" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + +class TestApi8SecurityHeaders(unittest.TestCase): + + def test_missing_x_content_type_options_low(self): + ep = ApiFunctionEndpoint(path="/api/me/") + p = _make_probe(function_endpoints=[ep]) + p.auth.official_session.get.return_value = _resp( + headers={"Cache-Control": "no-store"}, + ) + p.run_safe("api_security_headers", p._test_security_headers) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI8-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "LOW") + + +class TestApi8DebugEndpointExposed(unittest.TestCase): + + def test_actuator_env_emits_medium(self): + p = _make_probe() + p.auth.official_session.get.return_value = _resp( + status=200, + text='{"swagger":"2.0","DEBUG":true}', + ) + p.run_safe("api_debug_endpoint", p._test_debug_endpoint_exposed) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI8-03" and f.status == "vulnerable"] + self.assertTrue(len(vuln) >= 1) + self.assertEqual(vuln[0].severity, "MEDIUM") + + +class TestApi8VerboseError(unittest.TestCase): + + def test_stack_trace_in_response_medium(self): + ep = ApiFunctionEndpoint(path="/api/me/", allow_malformed_json_probe=True) + p = _make_probe(function_endpoints=[ep]) + p.auth.official_session.post.return_value = _resp( + status=500, + text='Traceback (most recent call last):\n File "/usr/lib/python3/foo.py", line 12', + ) + p.run_safe("api_verbose_error", p._test_verbose_error) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI8-04" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + +class TestApi8UnexpectedMethods(unittest.TestCase): + + def test_trace_method_advertised_low(self): + ep = ApiFunctionEndpoint(path="/api/me/", method="GET") + p = _make_probe(function_endpoints=[ep]) + p.auth.official_session.options.return_value = _resp( + status=200, headers={"Allow": "GET, POST, TRACE, DELETE"}, + ) + p.run_safe("api_unexpected_methods", p._test_unexpected_methods) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI8-05" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + +class TestApi9OpenApiExposed(unittest.TestCase): + + def test_swagger_with_private_paths_medium(self): + inv = ApiInventoryPaths( + openapi_candidates=["/openapi.json"], + private_path_patterns=["/internal/"], + ) + p = _make_probe(inventory_paths=inv) + p.auth.make_anonymous_session.return_value.get.return_value = _resp( + json_body={ + "openapi": "3.0.0", + "paths": {"/api/v2/users/": {}, "/api/internal/admin/": {}}, + }, + ) + p.run_safe("api_openapi_exposed", p._test_openapi_exposed) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI9-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "MEDIUM") + + +class TestApi9VersionSprawl(unittest.TestCase): + + def test_legacy_v1_alive_emits_medium(self): + inv = ApiInventoryPaths( + current_version="/api/v2/", + canonical_probe_path="/api/v2/records/1/", + version_sibling_candidates=["/api/v1/"], + ) + p = _make_probe(inventory_paths=inv) + # The v2 baseline is implicit; we only probe siblings. + p.auth.official_session.get.return_value = _resp( + status=200, json_body={"id": 1}, + ) + p.run_safe("api_version_sprawl", p._test_version_sprawl) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI9-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + +class TestApi9DeprecatedLive(unittest.TestCase): + + def test_deprecated_returns_200_emits_medium(self): + inv = ApiInventoryPaths(deprecated_paths=["/api/v1/legacy/"]) + p = _make_probe(inventory_paths=inv) + p.auth.official_session.get.return_value = _resp( + status=200, json_body={"ok": True}, + ) + p.run_safe("api_deprecated_live", p._test_deprecated_live) + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI9-03" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_api_data.py b/extensions/business/cybersec/red_mesh/tests/test_probes_api_data.py new file mode 100644 index 00000000..2ab29ddf --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_api_data.py @@ -0,0 +1,168 @@ +"""OWASP API Top 10 — Subphases 2.2 + 3.1. + +Covers `ApiDataProbes`: + PT-OAPI3-01 — excessive property exposure (read-only) + PT-OAPI3-02 — mass-assignment property tampering (stateful) +""" + +from __future__ import annotations + +import json +import unittest +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.probes.api_data import ( + ApiDataProbes, +) +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiPropertyEndpoint, ApiSecurityConfig, GrayboxTargetConfig, +) + + +def _mock_response(status=200, json_body=None, + content_type="application/json"): + resp = MagicMock() + resp.status_code = status + resp.headers = {"content-type": content_type} + if json_body is not None: + resp.json.return_value = json_body + resp.text = json.dumps(json_body) + else: + resp.json.side_effect = ValueError("not json") + resp.text = "" + return resp + + +def _make_probe(*, property_endpoints=None, allow_stateful=False, + sensitive_field_patterns=None, tampering_fields=None, + regular_username="alice"): + api_cfg_kwargs = { + "property_endpoints": list(property_endpoints or []), + } + if sensitive_field_patterns is not None: + api_cfg_kwargs["sensitive_field_patterns"] = sensitive_field_patterns + if tampering_fields is not None: + api_cfg_kwargs["tampering_fields"] = tampering_fields + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig(**api_cfg_kwargs)) + auth = MagicMock() + auth.regular_session = MagicMock() + auth.official_session = MagicMock() + safety = MagicMock() + safety.throttle = MagicMock() + safety.sanitize_error = MagicMock(side_effect=lambda s: s) + return ApiDataProbes( + target_url="http://api.example", + auth_manager=auth, + target_config=cfg, + safety=safety, + regular_username=regular_username, + allow_stateful=allow_stateful, + ) + + +class TestApi3PropertyExposure(unittest.TestCase): + """PT-OAPI3-01.""" + + def test_password_hash_in_response_emits_vulnerable(self): + ep = ApiPropertyEndpoint(path="/api/profile/{id}/", test_id=1) + p = _make_probe(property_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"username": "alice", "password_hash": "$2b$12$abc"}, + ) + p.run() + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI3-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].severity, "HIGH") + leaked = next(e for e in vuln[0].evidence if e.startswith("sensitive_fields_present=")) + self.assertIn("password_hash", leaked) + + def test_clean_response_emits_not_vulnerable(self): + ep = ApiPropertyEndpoint(path="/api/profile/{id}/", test_id=1) + p = _make_probe(property_endpoints=[ep]) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"username": "alice", "display_name": "Alice"}, + ) + p.run() + clean = [f for f in p.findings + if f.scenario_id == "PT-OAPI3-01" and f.status == "not_vulnerable"] + self.assertEqual(len(clean), 1) + + def test_custom_sensitive_pattern_appended(self): + ep = ApiPropertyEndpoint(path="/api/profile/{id}/", test_id=1) + p = _make_probe( + property_endpoints=[ep], + sensitive_field_patterns=[r"internal_"], + ) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"id": 1, "internal_audit_trail": [1, 2]}, + ) + p.run() + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI3-01" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + + +class TestApi3PropertyTampering(unittest.TestCase): + """PT-OAPI3-02 — stateful.""" + + def test_stateful_disabled_emits_inconclusive(self): + ep = ApiPropertyEndpoint(path="/api/profile/{id}/", test_id=1) + p = _make_probe(property_endpoints=[ep], allow_stateful=False) + p.auth.regular_session.get.return_value = _mock_response( + json_body={"is_admin": False}, + ) + p.run() + incon = [f for f in p.findings + if f.scenario_id == "PT-OAPI3-02" and f.status == "inconclusive"] + self.assertEqual(len(incon), 1) + self.assertIn("stateful_probes_disabled", + "\n".join(incon[0].evidence)) + + def test_mass_assignment_confirmed_emits_vulnerable(self): + ep = ApiPropertyEndpoint(path="/api/profile/{id}/", test_id=1, + method_write="PATCH") + p = _make_probe(property_endpoints=[ep], allow_stateful=True, + tampering_fields=["is_admin"]) + # PT-OAPI3-01 runs first (reads the endpoint to check sensitive fields), + # then PT-OAPI3-02 baseline + verify each call session.get once. + p.auth.regular_session.get.side_effect = [ + _mock_response(json_body={"username": "alice"}), # 3-01 read (clean) + _mock_response(json_body={"is_admin": False}), # 3-02 baseline + _mock_response(json_body={"is_admin": True}), # 3-02 verify + ] + p.auth.regular_session.patch.return_value = _mock_response( + json_body={"is_admin": True} + ) + p.run() + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI3-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].rollback_status, "reverted") + self.assertEqual(vuln[0].severity, "HIGH") + + def test_mass_assignment_new_field_marks_revert_failed(self): + ep = ApiPropertyEndpoint(path="/api/profile/{id}/", test_id=1, + method_write="PATCH") + p = _make_probe(property_endpoints=[ep], allow_stateful=True, + tampering_fields=["is_admin"]) + p.auth.regular_session.get.side_effect = [ + _mock_response(json_body={"username": "alice"}), # 3-01 read + _mock_response(json_body={"username": "alice"}), # 3-02 baseline lacks is_admin + _mock_response(json_body={"username": "alice", "is_admin": True}), + ] + p.auth.regular_session.patch.return_value = _mock_response( + json_body={"is_admin": True} + ) + + p.run() + + vuln = [f for f in p.findings + if f.scenario_id == "PT-OAPI3-02" and f.status == "vulnerable"] + self.assertEqual(len(vuln), 1) + self.assertEqual(vuln[0].rollback_status, "revert_failed") + self.assertEqual(vuln[0].severity, "CRITICAL") + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py b/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py index 54d2bca4..ea13ae2a 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py +++ b/extensions/business/cybersec/red_mesh/tests/test_probes_injection.py @@ -81,6 +81,18 @@ def test_ssrf_reflected(self): self.assertEqual(vuln[0].severity, "MEDIUM") self.assertIn("CWE-918", vuln[0].cwe) + def test_ssrf_respects_runtime_assignment_gate(self): + ep = SsrfEndpoint(path="/api/fetch/", param="url") + probe = _make_probe(ssrf_endpoints=[ep]) + probe.allowed_scenario_ids = {"PT-OAPI2-01"} + + probe.run_safe_scenario("PT-API7-01", "ssrf", probe._test_ssrf) + + probe.auth.official_session.get.assert_not_called() + self.assertFalse( + any(f.scenario_id == "PT-API7-01" for f in probe.findings), + ) + def test_ssrf_no_hit(self): """Normal response → no finding.""" ep = SsrfEndpoint(path="/api/fetch/", param="url") diff --git a/extensions/business/cybersec/red_mesh/tests/test_safety.py b/extensions/business/cybersec/red_mesh/tests/test_safety.py index 8a1d0a46..af46c36a 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_safety.py +++ b/extensions/business/cybersec/red_mesh/tests/test_safety.py @@ -56,19 +56,19 @@ def test_sanitize_error_password(self): """Password values are scrubbed.""" msg = SafetyControls.sanitize_error('Error: password="secret123" is wrong') self.assertNotIn("secret123", msg) - self.assertIn("***", msg) + self.assertIn("", msg) def test_sanitize_error_token(self): """Token values are scrubbed.""" msg = SafetyControls.sanitize_error("token=abc123def in header") self.assertNotIn("abc123def", msg) - self.assertIn("***", msg) + self.assertIn("", msg) def test_sanitize_error_secret(self): """Secret values are scrubbed.""" msg = SafetyControls.sanitize_error("secret=mysecretvalue leaked") self.assertNotIn("mysecretvalue", msg) - self.assertIn("***", msg) + self.assertIn("", msg) def test_sanitize_error_preserves_normal_text(self): """Normal text without credentials is preserved.""" diff --git a/extensions/business/cybersec/red_mesh/tests/test_scenario_runtime.py b/extensions/business/cybersec/red_mesh/tests/test_scenario_runtime.py new file mode 100644 index 00000000..65ebdbb2 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_scenario_runtime.py @@ -0,0 +1,448 @@ +"""Runtime scenario manifest and assignment-gate tests.""" + +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from extensions.business.cybersec.red_mesh.constants import ( + GRAYBOX_PROBE_REGISTRY, +) +from extensions.business.cybersec.red_mesh.graybox.models import ( + DiscoveryResult, +) +from extensions.business.cybersec.red_mesh.graybox.models.target_config import ( + ApiSecurityConfig, + ApiTokenEndpoint, + AuthDescriptor, + GrayboxTargetConfig, +) +from extensions.business.cybersec.red_mesh.graybox.probes.api_auth import ( + ApiAuthProbes, + _forge_jwt, +) +from extensions.business.cybersec.red_mesh.graybox.scenario_catalog import ( + GRAYBOX_SCENARIO_CATALOG, +) +from extensions.business.cybersec.red_mesh.graybox.scenario_runtime import ( + GRAYBOX_ASSIGNMENT_MIRROR, + GRAYBOX_ASSIGNMENT_SLICE, + GRAYBOX_BUDGET_PER_SCAN, + GRAYBOX_BUDGET_PER_WORKER, + GrayboxWorkerAssignment, + build_graybox_worker_assignments, + compute_assignment_hash, + rehash_worker_assignment_dict, + runtime_scenario_ids, + runtime_scenarios, + summarize_graybox_worker_assignments, + synthesize_legacy_mirror_assignment, +) +from extensions.business.cybersec.red_mesh.graybox.worker import ( + GrayboxLocalWorker, +) + + +EXPECTED_RUNTIME_IDS = ( + "PT-OAPI1-01", + "PT-OAPI2-01", + "PT-OAPI2-02", + "PT-OAPI2-03", + "PT-OAPI3-01", + "PT-OAPI3-02", + "PT-OAPI4-01", + "PT-OAPI4-02", + "PT-OAPI4-03", + "PT-OAPI5-01", + "PT-OAPI5-02", + "PT-OAPI5-03", + "PT-OAPI5-04", + "PT-OAPI6-01", + "PT-OAPI6-02", + "PT-OAPI8-01", + "PT-OAPI8-02", + "PT-OAPI8-03", + "PT-OAPI8-04", + "PT-OAPI8-05", + "PT-OAPI9-01", + "PT-OAPI9-02", + "PT-OAPI9-03", + "PT-API7-01", +) + + +def _hs256_jwt(payload: dict, secret: str) -> str: + return _forge_jwt({"alg": "HS256", "typ": "JWT"}, payload, secret=secret) + + +def _resp(status=200, json_body=None): + r = MagicMock() + r.status_code = status + r.headers = {} + if json_body is not None: + r.json.return_value = json_body + else: + r.json.side_effect = ValueError("not json") + r.text = "" + return r + + +def _make_api_auth_probe(*, allowed_scenario_ids=None, unverified_api_auth=False): + auth_descriptor = AuthDescriptor() + if unverified_api_auth: + auth_descriptor = AuthDescriptor( + auth_type="bearer", + allow_unverified_auth=True, + ) + cfg = GrayboxTargetConfig(api_security=ApiSecurityConfig( + auth=auth_descriptor, + token_endpoints=ApiTokenEndpoint( + token_path="/api/token/", + protected_path="/api/me/", + logout_path="/api/logout/", + weak_secret_candidates=["changeme"], + ), + )) + auth = MagicMock() + auth.official_session = MagicMock() + auth.regular_session = MagicMock() + auth.verify_tls = True + auth.make_anonymous_session = MagicMock(return_value=MagicMock()) + safety = MagicMock() + safety.throttle = MagicMock() + safety.sanitize_error = MagicMock(side_effect=lambda s: s) + return ApiAuthProbes( + target_url="http://api.example", + auth_manager=auth, + target_config=cfg, + safety=safety, + allow_stateful=True, + allowed_scenario_ids=allowed_scenario_ids, + ) + + +def _make_worker(*, assigned_scenario_ids=None): + owner = MagicMock() + cfg = MagicMock() + cfg.scan_type = "webapp" + cfg.target_url = "http://testapp.local:8000" + cfg.target_config = None + cfg.verify_tls = True + cfg.scan_min_delay = 0 + cfg.allow_stateful_probes = False + cfg.app_routes = [] + cfg.excluded_features = [] + cfg.weak_candidates = [] + cfg.max_weak_attempts = 5 + cfg.official_username = "admin" + cfg.official_password = "secret" + cfg.regular_username = "" + cfg.regular_password = "" + cfg.bearer_token = "" + cfg.bearer_refresh_token = "" + cfg.api_key = "" + cfg.regular_bearer_token = "" + cfg.regular_bearer_refresh_token = "" + cfg.regular_api_key = "" + assignments, error = build_graybox_worker_assignments(["node-1"]) + if error is None: + for key, value in assignments["node-1"].items(): + setattr(cfg, key, value) + if assigned_scenario_ids is not None: + cfg.assigned_scenario_ids = list(assigned_scenario_ids) + cfg.assignment_hash = compute_assignment_hash( + strategy=cfg.graybox_assignment_strategy, + assigned_scenario_ids=cfg.assigned_scenario_ids, + assigned_request_budget=cfg.assigned_request_budget, + budget_scope=cfg.budget_scope, + assignment_revision=cfg.assignment_revision, + stateful_policy=cfg.stateful_policy, + ) + + with patch("extensions.business.cybersec.red_mesh.graybox.worker.SafetyControls"): + with patch("extensions.business.cybersec.red_mesh.graybox.worker.AuthManager"): + with patch("extensions.business.cybersec.red_mesh.graybox.worker.DiscoveryModule"): + return GrayboxLocalWorker( + owner=owner, + job_id="job-1", + target_url=cfg.target_url, + job_config=cfg, + local_id="1", + initiator="launcher", + ) + + +class TestRuntimeScenarioManifest(unittest.TestCase): + + def test_manifest_order_is_stable(self): + self.assertEqual(runtime_scenario_ids(), EXPECTED_RUNTIME_IDS) + + def test_manifest_covers_api_catalog_entries(self): + catalog_ids = { + entry["id"] + for entry in GRAYBOX_SCENARIO_CATALOG + if entry["id"].startswith("PT-OAPI") or entry["id"] == "PT-API7-01" + } + self.assertEqual(set(runtime_scenario_ids()), catalog_ids) + + def test_manifest_entries_are_unique_and_runnable(self): + ids = runtime_scenario_ids() + self.assertEqual(len(ids), len(set(ids))) + + registry = {entry["key"]: entry["cls"] for entry in GRAYBOX_PROBE_REGISTRY} + for scenario in runtime_scenarios(): + self.assertGreater(scenario.estimated_budget, 0) + self.assertIn(scenario.probe_key, registry) + cls = GrayboxLocalWorker._import_probe(registry[scenario.probe_key]) + self.assertTrue( + hasattr(cls, scenario.runner), + f"{scenario.scenario_id} runner missing: {scenario.runner}", + ) + + +class TestGrayboxWorkerAssignments(unittest.TestCase): + + def test_slice_assignments_are_disjoint_and_budgeted_per_scan(self): + assignments, error = build_graybox_worker_assignments( + ["node-a", "node-b", "node-c"], + strategy=GRAYBOX_ASSIGNMENT_SLICE, + total_request_budget=30, + ) + + self.assertIsNone(error) + assigned_sets = [ + set(assignments[node]["assigned_scenario_ids"]) + for node in ("node-a", "node-b", "node-c") + ] + for left_index, left in enumerate(assigned_sets): + for right in assigned_sets[left_index + 1:]: + self.assertFalse(left & right) + union = set().union(*assigned_sets) + self.assertEqual(union, set(runtime_scenario_ids())) + self.assertEqual( + {assignments[node]["budget_scope"] for node in assignments}, + {GRAYBOX_BUDGET_PER_SCAN}, + ) + self.assertEqual( + sum(assignments[node]["assigned_request_budget"] for node in assignments), + 30, + ) + + def test_mirror_multi_worker_default_divides_budget(self): + """B5: multi-worker MIRROR without opt-in divides the per-scan budget.""" + assignments, error = build_graybox_worker_assignments( + ["node-a", "node-b", "node-c"], + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + total_request_budget=30, + ) + + self.assertIsNone(error) + expected = list(runtime_scenario_ids()) + for assignment in assignments.values(): + self.assertEqual(assignment["assigned_scenario_ids"], expected) + self.assertEqual(assignment["budget_scope"], GRAYBOX_BUDGET_PER_SCAN) + self.assertEqual( + sum(a["assigned_request_budget"] for a in assignments.values()), + 30, + ) + + def test_mirror_multi_worker_per_worker_budget_with_explicit_opt_in(self): + assignments, error = build_graybox_worker_assignments( + ["node-a", "node-b", "node-c"], + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + total_request_budget=30, + allow_mirror_per_worker_budget=True, + ) + + self.assertIsNone(error) + for assignment in assignments.values(): + self.assertEqual(assignment["assigned_request_budget"], 30) + self.assertEqual(assignment["budget_scope"], GRAYBOX_BUDGET_PER_WORKER) + + def test_mirror_single_worker_keeps_per_worker_budget(self): + """Single-worker MIRROR is meaningfully per-worker (no traffic multiplier).""" + assignments, error = build_graybox_worker_assignments( + ["node-a"], + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + total_request_budget=30, + ) + + self.assertIsNone(error) + a = assignments["node-a"] + self.assertEqual(a["assigned_request_budget"], 30) + self.assertEqual(a["budget_scope"], GRAYBOX_BUDGET_PER_WORKER) + + def test_mirror_stateful_multi_worker_requires_override(self): + assignments, error = build_graybox_worker_assignments( + ["node-a", "node-b"], + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + allow_stateful=True, + ) + + self.assertIsNone(assignments) + self.assertIn("MIRROR with stateful", error) + + def test_synthesize_legacy_mirror_for_assignmentless_worker(self): + """B7: assignmentless worker entries get a synthesized MIRROR assignment.""" + worker_entry = { + "start_port": 443, + "end_port": 443, + "finished": False, + } + job_config = {"target_config": {"api_security": {"max_total_requests": 25}}} + compat = synthesize_legacy_mirror_assignment(job_config, worker_entry) + self.assertIsNotNone(compat) + self.assertEqual(compat["graybox_assignment_strategy"], GRAYBOX_ASSIGNMENT_MIRROR) + self.assertEqual(compat["assigned_request_budget"], 25) + self.assertEqual(compat["budget_scope"], GRAYBOX_BUDGET_PER_WORKER) + self.assertEqual(compat["assignment_compat_mode"], "legacy_mirror") + self.assertTrue(compat["assignment_hash"]) + + def test_synthesize_legacy_mirror_refuses_partial_assignment(self): + """A single new assignment field present must NOT trigger legacy compat.""" + worker_entry = { + "start_port": 443, + "end_port": 443, + "graybox_assignment_strategy": "MIRROR", # only one of the fields + } + self.assertIsNone( + synthesize_legacy_mirror_assignment({}, worker_entry), + ) + + def test_synthesize_legacy_mirror_falls_back_to_default_budget(self): + """No max_total_requests configured -> use the default budget.""" + compat = synthesize_legacy_mirror_assignment({}, {"start_port": 443, "end_port": 443}) + self.assertIsNotNone(compat) + self.assertGreater(compat["assigned_request_budget"], 0) + + def test_rehash_after_revision_bump_yields_valid_assignment(self): + """B6: bumping assignment_revision must also recompute assignment_hash.""" + assignments, error = build_graybox_worker_assignments( + ["node-a"], + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + total_request_budget=20, + ) + self.assertIsNone(error) + entry = dict(assignments["node-a"]) + original_hash = entry["assignment_hash"] + self.assertTrue(original_hash) + + entry["assignment_revision"] += 1 + rehash_worker_assignment_dict(entry) + self.assertNotEqual(entry["assignment_hash"], original_hash) + + # GrayboxWorkerAssignment.from_job_config validates by recomputing the + # hash from the same payload — the rehashed entry must round-trip. + from types import SimpleNamespace + job_config = SimpleNamespace(scan_type="webapp", **entry) + assignment = GrayboxWorkerAssignment.from_job_config(job_config) + self.assertTrue(assignment.is_valid, assignment.validation_error) + + def test_summary_aggregates_consistent_worker_assignments(self): + """B5: job-level summary surfaces strategy/budget/scope/scenarios for the dashboard.""" + assignments, error = build_graybox_worker_assignments( + ["node-a", "node-b"], + strategy=GRAYBOX_ASSIGNMENT_SLICE, + total_request_budget=30, + ) + self.assertIsNone(error) + summary = summarize_graybox_worker_assignments(assignments) + self.assertEqual(summary["graybox_assignment_strategy"], GRAYBOX_ASSIGNMENT_SLICE) + self.assertEqual(summary["budget_scope"], GRAYBOX_BUDGET_PER_SCAN) + self.assertEqual(summary["assigned_request_budget"], 30) + self.assertEqual(summary["total_assigned_scenarios"], len(runtime_scenario_ids())) + self.assertEqual(len(summary["worker_assignment_summary"]), 2) + + def test_summary_marks_mixed_when_workers_disagree(self): + """Manual edits could break the launcher contract; summary records 'mixed' for visibility.""" + assignments = { + "node-a": { + "graybox_assignment_strategy": "SLICE", + "assigned_request_budget": 15, + "budget_scope": "per_scan", + "assigned_scenario_ids": ["PT-OAPI1-01"], + }, + "node-b": { + "graybox_assignment_strategy": "MIRROR", + "assigned_request_budget": 15, + "budget_scope": "per_worker", + "assigned_scenario_ids": ["PT-OAPI1-02"], + }, + } + summary = summarize_graybox_worker_assignments(assignments) + self.assertEqual(summary["graybox_assignment_strategy"], "mixed") + self.assertEqual(summary["budget_scope"], "mixed") + + def test_invalid_request_budget_fails_assignment(self): + assignments, error = build_graybox_worker_assignments( + ["node-a"], + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + total_request_budget="abc", + ) + self.assertIsNone(assignments) + self.assertIn("positive integer", error) + + assignments, error = build_graybox_worker_assignments( + ["node-a"], + strategy=GRAYBOX_ASSIGNMENT_MIRROR, + total_request_budget=0, + ) + self.assertIsNone(assignments) + self.assertIn("positive integer", error) + + +class TestScenarioAssignmentGates(unittest.TestCase): + + def test_unassigned_api_auth_scenarios_make_zero_http_calls(self): + probe = _make_api_auth_probe(allowed_scenario_ids=("PT-OAPI2-02",)) + token = _hs256_jwt({"sub": "alice"}, "changeme") + probe.auth.official_session.post.return_value = _resp( + json_body={"token": token}, + ) + + probe.run() + + self.assertEqual({f.scenario_id for f in probe.findings}, {"PT-OAPI2-02"}) + probe.auth.make_anonymous_session.assert_not_called() + + def test_unverified_api_auth_emits_inconclusive_without_http_calls(self): + probe = _make_api_auth_probe( + allowed_scenario_ids=("PT-OAPI2-01",), + unverified_api_auth=True, + ) + + probe.run() + + self.assertEqual(len(probe.findings), 1) + finding = probe.findings[0] + self.assertEqual(finding.scenario_id, "PT-OAPI2-01") + self.assertEqual(finding.status, "inconclusive") + self.assertIn("reason=auth_unverified", finding.evidence) + probe.auth.official_session.post.assert_not_called() + probe.auth.make_anonymous_session.assert_not_called() + + def test_worker_context_carries_launcher_assignment(self): + worker = _make_worker(assigned_scenario_ids=["PT-OAPI2-02"]) + context = worker._build_probe_kwargs(DiscoveryResult()) + + self.assertEqual(context.allowed_scenario_ids, ("PT-OAPI2-02",)) + + def test_invalid_assignment_aborts_before_target_preflight(self): + worker = _make_worker() + worker.assignment = GrayboxWorkerAssignment.invalid( + "missing_assigned_scenario_ids", + ) + worker.safety.validate_target.return_value = None + worker.auth.preflight_check.return_value = None + + worker.execute_job() + + self.assertTrue(worker.state["aborted"]) + self.assertEqual(worker.state["abort_phase"], "preflight") + self.assertIn("missing_assigned_scenario_ids", worker.state["abort_reason"]) + worker.safety.validate_target.assert_not_called() + worker.auth.preflight_check.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_secret_isolation.py b/extensions/business/cybersec/red_mesh/tests/test_secret_isolation.py new file mode 100644 index 00000000..b2627bb7 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_secret_isolation.py @@ -0,0 +1,655 @@ +"""OWASP API Top 10 — Subphase 1.5 commit #10. + +Asserts that raw `bearer_token`, `api_key`, and `bearer_refresh_token` +values never appear in: + - the persisted JobConfig (R1FS public archive) + - GrayboxFinding evidence + - the finding repr() / Credentials repr() + +The R1FS-secret-payload boundary (the place where secrets are split off +from the public config before put_job_config()) is the contract we are +verifying. +""" + +from __future__ import annotations + +import json +import os +import unittest +from unittest.mock import MagicMock, patch + +from extensions.business.cybersec.red_mesh.graybox.auth_credentials import Credentials +from extensions.business.cybersec.red_mesh.graybox.models import GrayboxCredentialSet +from extensions.business.cybersec.red_mesh.services.secrets import ( + _blank_graybox_secret_fields, + build_graybox_secret_payload, + persist_job_config_with_secrets, + resolve_job_config_secrets, + R1fsSecretStore, +) + + +SENSITIVE_VALUES = { + "bearer_token": "eyJ.SECRET-BEARER-TOKEN-VALUE-1234567890.abc", + "api_key": "SUPER-SECRET-API-KEY-9999", + "bearer_refresh_token": "REFRESH-TOKEN-MUST-NOT-LEAK", + "regular_bearer_token": "eyJ.REGULAR-SECRET-BEARER-TOKEN.abc", + "regular_api_key": "REGULAR-SECRET-API-KEY-9999", + "regular_bearer_refresh_token": "REGULAR-REFRESH-TOKEN-MUST-NOT-LEAK", +} + + +def _has_secrets(text: str) -> bool: + return any(v in text for v in SENSITIVE_VALUES.values()) + + +class TestSecretIsolationInBuildPayload(unittest.TestCase): + + def test_build_payload_carries_new_secrets(self): + """The secret payload (R1FS-side) gets the new fields.""" + payload = build_graybox_secret_payload( + official_username="alice", official_password="apw", + **SENSITIVE_VALUES, + ) + self.assertEqual(payload["bearer_token"], SENSITIVE_VALUES["bearer_token"]) + self.assertEqual(payload["api_key"], SENSITIVE_VALUES["api_key"]) + self.assertEqual(payload["bearer_refresh_token"], SENSITIVE_VALUES["bearer_refresh_token"]) + self.assertEqual(payload["regular_bearer_token"], SENSITIVE_VALUES["regular_bearer_token"]) + self.assertEqual(payload["regular_api_key"], SENSITIVE_VALUES["regular_api_key"]) + self.assertEqual(payload["regular_bearer_refresh_token"], SENSITIVE_VALUES["regular_bearer_refresh_token"]) + + def test_build_payload_carries_target_config_secrets(self): + payload = build_graybox_secret_payload( + target_config_secrets={"oauth_client_secret": "OAUTH-CLIENT-SECRET"}, + ) + self.assertEqual( + payload["target_config_secrets"], + {"oauth_client_secret": "OAUTH-CLIENT-SECRET"}, + ) + + def test_blank_strips_all_new_secrets(self): + """_blank_graybox_secret_fields zeroes every new secret field.""" + sanitized = _blank_graybox_secret_fields({ + "official_username": "alice", "official_password": "apw", + **SENSITIVE_VALUES, + }) + self.assertEqual(sanitized["bearer_token"], "") + self.assertEqual(sanitized["api_key"], "") + self.assertEqual(sanitized["bearer_refresh_token"], "") + self.assertEqual(sanitized["regular_bearer_token"], "") + self.assertEqual(sanitized["regular_api_key"], "") + self.assertEqual(sanitized["regular_bearer_refresh_token"], "") + + +class TestSecretStoreKeySeparation(unittest.TestCase): + + @patch.dict(os.environ, {}, clear=True) + def test_no_dedicated_key_uses_default_with_metadata(self): + """Without any dedicated key configured, the built-in default is used and the + resulting metadata records `unsafe_key_fallback=True` for audit.""" + owner = MagicMock() + owner.P = MagicMock() + owner.cfg_redmesh_secret_store_key = "" + owner.r1fs.add_json.return_value = "fake://secret/cid" + + secret_ref = R1fsSecretStore(owner).save_graybox_credentials( + "job-1", + {"official_password": "secret"}, + ) + + self.assertEqual(secret_ref, "fake://secret/cid") + secret_doc = owner.r1fs.add_json.call_args[0][0] + self.assertTrue(secret_doc["unsafe_key_fallback"]) + self.assertEqual(secret_doc["key_id"], "redmesh:default_plugin_key") + self.assertEqual(secret_doc["key_version"], "v1") + + @patch.dict( + os.environ, + { + "REDMESH_SECRET_STORE_KEY": "dedicated-secret-store-key", + "REDMESH_SECRET_STORE_KEY_ID": "kms/redmesh/env", + "REDMESH_SECRET_STORE_KEY_VERSION": "2026-05", + }, + clear=True, + ) + def test_dedicated_env_key_records_metadata(self): + owner = MagicMock() + owner.P = MagicMock() + owner.cfg_redmesh_secret_store_key = "" + owner.r1fs.add_json.return_value = "fake://secret/cid" + + store = R1fsSecretStore(owner) + secret_ref = store.save_graybox_credentials( + "job-1", + {"official_password": "secret"}, + ) + + self.assertEqual(secret_ref, "fake://secret/cid") + secret_doc = owner.r1fs.add_json.call_args[0][0] + secret_kwargs = owner.r1fs.add_json.call_args[1] + self.assertEqual(secret_doc["key_id"], "kms/redmesh/env") + self.assertEqual(secret_doc["key_version"], "2026-05") + self.assertFalse(secret_doc["unsafe_key_fallback"]) + self.assertEqual(secret_kwargs["secret"], "dedicated-secret-store-key") + + +class TestSecretIsolationInPersistedConfig(unittest.TestCase): + + def _build_owner(self): + owner = MagicMock() + owner.P = MagicMock() + fake_store = MagicMock() + fake_store.save_graybox_credentials.return_value = "fake://secret/cid" + return owner, fake_store + + @patch("extensions.business.cybersec.red_mesh.services.secrets.R1fsSecretStore") + @patch("extensions.business.cybersec.red_mesh.services.secrets._artifact_repo") + def test_persisted_jobconfig_contains_no_raw_secrets(self, mock_repo, mock_store_cls): + """Bearer/API-key values do not appear anywhere in the archived JobConfig.""" + fake_store = MagicMock() + fake_store.save_graybox_credentials.return_value = "fake://secret/cid" + mock_store_cls.return_value = fake_store + fake_repo = MagicMock() + fake_repo.put_job_config.return_value = "fake://config/cid" + mock_repo.return_value = fake_repo + + config_dict = { + "target": "api.example.com", + "target_url": "https://api.example.com", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "official_username": "alice", "official_password": "apw", + **SENSITIVE_VALUES, + } + + owner, _ = self._build_owner() + persisted_config, _cid = persist_job_config_with_secrets( + owner, job_id="test-job-xyz", config_dict=config_dict, + ) + + serialized = json.dumps(persisted_config) + self.assertFalse( + _has_secrets(serialized), + f"Secret value leaked into persisted JobConfig: {serialized!r}", + ) + + # Non-secret capability flags ARE present. + self.assertTrue(persisted_config["has_bearer_token"]) + self.assertTrue(persisted_config["has_api_key"]) + self.assertTrue(persisted_config["has_bearer_refresh_token"]) + self.assertTrue(persisted_config["has_regular_bearer_token"]) + self.assertTrue(persisted_config["has_regular_api_key"]) + self.assertTrue(persisted_config["has_regular_bearer_refresh_token"]) + self.assertEqual(persisted_config["secret_ref"], "fake://secret/cid") + # Raw secret slots are blanked. + self.assertEqual(persisted_config["bearer_token"], "") + self.assertEqual(persisted_config["api_key"], "") + self.assertEqual(persisted_config["bearer_refresh_token"], "") + self.assertEqual(persisted_config["regular_bearer_token"], "") + self.assertEqual(persisted_config["regular_api_key"], "") + self.assertEqual(persisted_config["regular_bearer_refresh_token"], "") + + @patch("extensions.business.cybersec.red_mesh.services.secrets.R1fsSecretStore") + @patch("extensions.business.cybersec.red_mesh.services.secrets._artifact_repo") + def test_target_config_secret_ref_values_do_not_persist(self, mock_repo, mock_store_cls): + """Nested secret-ref values live only in the separate secret payload.""" + fake_store = MagicMock() + fake_store.save_graybox_credentials.return_value = "fake://secret/cid" + mock_store_cls.return_value = fake_store + fake_repo = MagicMock() + fake_repo.put_job_config.return_value = "fake://config/cid" + mock_repo.return_value = fake_repo + + config_dict = { + "target": "api.example.com", + "target_url": "https://api.example.com", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "target_config": { + "api_security": { + "token_endpoints": { + "token_request_body": { + "client_secret": {"secret_ref": "oauth_client_secret"}, + }, + }, + }, + }, + "target_config_secrets": { + "oauth_client_secret": "OAUTH-CLIENT-SECRET", + }, + } + + persisted_config, _cid = persist_job_config_with_secrets( + MagicMock(), job_id="test-job-xyz", config_dict=config_dict, + ) + + payload = fake_store.save_graybox_credentials.call_args[0][1] + self.assertEqual( + payload["target_config_secrets"]["oauth_client_secret"], + "OAUTH-CLIENT-SECRET", + ) + serialized = json.dumps(persisted_config) + self.assertNotIn("OAUTH-CLIENT-SECRET", serialized) + self.assertNotIn("target_config_secrets", persisted_config) + self.assertEqual( + persisted_config["target_config"]["api_security"]["token_endpoints"][ + "token_request_body" + ]["client_secret"], + {"secret_ref": "oauth_client_secret"}, + ) + + @patch.dict(os.environ, {}, clear=True) + def test_persist_records_dedicated_key_metadata(self): + owner = MagicMock() + owner.P = MagicMock() + owner.cfg_redmesh_secret_store_key = "dedicated-secret-store-key" + owner.cfg_redmesh_secret_store_key_id = "kms/redmesh/graybox" + owner.cfg_redmesh_secret_store_key_version = "2026-05" + owner.r1fs.add_json.side_effect = ["fake://secret/cid", "fake://config/cid"] + + persisted_config, _cid = persist_job_config_with_secrets( + owner, + job_id="test-job-xyz", + config_dict={ + "target": "api.example.com", + "target_url": "https://api.example.com", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "official_password": "apw", + }, + ) + + secret_doc = owner.r1fs.add_json.call_args_list[0][0][0] + self.assertEqual(secret_doc["key_id"], "kms/redmesh/graybox") + self.assertEqual(secret_doc["key_version"], "2026-05") + self.assertFalse(secret_doc["unsafe_key_fallback"]) + self.assertEqual(persisted_config["secret_store_key_id"], "kms/redmesh/graybox") + self.assertEqual(persisted_config["secret_store_key_version"], "2026-05") + self.assertFalse(persisted_config["secret_store_unsafe_fallback"]) + + @patch("extensions.business.cybersec.red_mesh.services.secrets.R1fsSecretStore") + def test_resolve_repopulates_secrets_for_worker(self, mock_store_cls): + """Worker-side resolve_job_config_secrets repopulates the runtime fields.""" + fake_store = MagicMock() + fake_store.load_graybox_credentials.return_value = { + "official_username": "alice", "official_password": "apw", + **SENSITIVE_VALUES, + } + mock_store_cls.return_value = fake_store + + persisted = { + "target": "api.example.com", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "secret_ref": "fake://secret/cid", + "official_username": "", "official_password": "", + "bearer_token": "", "api_key": "", "bearer_refresh_token": "", + "regular_bearer_token": "", "regular_api_key": "", + "regular_bearer_refresh_token": "", + "has_bearer_token": True, "has_api_key": True, + "has_bearer_refresh_token": True, + "has_regular_bearer_token": True, "has_regular_api_key": True, + "has_regular_bearer_refresh_token": True, + } + resolved = resolve_job_config_secrets(MagicMock(), persisted) + for k, v in SENSITIVE_VALUES.items(): + self.assertEqual(resolved[k], v) + + @patch("extensions.business.cybersec.red_mesh.services.secrets.R1fsSecretStore") + def test_resolve_target_config_secret_refs_for_worker(self, mock_store_cls): + """Worker runtime config gets body secrets without mutating persisted config.""" + fake_store = MagicMock() + fake_store.load_graybox_credentials.return_value = { + "target_config_secrets": { + "oauth_client_secret": "OAUTH-CLIENT-SECRET", + }, + } + mock_store_cls.return_value = fake_store + + persisted = { + "target": "api.example.com", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "secret_ref": "fake://secret/cid", + "target_config": { + "api_security": { + "token_endpoints": { + "token_request_body": { + "client_id": "redmesh", + "client_secret": {"secret_ref": "oauth_client_secret"}, + }, + }, + }, + }, + } + + resolved = resolve_job_config_secrets(MagicMock(), persisted) + + self.assertEqual( + resolved["target_config"]["api_security"]["token_endpoints"][ + "token_request_body" + ]["client_secret"], + "OAUTH-CLIENT-SECRET", + ) + self.assertEqual( + persisted["target_config"]["api_security"]["token_endpoints"][ + "token_request_body" + ]["client_secret"], + {"secret_ref": "oauth_client_secret"}, + ) + + @patch("extensions.business.cybersec.red_mesh.services.secrets.R1fsSecretStore") + def test_resolve_missing_target_config_secret_refs_fails_closed(self, mock_store_cls): + fake_store = MagicMock() + fake_store.load_graybox_credentials.return_value = { + "official_username": "alice", + } + mock_store_cls.return_value = fake_store + + persisted = { + "target": "api.example.com", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "secret_ref": "fake://secret/cid", + "target_config": { + "api_security": { + "token_endpoints": { + "token_request_body": { + "client_secret": {"secret_ref": "oauth_client_secret"}, + }, + }, + }, + }, + } + + with self.assertRaises(ValueError) as cm: + resolve_job_config_secrets(MagicMock(), persisted) + self.assertIn("target_config secret_ref", str(cm.exception)) + + @patch("extensions.business.cybersec.red_mesh.services.secrets.R1fsSecretStore") + def test_resolve_passes_expected_job_id_before_jobconfig_coercion(self, mock_store_cls): + """job_id is not part of JobConfig; preserve it before coercion for secret binding.""" + fake_store = MagicMock() + fake_store.load_graybox_credentials.return_value = { + "official_username": "alice", "official_password": "apw", + **SENSITIVE_VALUES, + } + mock_store_cls.return_value = fake_store + + persisted = { + "job_id": "job-A", + "target": "api.example.com", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "secret_ref": "fake://secret/cid", + } + + resolve_job_config_secrets(MagicMock(), persisted) + + fake_store.load_graybox_credentials.assert_called_once_with( + "fake://secret/cid", expected_job_id="job-A", + ) + + +class _FakeR1FSBackend: + """In-memory R1FS that mimics symmetric secret-keyed put/get. + + Mirrors the contract used by ``ArtifactRepository.put_json`` / + ``get_json``: stores payloads under a CID, only returns them if the + ``secret`` arg matches what was used at put-time. Lets us exercise the + real ``R1fsSecretStore`` end-to-end without mocking it. + """ + + def __init__(self): + self._store: dict[str, tuple[dict, str]] = {} + self._counter = 0 + + def add_json(self, payload, show_logs=False, secret=None): + self._counter += 1 + cid = f"Qm{self._counter:040d}" + self._store[cid] = (json.loads(json.dumps(payload)), secret or "") + return cid + + def get_json(self, cid, secret=None): + if cid not in self._store: + return None + payload, stored_secret = self._store[cid] + if (secret or "") != stored_secret: + return None + return json.loads(json.dumps(payload)) + + +class _FakeNode: + """Minimal stand-in for an EE plugin instance.""" + + def __init__( + self, + r1fs: _FakeR1FSBackend, + *, + cfg_redmesh_secret_store_key: str = "", + ): + self.r1fs = r1fs + self.cfg_redmesh_secret_store_key = cfg_redmesh_secret_store_key + self.cfg_redmesh_secret_store_key_id = "" + self.cfg_redmesh_secret_store_key_version = "" + self.cfg_comms_host_key = "" + self.cfg_attestation = {"ENABLED": False, "PRIVATE_KEY": ""} + self.prints: list[str] = [] + + def P(self, msg, **k): + self.prints.append(str(msg)) + + +class TestSecretRoundTripAcrossNodes(unittest.TestCase): + """Simulates launcher (rm1) → worker (rm2) using a shared R1FS backend. + + This is the scenario that broke job 2e867b02 in dev: the launcher + persisted credentials via the built-in default secret-store key and + the worker resolved them via the *same* default key on a different + plugin instance. The test pins this contract so a regression is + caught at unit-test time instead of "official_login_failed" in a + live scan. + """ + + @patch.dict(os.environ, {}, clear=True) + def test_default_key_round_trip_restores_form_credentials(self): + r1fs = _FakeR1FSBackend() + launcher = _FakeNode(r1fs) + worker = _FakeNode(r1fs) + + config_dict = { + "job_id": "job-rt-1", + "target": "honeypot.local", + "target_url": "https://honeypot.local", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "official_username": "admin", + "official_password": "P3n13st3R", + "regular_username": "user", + "regular_password": "12345678", + } + + persisted_config, config_cid = persist_job_config_with_secrets( + launcher, job_id="job-rt-1", config_dict=config_dict, + ) + + self.assertTrue(config_cid, "launcher failed to persist JobConfig") + self.assertEqual(persisted_config["official_username"], "") + self.assertEqual(persisted_config["official_password"], "") + self.assertEqual(persisted_config["regular_username"], "") + self.assertEqual(persisted_config["regular_password"], "") + self.assertTrue(persisted_config["secret_ref"]) + self.assertEqual( + persisted_config["secret_store_key_id"], "redmesh:default_plugin_key", + ) + + persisted_from_r1fs = r1fs.get_json(config_cid) + self.assertIsNotNone(persisted_from_r1fs) + persisted_from_r1fs["job_id"] = "job-rt-1" + + resolved = resolve_job_config_secrets(worker, persisted_from_r1fs) + + self.assertEqual(resolved["official_username"], "admin") + self.assertEqual(resolved["official_password"], "P3n13st3R") + self.assertEqual(resolved["regular_username"], "user") + self.assertEqual(resolved["regular_password"], "12345678") + + @patch.dict(os.environ, {}, clear=True) + def test_default_key_round_trip_handles_api_native_secrets(self): + r1fs = _FakeR1FSBackend() + launcher = _FakeNode(r1fs) + worker = _FakeNode(r1fs) + + config_dict = { + "job_id": "job-rt-2", + "target": "api.local", + "target_url": "https://api.local", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "official_username": "alice", + "official_password": "", + "bearer_token": SENSITIVE_VALUES["bearer_token"], + "api_key": SENSITIVE_VALUES["api_key"], + "regular_bearer_token": SENSITIVE_VALUES["regular_bearer_token"], + } + + persisted_config, _cid = persist_job_config_with_secrets( + launcher, job_id="job-rt-2", config_dict=config_dict, + ) + + self.assertTrue(persisted_config["has_bearer_token"]) + self.assertTrue(persisted_config["has_api_key"]) + self.assertEqual(persisted_config["bearer_token"], "") + self.assertEqual(persisted_config["api_key"], "") + + persisted_config["job_id"] = "job-rt-2" + resolved = resolve_job_config_secrets(worker, persisted_config) + + self.assertEqual(resolved["bearer_token"], SENSITIVE_VALUES["bearer_token"]) + self.assertEqual(resolved["api_key"], SENSITIVE_VALUES["api_key"]) + self.assertEqual( + resolved["regular_bearer_token"], + SENSITIVE_VALUES["regular_bearer_token"], + ) + + @patch.dict(os.environ, {}, clear=True) + def test_custom_key_on_one_node_default_on_other_fails_closed(self): + """Launcher set REDMESH_SECRET_STORE_KEY but worker did not — must fail.""" + r1fs = _FakeR1FSBackend() + launcher = _FakeNode(r1fs, cfg_redmesh_secret_store_key="operator-only-key") + worker = _FakeNode(r1fs) + + persisted_config, _cid = persist_job_config_with_secrets( + launcher, + job_id="job-rt-3", + config_dict={ + "job_id": "job-rt-3", + "target": "honeypot.local", + "target_url": "https://honeypot.local", + "start_port": 0, "end_port": 0, + "scan_type": "webapp", + "official_username": "admin", + "official_password": "P3n13st3R", + }, + ) + self.assertEqual(persisted_config["secret_store_key_source"], "config") + self.assertFalse(persisted_config["secret_store_unsafe_fallback"]) + + persisted_config["job_id"] = "job-rt-3" + with self.assertRaises(ValueError): + resolve_job_config_secrets(worker, persisted_config) + + +class TestSecretIsolationInCredentialsRepr(unittest.TestCase): + + def test_credentials_repr_never_leaks_secrets(self): + c = Credentials( + username="alice", password="formpw", + bearer_token=SENSITIVE_VALUES["bearer_token"], + api_key=SENSITIVE_VALUES["api_key"], + bearer_refresh_token=SENSITIVE_VALUES["bearer_refresh_token"], + ) + r = repr(c) + self.assertFalse( + _has_secrets(r), + f"Credentials repr leaked secrets: {r!r}", + ) + self.assertNotIn("formpw", r) + self.assertNotIn("alice", r) + # But capability booleans are visible + self.assertIn("has_bearer_token=True", r) + self.assertIn("has_api_key=True", r) + + +class TestSecretIsolationInRuntimeCredentials(unittest.TestCase): + + def test_worker_credential_set_carries_resolved_api_secrets(self): + """Resolved runtime config reaches AuthManager without persisting raw secrets.""" + cfg = MagicMock() + cfg.official_username = "" + cfg.official_password = "" + cfg.regular_username = "" + cfg.regular_password = "" + cfg.weak_candidates = [] + cfg.max_weak_attempts = 5 + cfg.bearer_token = SENSITIVE_VALUES["bearer_token"] + cfg.api_key = SENSITIVE_VALUES["api_key"] + cfg.bearer_refresh_token = SENSITIVE_VALUES["bearer_refresh_token"] + cfg.regular_bearer_token = "" + cfg.regular_api_key = "" + cfg.regular_bearer_refresh_token = "" + + creds = GrayboxCredentialSet.from_job_config(cfg) + official = creds.official.to_credentials() + + self.assertEqual(official.bearer_token, SENSITIVE_VALUES["bearer_token"]) + self.assertEqual(official.api_key, SENSITIVE_VALUES["api_key"]) + self.assertEqual(official.bearer_refresh_token, SENSITIVE_VALUES["bearer_refresh_token"]) + self.assertTrue(creds.official.is_configured) + + def test_worker_credential_set_carries_regular_api_secrets(self): + cfg = MagicMock() + cfg.official_username = "" + cfg.official_password = "" + cfg.bearer_token = "" + cfg.api_key = "" + cfg.bearer_refresh_token = "" + cfg.regular_username = "" + cfg.regular_password = "" + cfg.regular_bearer_token = SENSITIVE_VALUES["regular_bearer_token"] + cfg.regular_api_key = SENSITIVE_VALUES["regular_api_key"] + cfg.regular_bearer_refresh_token = SENSITIVE_VALUES["regular_bearer_refresh_token"] + cfg.weak_candidates = [] + cfg.max_weak_attempts = 5 + + creds = GrayboxCredentialSet.from_job_config(cfg) + + self.assertIsNotNone(creds.regular) + self.assertEqual(creds.regular.bearer_token, SENSITIVE_VALUES["regular_bearer_token"]) + self.assertEqual(creds.regular.api_key, SENSITIVE_VALUES["regular_api_key"]) + self.assertEqual(creds.regular.principal, "regular") + + def test_runtime_credential_dict_exposes_only_secret_capabilities(self): + cfg = MagicMock() + cfg.official_username = "alice" + cfg.official_password = "formpw" + cfg.regular_username = "" + cfg.regular_password = "" + cfg.weak_candidates = [] + cfg.max_weak_attempts = 5 + cfg.bearer_token = SENSITIVE_VALUES["bearer_token"] + cfg.api_key = SENSITIVE_VALUES["api_key"] + cfg.bearer_refresh_token = SENSITIVE_VALUES["bearer_refresh_token"] + cfg.regular_bearer_token = "" + cfg.regular_api_key = "" + cfg.regular_bearer_refresh_token = "" + + serialized = json.dumps(GrayboxCredentialSet.from_job_config(cfg).official.to_dict()) + + self.assertFalse(_has_secrets(serialized), serialized) + self.assertNotIn("formpw", serialized) + self.assertIn('"has_bearer_token": true', serialized) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_stateful_contract.py b/extensions/business/cybersec/red_mesh/tests/test_stateful_contract.py new file mode 100644 index 00000000..b480473b --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_stateful_contract.py @@ -0,0 +1,394 @@ +"""OWASP API Top 10 — Subphase 1.8 commits #3 + #4. + +End-to-end coverage of `ProbeBase.run_stateful` (the baseline → mutate +→ verify → revert contract) plus a lint test asserting that no probe +in the new API families bypasses `run_stateful` for direct mutating +HTTP calls. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path +from unittest.mock import MagicMock + +from extensions.business.cybersec.red_mesh.graybox.budget import RequestBudget +from extensions.business.cybersec.red_mesh.graybox.probes.base import ProbeBase +from extensions.business.cybersec.red_mesh.graybox.rollback import ( + MUTATION_ATTEMPTED_UNKNOWN, + RollbackJournalRepository, +) + + +class _StatefulProbe(ProbeBase): + def run(self): + return self.findings + + +def _make_probe(*, allow_stateful=False, rollback_journal=None): + return _StatefulProbe( + target_url="http://x", auth_manager=MagicMock(), + target_config=MagicMock(), safety=MagicMock(spec=["sanitize_error"]), + allow_stateful=allow_stateful, + rollback_journal=rollback_journal, + ) + + +class TestRunStatefulGating(unittest.TestCase): + + def test_skipped_when_stateful_disabled(self): + p = _make_probe(allow_stateful=False) + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: None, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=lambda b: True, + finding_kwargs={"title": "T", "owasp": "API3:2023"}, + ) + self.assertEqual(len(p.findings), 1) + f = p.findings[0] + self.assertEqual(f.status, "inconclusive") + self.assertIn("stateful_probes_disabled", f.evidence[0]) + + def test_skipped_when_no_revert_fn(self): + p = _make_probe(allow_stateful=True) + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: None, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=None, + finding_kwargs={"title": "T", "owasp": "API3:2023"}, + ) + self.assertEqual(p.findings[0].status, "inconclusive") + self.assertIn("no_revert_path_configured", p.findings[0].evidence[0]) + + +class TestRunStatefulHappyPath(unittest.TestCase): + + def test_vulnerable_with_successful_revert(self): + p = _make_probe(allow_stateful=True) + revert_called = [False] + + def revert(_b): + revert_called[0] = True + return True + + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: {"is_admin": False}, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=revert, + finding_kwargs={"title": "Mass assignment", "owasp": "API3:2023", + "severity": "HIGH", "cwe": ["CWE-915"]}, + ) + self.assertTrue(revert_called[0]) + f = p.findings[0] + self.assertEqual(f.status, "vulnerable") + self.assertEqual(f.severity, "HIGH") + self.assertEqual(f.rollback_status, "reverted") + + def test_inconclusive_when_verify_fails_after_mutation(self): + p = _make_probe(allow_stateful=True) + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: {"is_admin": False}, + mutate_fn=lambda b: True, + verify_fn=lambda b: False, # mutation didn't take + revert_fn=lambda b: True, + finding_kwargs={"title": "Mass assignment", "owasp": "API3:2023"}, + ) + f = p.findings[0] + self.assertEqual(f.status, "inconclusive") + self.assertIn("mutation_unverified", f.evidence[0]) + self.assertEqual(f.rollback_status, "reverted") + + def test_clean_when_verify_false_opt_in(self): + p = _make_probe(allow_stateful=True) + p.run_stateful( + "PT-OAPI2-03", + baseline_fn=lambda: {"token": "disposable"}, + mutate_fn=lambda b: True, + verify_fn=lambda b: False, + revert_fn=lambda b: True, + clean_when_verify_false=True, + finding_kwargs={ + "title": "Logout invalidation", + "owasp": "API2:2023", + "evidence": ["logout_path=/api/auth/logout/"], + }, + ) + f = p.findings[0] + self.assertEqual(f.status, "not_vulnerable") + self.assertEqual(f.rollback_status, "reverted") + self.assertIn("logout_path=/api/auth/logout/", f.evidence) + + def test_journal_record_is_pending_before_mutate_and_reverted_after(self): + journal = RollbackJournalRepository(job_id="job-1", worker_id="worker-1") + p = _make_probe(allow_stateful=True, rollback_journal=journal) + + def mutate(_b): + self.assertEqual(len(journal.records), 1) + self.assertEqual(journal.records[0]["status"], "pending") + return True + + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: {"is_admin": False}, + mutate_fn=mutate, + verify_fn=lambda b: True, + revert_fn=lambda b: True, + finding_kwargs={"title": "Mass assignment", "owasp": "API3:2023"}, + ) + + self.assertEqual(journal.records[0]["status"], "reverted") + self.assertEqual(journal.records[0]["scenario_id"], "PT-OAPI3-02") + + def test_attempted_unknown_still_reverts_and_is_inconclusive(self): + journal = RollbackJournalRepository(job_id="job-1", worker_id="worker-1") + p = _make_probe(allow_stateful=True, rollback_journal=journal) + revert_called = [False] + + def revert(_b): + revert_called[0] = True + return True + + p.run_stateful( + "PT-OAPI5-04", + baseline_fn=lambda: None, + mutate_fn=lambda b: MUTATION_ATTEMPTED_UNKNOWN, + verify_fn=lambda b: False, + revert_fn=revert, + finding_kwargs={"title": "BFLA", "owasp": "API5:2023"}, + ) + + self.assertTrue(revert_called[0]) + f = p.findings[0] + self.assertEqual(f.status, "inconclusive") + self.assertIn("mutation_attempted_unknown", f.evidence[0]) + self.assertEqual(f.rollback_status, "reverted") + self.assertEqual(journal.records[0]["status"], "reverted") + + def test_verify_attempted_unknown_must_not_become_vulnerable(self): + """B4 (PR406): MUTATION_ATTEMPTED_UNKNOWN from verify_fn is not a confirmation.""" + p = _make_probe(allow_stateful=True) + revert_called = [False] + + def revert(_b): + revert_called[0] = True + return True + + p.run_stateful( + "PT-OAPI6-01", + baseline_fn=lambda: None, + mutate_fn=lambda b: True, + verify_fn=lambda b: MUTATION_ATTEMPTED_UNKNOWN, + revert_fn=revert, + finding_kwargs={"title": "Verify uncertain", "owasp": "API6:2023"}, + ) + + self.assertTrue(revert_called[0]) + f = p.findings[0] + self.assertEqual(f.status, "inconclusive") + self.assertIn("mutation_attempted_unknown", f.evidence[0]) + self.assertEqual(f.rollback_status, "reverted") + + def test_verify_non_bool_truthy_value_does_not_become_vulnerable(self): + """Stray non-bool returns (dicts, strings) must NOT be confirmed via Python truthiness.""" + p = _make_probe(allow_stateful=True) + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: {"is_admin": False}, + mutate_fn=lambda b: True, + verify_fn=lambda b: {"changed": True}, # truthy dict, not a confirmation + revert_fn=lambda b: True, + finding_kwargs={"title": "Mass assignment", "owasp": "API3:2023"}, + ) + + f = p.findings[0] + self.assertEqual(f.status, "inconclusive") + self.assertNotEqual(f.status, "vulnerable") + self.assertEqual(f.rollback_status, "reverted") + + def test_cleanup_revert_not_blocked_by_exhausted_probe_budget(self): + p = _make_probe(allow_stateful=True) + p.request_budget = RequestBudget(remaining=0, total=0) + + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: None, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=lambda b: p.cleanup_budget(), + finding_kwargs={"title": "Mass assignment", "owasp": "API3:2023"}, + ) + + self.assertEqual(p.findings[0].status, "vulnerable") + self.assertEqual(p.findings[0].rollback_status, "reverted") + + +class TestRunStatefulRevertFailureBumpsSeverity(unittest.TestCase): + + def test_revert_failure_escalates_high_to_critical(self): + p = _make_probe(allow_stateful=True) + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: None, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=lambda b: False, # revert refused / failed + finding_kwargs={"title": "Mass assignment", "owasp": "API3:2023", + "severity": "HIGH"}, + ) + f = p.findings[0] + self.assertEqual(f.status, "vulnerable") + self.assertEqual(f.severity, "CRITICAL") + self.assertEqual(f.rollback_status, "revert_failed") + self.assertIn("Manual cleanup required", f.remediation) + + def test_revert_failure_marks_journal_manual_cleanup_required(self): + journal = RollbackJournalRepository(job_id="job-1", worker_id="worker-1") + p = _make_probe(allow_stateful=True, rollback_journal=journal) + + p.run_stateful( + "PT-OAPI5-04", + baseline_fn=lambda: None, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=lambda b: False, + finding_kwargs={"title": "BFLA", "owasp": "API5:2023"}, + ) + + self.assertEqual(journal.records[0]["status"], "manual_cleanup_required") + + def test_revert_exception_treated_as_failure(self): + p = _make_probe(allow_stateful=True) + + def revert(_b): + raise RuntimeError("revert HTTP exploded") + + p.run_stateful( + "PT-OAPI5-04", + baseline_fn=lambda: None, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=revert, + finding_kwargs={"title": "BFLA mut", "owasp": "API5:2023", + "severity": "MEDIUM"}, + ) + f = p.findings[0] + self.assertEqual(f.severity, "HIGH") # MEDIUM bumped + self.assertEqual(f.rollback_status, "revert_failed") + + +class TestRunStatefulErrorPaths(unittest.TestCase): + + def test_baseline_failure_inconclusive(self): + p = _make_probe(allow_stateful=True) + p.safety.sanitize_error = MagicMock(side_effect=lambda s: s) + + def baseline(): + raise ConnectionError("target unreachable") + + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=baseline, + mutate_fn=lambda b: True, + verify_fn=lambda b: True, + revert_fn=lambda b: True, + finding_kwargs={"title": "T", "owasp": "API3:2023"}, + ) + f = p.findings[0] + self.assertEqual(f.status, "inconclusive") + self.assertIn("baseline_failed", f.evidence[0]) + + def test_mutate_failure_inconclusive(self): + p = _make_probe(allow_stateful=True) + p.safety.sanitize_error = MagicMock(side_effect=lambda s: s) + + def mutate(_b): + raise RuntimeError("write failed") + + p.run_stateful( + "PT-OAPI3-02", + baseline_fn=lambda: None, + mutate_fn=mutate, + verify_fn=lambda b: True, + revert_fn=lambda b: True, + finding_kwargs={"title": "T", "owasp": "API3:2023"}, + ) + self.assertIn("mutate_failed", p.findings[0].evidence[0]) + + +class TestRollbackJournalRecovery(unittest.TestCase): + + def test_claim_and_replay_pending_record(self): + journal = RollbackJournalRepository(job_id="job-1", worker_id="worker-1") + record_id = journal.record_pending("PT-OAPI5-04", {"path": "/api/x/"}) + + claimed = journal.claim_pending("launcher", lease_expires_at=123) + self.assertEqual(len(claimed), 1) + self.assertEqual(claimed[0]["record_id"], record_id) + + journal.replay_claimed({record_id: lambda record: True}) + + self.assertEqual(journal.records[0]["status"], "reverted") + + +class TestStatefulContractLint(unittest.TestCase): + """Lint guard: no PT-OAPI* family probe issues a mutating HTTP call + outside of `run_stateful`. The check greps each api_* probe file for + direct ``session.post/put/patch/delete`` calls and asserts they all + appear inside a function whose source path contains ``run_stateful``. + + Skeleton probe files (Subphase 1.3) have no HTTP calls yet, so the + check is currently vacuous; it becomes meaningful once Phase 3 + stateful probe methods land. Failing this lint then requires either + routing the call through run_stateful or moving it into a non-mutating + family file. + """ + + def test_mutating_calls_in_api_probe_families_use_run_stateful(self): + """Every api_*.py file that issues mutating HTTP calls MUST also + invoke `run_stateful` somewhere — those calls belong inside + baseline/mutate/verify/revert callbacks per the Subphase 1.8 contract. + + This is a heuristic lint (not full AST analysis): it checks the + same source file co-locates both patterns. False positives are + possible if a file legitimately uses POST for non-mutating actions + AND happens not to call run_stateful — when that case arises, + revisit this lint. + """ + pkg_dir = Path(__file__).resolve().parents[1] / "graybox" / "probes" + api_files = sorted(pkg_dir.glob("api_*.py")) + self.assertTrue(api_files, "no API probe files found — check pkg layout") + + # POST is overloaded (e.g., PT-OAPI8-04 POSTs malformed JSON to + # trigger a verbose-error response — non-mutating). PATCH / PUT / + # DELETE are unambiguously state-changing in REST conventions, so + # the lint targets those only. + mut_pat = re.compile( + r"\bsession\.(put|patch|delete)\(", + re.IGNORECASE, + ) + offenders = [] + for f in api_files: + src = f.read_text() + if mut_pat.search(src) and "run_stateful" not in src: + offenders.append(f.name) + self.assertEqual( + offenders, [], + f"Files with mutating HTTP calls but no run_stateful: {offenders}", + ) + + def test_run_stateful_marker_present_on_probebase(self): + """ProbeBase advertises the lint marker so probe authors can grep + for it / future mypy plugins can key off it.""" + self.assertTrue(hasattr(ProbeBase, "STATEFUL_PROBE_LINT_MARKER")) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_target_config.py b/extensions/business/cybersec/red_mesh/tests/test_target_config.py index 7ac8bb78..ed6f3ae4 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_target_config.py +++ b/extensions/business/cybersec/red_mesh/tests/test_target_config.py @@ -13,6 +13,14 @@ InjectionConfig, BusinessLogicConfig, DiscoveryConfig, + ApiObjectEndpoint, + ApiPropertyEndpoint, + ApiFunctionEndpoint, + ApiResourceEndpoint, + ApiBusinessFlow, + ApiTokenEndpoint, + ApiInventoryPaths, + ApiSecurityConfig, COMMON_CSRF_FIELDS, ) from extensions.business.cybersec.red_mesh.constants import ( @@ -66,10 +74,11 @@ def test_from_dict_roundtrip(self): self.assertEqual(restored.discovery.scope_prefix, "/api/") self.assertEqual(restored.discovery.max_pages, 100) - def test_from_dict_ignores_unknown(self): - """Extra keys in dict don't raise.""" - cfg = GrayboxTargetConfig.from_dict({"unknown_key": "value", "nested": {"foo": 1}}) - self.assertEqual(cfg.login_path, "/auth/login/") + def test_from_dict_rejects_unknown(self): + """Extra keys are rejected instead of silently dropped.""" + with self.assertRaises(ValueError) as cm: + GrayboxTargetConfig.from_dict({"unknown_key": "value", "nested": {"foo": 1}}) + self.assertIn("unknown field", str(cm.exception)) def test_from_dict_empty(self): """Empty dict produces all defaults.""" @@ -77,6 +86,18 @@ def test_from_dict_empty(self): self.assertEqual(cfg.login_path, "/auth/login/") self.assertEqual(cfg.access_control.idor_endpoints, []) + def test_from_dict_rejects_nested_unknown(self): + """Typos inside nested typed sections are rejected.""" + with self.assertRaises(ValueError) as cm: + GrayboxTargetConfig.from_dict({ + "api_security": { + "object_endpoints": [ + {"path": "/api/records/{id}/", "typo": True}, + ], + }, + }) + self.assertIn("typo", str(cm.exception)) + class TestTypedEndpoints(unittest.TestCase): @@ -169,12 +190,40 @@ def test_registry_keys_only(self): f"Registry entry has extra keys: {entry}") def test_registry_has_expected_probes(self): - """Registry includes access_control, misconfig, injection, business_logic.""" + """Registry includes all legacy + five OWASP API Top 10 probe families.""" keys = [e["key"] for e in GRAYBOX_PROBE_REGISTRY] + # Legacy (Web Top 10) self.assertIn("_graybox_access_control", keys) self.assertIn("_graybox_misconfig", keys) self.assertIn("_graybox_injection", keys) self.assertIn("_graybox_business_logic", keys) + # OWASP API Top 10 2023 (Subphase 1.3) + self.assertIn("_graybox_api_access", keys) + self.assertIn("_graybox_api_auth", keys) + self.assertIn("_graybox_api_data", keys) + self.assertIn("_graybox_api_config", keys) + self.assertIn("_graybox_api_abuse", keys) + + def test_api_family_classes_importable(self): + """Each new API family resolves via its module-relative dotted path.""" + import importlib + api_keys = ( + "_graybox_api_access", "_graybox_api_auth", "_graybox_api_data", + "_graybox_api_config", "_graybox_api_abuse", + ) + by_key = {e["key"]: e for e in GRAYBOX_PROBE_REGISTRY} + pkg = "extensions.business.cybersec.red_mesh.graybox.probes" + for key in api_keys: + with self.subTest(key=key): + entry = by_key[key] + module_name, class_name = entry["cls"].split(".", 1) + mod = importlib.import_module(f"{pkg}.{module_name}") + cls = getattr(mod, class_name) + # ProbeBase capability flags present and probe is non-stateful by default. + self.assertTrue(cls.requires_auth) + self.assertFalse(cls.is_stateful) + # run() returns iterable (skeleton returns self.findings == []) + # Instantiation requires a context; we only verify class import here. class TestCsrfFields(unittest.TestCase): @@ -188,5 +237,269 @@ def test_common_csrf_fields(self): self.assertIn("_token", COMMON_CSRF_FIELDS) +class TestApiSecurityConfig(unittest.TestCase): + """Round-trip + defaults for the OWASP API Top 10 sub-models (Subphase 1.1).""" + + # ── ApiObjectEndpoint ────────────────────────────────────────────────── + def test_api_object_endpoint_defaults(self): + ep = ApiObjectEndpoint.from_dict({"path": "/api/records/{id}/"}) + self.assertEqual(ep.path, "/api/records/{id}/") + self.assertEqual(ep.test_ids, [1, 2]) + self.assertEqual(ep.owner_field, "owner") + self.assertEqual(ep.id_param, "id") + self.assertEqual(ep.tenant_field, "") + + def test_api_object_endpoint_full(self): + ep = ApiObjectEndpoint.from_dict({ + "path": "/api/orgs/{org}/users/{id}/", + "test_ids": [5, 7, 11], + "owner_field": "user_id", + "id_param": "uid", + "tenant_field": "org_id", + }) + self.assertEqual(ep.test_ids, [5, 7, 11]) + self.assertEqual(ep.tenant_field, "org_id") + + def test_api_object_endpoint_missing_path(self): + with self.assertRaises(KeyError): + ApiObjectEndpoint.from_dict({"test_ids": [1]}) + + # ── ApiPropertyEndpoint ──────────────────────────────────────────────── + def test_api_property_endpoint_defaults(self): + ep = ApiPropertyEndpoint.from_dict({"path": "/api/profile/{id}/"}) + self.assertEqual(ep.method_read, "GET") + self.assertEqual(ep.method_write, "PATCH") + self.assertEqual(ep.test_id, 1) + + # ── ApiFunctionEndpoint ──────────────────────────────────────────────── + def test_api_function_endpoint_defaults(self): + ep = ApiFunctionEndpoint.from_dict({"path": "/api/admin/users/"}) + self.assertEqual(ep.method, "GET") + self.assertEqual(ep.privilege, "admin") + self.assertEqual(ep.revert_path, "") + self.assertEqual(ep.revert_body, {}) + + def test_api_function_endpoint_with_revert(self): + ep = ApiFunctionEndpoint.from_dict({ + "path": "/api/admin/users/{uid}/promote/", + "method": "POST", + "revert_path": "/api/admin/users/{uid}/demote/", + "revert_body": {"reason": "test"}, + }) + self.assertEqual(ep.revert_path, "/api/admin/users/{uid}/demote/") + self.assertEqual(ep.revert_body, {"reason": "test"}) + + def test_api_function_endpoint_rejects_secret_revert_body(self): + with self.assertRaises(ValueError) as cm: + ApiFunctionEndpoint.from_dict({ + "path": "/api/admin/users/{uid}/promote/", + "method": "POST", + "revert_body": {"refresh_token": "plain-token"}, + }) + self.assertIn("secret-looking", str(cm.exception)) + + # ── ApiResourceEndpoint ──────────────────────────────────────────────── + def test_api_resource_endpoint_defaults(self): + ep = ApiResourceEndpoint.from_dict({"path": "/api/records/"}) + self.assertEqual(ep.limit_param, "limit") + self.assertEqual(ep.baseline_limit, 10) + self.assertEqual(ep.abuse_limit, 999_999) + self.assertFalse(ep.rate_limit_expected) + + # ── ApiBusinessFlow ──────────────────────────────────────────────────── + def test_api_business_flow_defaults(self): + bf = ApiBusinessFlow.from_dict({"path": "/api/auth/signup/"}) + self.assertEqual(bf.method, "POST") + self.assertEqual(bf.flow_name, "signup") + self.assertEqual(bf.body_template, {}) + self.assertEqual(bf.verify_method, "GET") + self.assertEqual(bf.revert_path, "") + self.assertEqual(bf.revert_method, "POST") + self.assertEqual(bf.revert_body, {}) + self.assertFalse(bf.allow_static_test_account_body) + + def test_api_business_flow_rejects_secret_body_template(self): + with self.assertRaises(ValueError) as cm: + ApiBusinessFlow.from_dict({ + "path": "/api/auth/signup/", + "body_template": {"username": "canary", "password": "plain-secret"}, + }) + self.assertIn("secret-looking", str(cm.exception)) + + def test_api_business_flow_accepts_redmesh_canary_placeholder(self): + bf = ApiBusinessFlow.from_dict({ + "path": "/api/auth/signup/", + "body_template": { + "username": "canary", + "password": "__redmesh_canary_password__", + }, + }) + self.assertEqual( + bf.body_template["password"], + "__redmesh_canary_password__", + ) + + # ── ApiTokenEndpoint ─────────────────────────────────────────────────── + def test_api_token_endpoint_defaults(self): + tok = ApiTokenEndpoint.from_dict({}) + self.assertEqual(tok.token_path, "") + self.assertEqual(tok.protected_path, "") + self.assertEqual(tok.logout_path, "") + # Defaults include at least the obvious weak-secret entries + self.assertIn("secret", tok.weak_secret_candidates) + self.assertIn("changeme", tok.weak_secret_candidates) + + def test_api_token_endpoint_custom_wordlist(self): + tok = ApiTokenEndpoint.from_dict({ + "token_path": "/api/token/", + "protected_path": "/api/me/", + "logout_path": "/api/auth/logout/", + "weak_secret_candidates": ["a", "b"], + }) + self.assertEqual(tok.weak_secret_candidates, ["a", "b"]) + + def test_api_token_endpoint_rejects_inline_secret_request_body(self): + with self.assertRaises(ValueError) as cm: + ApiTokenEndpoint.from_dict({ + "token_request_body": { + "client_id": "redmesh", + "client_secret": "plain-secret", + }, + }) + self.assertIn("secret-looking", str(cm.exception)) + + def test_api_token_endpoint_accepts_typed_secret_ref(self): + tok = ApiTokenEndpoint.from_dict({ + "token_request_body": { + "client_id": "redmesh", + "client_secret": {"secret_ref": "oauth-client-secret"}, + }, + }) + self.assertEqual( + tok.token_request_body["client_secret"], + {"secret_ref": "oauth-client-secret"}, + ) + + # ── ApiInventoryPaths ────────────────────────────────────────────────── + def test_api_inventory_paths_defaults(self): + inv = ApiInventoryPaths.from_dict({}) + self.assertIn("/openapi.json", inv.openapi_candidates) + self.assertIn("/swagger.json", inv.openapi_candidates) + self.assertEqual(inv.current_version, "") + self.assertEqual(inv.deprecated_paths, []) + + # ── ApiSecurityConfig wrapper ────────────────────────────────────────── + def test_api_security_config_defaults(self): + cfg = ApiSecurityConfig.from_dict({}) + self.assertEqual(cfg.object_endpoints, []) + self.assertEqual(cfg.function_endpoints, []) + self.assertEqual(cfg.business_flows, []) + # Default SSRF body fields populated + self.assertIn("url", cfg.ssrf_body_fields) + self.assertIn("webhook", cfg.ssrf_body_fields) + # Default tampering fields populated + self.assertIn("is_admin", cfg.tampering_fields) + # Default debug paths populated + self.assertIn("/api/debug", cfg.debug_path_candidates) + + def test_api_security_config_accepts_safe_secret_named_metadata(self): + cfg = ApiSecurityConfig.from_dict({ + "token_endpoints": { + "token_response_field": "access_token", + "weak_secret_candidates": ["secret", "changeme"], + }, + "auth": { + "auth_type": "api_key", + "api_key_header_name": "X-Customer-Api-Key", + }, + "sensitive_field_patterns": ["custom_*_secret"], + }) + self.assertEqual(cfg.token_endpoints.token_response_field, "access_token") + self.assertEqual(cfg.auth.api_key_header_name, "X-Customer-Api-Key") + self.assertEqual(cfg.sensitive_field_patterns, ["custom_*_secret"]) + + def test_api_security_config_full_roundtrip(self): + """Populated payload survives from_dict cleanly.""" + payload = { + "object_endpoints": [ + {"path": "/api/records/{id}/", "test_ids": [1, 2], "tenant_field": "tenant_id"}, + ], + "property_endpoints": [ + {"path": "/api/profile/{id}/", "method_write": "PUT", "test_id": 42}, + ], + "function_endpoints": [ + {"path": "/api/admin/users/{uid}/promote/", + "method": "POST", "privilege": "admin", + "revert_path": "/api/admin/users/{uid}/demote/"}, + ], + "resource_endpoints": [ + {"path": "/api/records/list/", "abuse_limit": 50000, + "rate_limit_expected": True}, + ], + "business_flows": [ + {"path": "/api/auth/signup/", "flow_name": "signup", + "body_template": {"username": "x", "email": "x@x"}, + "verify_path": "/api/auth/signup/verify/", + "verify_method": "GET", + "revert_path": "/api/auth/signup/cleanup/", + "revert_method": "DELETE", + "revert_body": {"username": "x"}, + "allow_static_test_account_body": True}, + ], + "token_endpoints": { + "token_path": "/api/token/", + "protected_path": "/api/me/", + "logout_path": "/api/auth/logout/", + }, + "inventory_paths": { + "current_version": "/api/v2/", + "canonical_probe_path": "/api/v2/records/1/", + "deprecated_paths": ["/api/v1/legacy/"], + }, + "sensitive_field_patterns": ["custom_*_secret"], + "ssrf_body_fields": ["redirect_uri"], + } + cfg = ApiSecurityConfig.from_dict(payload) + self.assertEqual(len(cfg.object_endpoints), 1) + self.assertEqual(cfg.object_endpoints[0].tenant_field, "tenant_id") + self.assertEqual(cfg.property_endpoints[0].method_write, "PUT") + self.assertEqual(cfg.function_endpoints[0].revert_path, "/api/admin/users/{uid}/demote/") + self.assertTrue(cfg.resource_endpoints[0].rate_limit_expected) + self.assertEqual(cfg.business_flows[0].body_template, {"username": "x", "email": "x@x"}) + self.assertEqual(cfg.business_flows[0].verify_path, "/api/auth/signup/verify/") + self.assertEqual(cfg.business_flows[0].verify_method, "GET") + self.assertEqual(cfg.business_flows[0].revert_path, "/api/auth/signup/cleanup/") + self.assertEqual(cfg.business_flows[0].revert_method, "DELETE") + self.assertEqual(cfg.business_flows[0].revert_body, {"username": "x"}) + self.assertTrue(cfg.business_flows[0].allow_static_test_account_body) + self.assertEqual(cfg.token_endpoints.logout_path, "/api/auth/logout/") + self.assertEqual(cfg.inventory_paths.canonical_probe_path, "/api/v2/records/1/") + self.assertEqual(cfg.sensitive_field_patterns, ["custom_*_secret"]) + # Explicit override replaces, not merges + self.assertEqual(cfg.ssrf_body_fields, ["redirect_uri"]) + + # ── GrayboxTargetConfig wiring ───────────────────────────────────────── + def test_target_config_includes_api_security_default(self): + cfg = GrayboxTargetConfig.from_dict({}) + self.assertIsInstance(cfg.api_security, ApiSecurityConfig) + self.assertEqual(cfg.api_security.object_endpoints, []) + + def test_target_config_propagates_api_security_payload(self): + cfg = GrayboxTargetConfig.from_dict({ + "api_security": { + "object_endpoints": [{"path": "/api/x/{id}/"}], + }, + }) + self.assertEqual(len(cfg.api_security.object_endpoints), 1) + self.assertEqual(cfg.api_security.object_endpoints[0].path, "/api/x/{id}/") + + def test_target_config_missing_required_path_raises(self): + """Missing required `path` should raise (mirrors IdorEndpoint contract).""" + with self.assertRaises(KeyError): + GrayboxTargetConfig.from_dict({ + "api_security": {"object_endpoints": [{"test_ids": [1]}]}, + }) + + if __name__ == '__main__': unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_worker.py b/extensions/business/cybersec/red_mesh/tests/test_worker.py index 898e6aac..a2b1c086 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_worker.py +++ b/extensions/business/cybersec/red_mesh/tests/test_worker.py @@ -13,6 +13,9 @@ GrayboxProbeDefinition, GrayboxProbeRunResult, ) +from extensions.business.cybersec.red_mesh.graybox.scenario_runtime import ( + build_graybox_worker_assignments, +) from extensions.business.cybersec.red_mesh.constants import ( ScanType, GRAYBOX_PROBE_REGISTRY, ) @@ -35,6 +38,10 @@ def _make_job_config(**overrides): cfg.excluded_features = [] cfg.scan_min_delay = 0.0 cfg.authorized = True + assignments, error = build_graybox_worker_assignments(["node-1"]) + if error is None: + for key, value in assignments["node-1"].items(): + setattr(cfg, key, value) for k, v in overrides.items(): setattr(cfg, k, v) return cfg @@ -127,6 +134,38 @@ def test_graybox_results_populated(self): self.assertIn("_test_probe", worker.state["graybox_results"]["8000"]) self.assertEqual(worker.state["web_tests_info"], {}) + def test_store_findings_redacts_configured_api_key_names(self): + worker = _make_worker(target_config={ + "api_security": { + "auth": { + "auth_type": "api_key", + "api_key_location": "query", + "api_key_query_param": "customer_key", + "api_key_header_name": "X-Customer-Api-Key", + }, + }, + }) + finding = GrayboxFinding( + scenario_id="PT-OAPI1-01", + title="API object-level authorization bypass (BOLA)", + status="vulnerable", + severity="HIGH", + owasp="API1:2023", + evidence=[ + "endpoint=https://api.example/users?customer_key=SECRET99&page=1", + "X-Customer-Api-Key: SECRET-HEADER", + ], + ) + + worker._store_findings("_graybox_api_access", [finding]) + + stored = worker.state["graybox_results"]["8000"]["_graybox_api_access"] + haystack = str(stored) + self.assertNotIn("SECRET99", haystack) + self.assertNotIn("SECRET-HEADER", haystack) + self.assertIn("customer_key=", haystack) + self.assertIn("X-Customer-Api-Key: ", haystack) + class TestStatus(unittest.TestCase): @@ -277,6 +316,58 @@ def test_supported_features_come_from_typed_probe_definitions(self): ["_graybox_alpha", "_graybox_weak_auth"], ) + def test_supported_features_include_api_top10_families(self): + """All five OWASP API Top 10 probe families dispatch via the worker.""" + features = GrayboxLocalWorker.get_supported_features() + for key in ( + "_graybox_api_access", "_graybox_api_auth", "_graybox_api_data", + "_graybox_api_config", "_graybox_api_abuse", + ): + with self.subTest(key=key): + self.assertIn(key, features) + + def test_api_family_skeletons_dispatch_cleanly(self): + """Skeleton run() returns an empty finding list on each new family. + + Confirms the worker registry can resolve each module-relative dotted + path and the class can be instantiated against a minimal context. + """ + import importlib + pkg = "extensions.business.cybersec.red_mesh.graybox.probes" + new_entries = [ + e for e in GRAYBOX_PROBE_REGISTRY + if e["key"] in { + "_graybox_api_access", "_graybox_api_auth", "_graybox_api_data", + "_graybox_api_config", "_graybox_api_abuse", + } + ] + self.assertEqual(len(new_entries), 5) + for entry in new_entries: + with self.subTest(key=entry["key"]): + module_name, class_name = entry["cls"].split(".", 1) + mod = importlib.import_module(f"{pkg}.{module_name}") + cls = getattr(mod, class_name) + auth = MagicMock() + auth.regular_session = None + safety = MagicMock() + # Skeleton instantiates with the base ProbeBase signature. + probe = cls( + target_url="http://testapp.local", + auth_manager=auth, + target_config=MagicMock(), + safety=safety, + ) + result = probe.run() + # Result must be iterable; the actual content depends on which + # subphase has wired probe methods. Subphase 1.3 acceptance was + # "dispatches cleanly without exception"; once Phase 2 lands real + # probes, MagicMock target_config produces inconclusive findings + # (which still satisfies the contract). + self.assertIsNotNone(result) + for f in result: + # Every emitted finding must at least carry a recognised status. + self.assertIn(f.status, ("vulnerable", "not_vulnerable", "inconclusive")) + def test_scenario_stats(self): """Scenario stats count findings by status.""" worker = _make_worker() diff --git a/ver.py b/ver.py index 6257bc8a..42bf1714 100644 --- a/ver.py +++ b/ver.py @@ -1,2 +1,2 @@ -__VER__ = '2.10.221' +__VER__ = '2.10.222'