diff --git a/.agents/.skills/mt5-httpapi/SKILL.md b/.agents/.skills/mt5-httpapi/SKILL.md index 6fe1e25..f37fffb 100644 --- a/.agents/.skills/mt5-httpapi/SKILL.md +++ b/.agents/.skills/mt5-httpapi/SKILL.md @@ -274,6 +274,162 @@ curl -H "Authorization: Bearer $MT5_API_TOKEN" "$MT5_API_URL/history/deals?from= Deal fields: `type` (0=buy, 1=sell), `entry` (0=opening, 1=closing), `profit` (0 for entries, realized P&L for exits). +### Backtest + +Run MT5 Strategy Tester via the API. Two-stage workflow: build the INI from a +JSON spec, then submit it together with the `.ex5` (and optional `.set`) for +async execution. Endpoints exist on every terminal but only run on a +`mode: backtest` terminal in `config.yaml` — MT5 is single-instance per +portable data dir, so a tester subprocess collides with a `mode: live` +terminal that already owns the directory and exits silently. The broker/account +in the URL determines which credentials are injected into the run's `[Common]` +section. Only one tester runs at a time per API process; extra submissions +queue. + +The expert and set file can be uploaded inline OR referenced by name from a +host-managed pool mounted at `assets/experts/*.ex5` and `assets/sets/*.set`. + +```bash +# 1. Build INI: NZDJPY M15, last 5 years, open prices only, 5 ms latency. +curl -sS -X POST -H "Authorization: Bearer $MT5_API_TOKEN" \ + -H "Content-Type: application/json" \ + $MT5_API_URL/backtest/build-ini \ + -d '{ + "symbol": "NZDJPY", + "timeframe": "M15", + "expert": "EA Studio NZDJPY M15 1615044595.ex5", + "lastYears": 5, + "modelling": "open-prices", + "latencyMs": 5, + "expertParameters": "ea studio nzdjpy m15 1615044595.set" + }' > tester.ini + +# 2. Submit. Use uploads OR host-managed asset names — here, both are host-managed. +JOB=$(curl -sS -X POST -H "Authorization: Bearer $MT5_API_TOKEN" \ + $MT5_API_URL/backtest \ + -F "ini=@tester.ini" \ + -F "expert_name=EA Studio NZDJPY M15 1615044595.ex5" \ + -F "set_name=ea studio nzdjpy m15 1615044595.set" \ + | jq -r .jobId) + +# 3. Poll. Status is queued → running → completed (or failed). +curl -H "Authorization: Bearer $MT5_API_TOKEN" $MT5_API_URL/backtest/$JOB + +# 4. Fetch the report HTML and the terminal log. +curl -H "Authorization: Bearer $MT5_API_TOKEN" $MT5_API_URL/backtest/$JOB/report -o report.htm +curl -H "Authorization: Bearer $MT5_API_TOKEN" $MT5_API_URL/backtest/$JOB/log -o run.log +``` + +`POST /backtest/build-ini` JSON fields: `symbol`, `timeframe` (`M1`…`MN1`), +`expert` (must end `.ex5`), and exactly one of `fromDate`+`toDate`, +`lastYears`, or `lastDays`. Optional: `modelling` (`every-tick` `1m-ohlc` +`open-prices` `real-ticks`), `latencyMs`, `deposit` (10000), `currency` +(`USD`), `leverage` (100, written as `1:N`), `expertParameters` (`.set`), +`reportName` (`backtest-report.htm`). + +`POST /backtest` multipart fields: `ini` (required), one of `expert` or +`expert_name`, optional `set` or `set_name`. Returns `202` with `jobId`, +`statusUrl`, `reportUrl`, `logUrl`, `pollAfterSeconds`, `queuePosition`. The +INI's `[Common]` `Login`/`Password`/`Server` are always overwritten with the +URL-selected account's credentials. Path traversal in `*_name` is rejected. + +`GET /backtest/` returns the job state. When `status: completed`, the +payload includes a `summary` parsed from the HTML (`netProfit`, `profitFactor`, +`recoveryFactor`, `expectedPayoff`, `sharpeRatio`, `maxDrawdown`, +`totalTrades`, `profitTrades`, `lossTrades`, …). Jobs left running when the +API restarts are marked `failed` on the next startup. + +### Real Backtest Runbook For Agents + +When the user asks for a real backtest run, do not stop at a built INI or a +`202 Accepted` submit response. The task is only complete after one of these is +true: + +- the job reaches `completed`, the report/log are downloaded, and the requested + summary fields are returned +- a request fails and you report the exact endpoint, HTTP status, and response + body +- the job reaches `failed` and you report the final status payload exactly + +Before submitting a backtest that references host-managed files: + +1. Verify the requested filenames exist on disk exactly as named under + `assets/experts/` and `assets/sets/`. +2. Verify `GET $MT5_API_URL/ping` returns backtest mode on the target terminal. + For a real tester run, expect `{"status":"ok","mode":"backtest"}`. +3. If the user specifies a concrete date window, prefer explicit UTC + `fromDate`/`toDate` and do not also send `lastYears` or `lastDays`. +4. If auth is needed and the repo owns the token, read it from `config/config.yaml` + instead of guessing or waiting for an env var to appear. + +Execution guidance: + +- Prefer one shell script or one tightly scoped command sequence that performs + verify -> ping -> build INI -> submit -> poll -> download artifacts. This + reduces uncertainty from partially completed attempts. +- Persist the local artifacts in a dedicated output directory: + `tester.ini`, `status.json`, `report.html` (or `.htm`), and `run.log`. +- Treat `queued` and `running` as normal intermediate states. Report the job ID + and latest status while polling. +- If no `jobId` has been captured yet, there is no confirmed backtest in + progress. Do not claim the server is still working without that evidence. +- Poll `GET /backtest/` using `pollAfterSeconds` from the submit/status + payload when available. If the user explicitly requests a cadence, follow it. +- On any non-2xx HTTP response, stop immediately and show: + endpoint, HTTP status, and raw response body. +- On `status: failed`, stop immediately and show the full final status payload. + +Completion guidance: + +- Download both `reportUrl` and `logUrl` before declaring success. +- Return the requested summary fields directly from the final status payload's + `summary` object. +- Include the local artifact paths so the user can inspect the exact report and + terminal log. + +Example agent-oriented flow: + +```bash +# 0. Verify host-managed assets exactly as named. +test -f "assets/experts/EA.ex5" +test -f "assets/sets/EA.set" + +# 1. Health check the target backtest terminal. +curl -sS -H "Authorization: Bearer $MT5_API_TOKEN" \ + "$MT5_API_URL/ping" + +# 2. Build the INI from an explicit UTC window. +curl -sS -X POST -H "Authorization: Bearer $MT5_API_TOKEN" \ + -H "Content-Type: application/json" \ + "$MT5_API_URL/backtest/build-ini" \ + -d '{ + "symbol": "GBPCAD", + "timeframe": "M15", + "expert": "EA.ex5", + "fromDate": "2021-05-11", + "toDate": "2026-05-11", + "modelling": "open-prices", + "latencyMs": 5, + "deposit": 1000, + "currency": "USD" + }' > tester.ini + +# 3. Submit and capture the job ID. +JOB=$(curl -sS -X POST -H "Authorization: Bearer $MT5_API_TOKEN" \ + "$MT5_API_URL/backtest" \ + -F "ini=@tester.ini" \ + -F "expert_name=EA.ex5" \ + -F "set_name=EA.set" | jq -r .jobId) + +# 4. Poll until completed or failed, then download artifacts. +curl -sS -H "Authorization: Bearer $MT5_API_TOKEN" \ + "$MT5_API_URL/backtest/$JOB" +curl -sS -H "Authorization: Bearer $MT5_API_TOKEN" \ + "$MT5_API_URL/backtest/$JOB/report" -o report.html +curl -sS -H "Authorization: Bearer $MT5_API_TOKEN" \ + "$MT5_API_URL/backtest/$JOB/log" -o run.log +``` + ## Position Sizing ``` diff --git a/.agents/.skills/mt5-httpapi/references/setup.md b/.agents/.skills/mt5-httpapi/references/setup.md index 710a3c5..5882ccf 100644 --- a/.agents/.skills/mt5-httpapi/references/setup.md +++ b/.agents/.skills/mt5-httpapi/references/setup.md @@ -72,6 +72,23 @@ wickworks: Per-terminal ports from `config.yaml`'s `terminals:` list stay container-internal. nginx routes `///...` to the right terminal via docker DNS, and the mt5 container's iptables DNAT forwards into the Windows VM. URL scheme: `http://localhost:8888///...`. noVNC is mainly useful for watching the install progress. +### Backtest assets (optional) + +The `/backtest` endpoint can pull experts and parameter files from a +host-managed pool instead of every request having to upload them: + +``` +assets/ + experts/ # *.ex5 files + sets/ # *.set files +``` + +`run.sh` creates these directories on first start and `docker-compose.yml` +mounts the tree into the VM as `/shared/assets:ro`. Reference them by +filename (basename only — path traversal is rejected) using the `expert_name` +and `set_name` multipart fields. Inline uploads via `expert` / `set` always +take precedence. + ## Management ```bash diff --git a/.dockerignore b/.dockerignore index db852d2..a30695a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,8 @@ CLAUDE.md data/ mt5installers/ +assets/experts/ +assets/sets/ config/config.yaml config/accounts.json config/terminals.json diff --git a/.gitignore b/.gitignore index 7b7b18b..627308f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ data/* !data/.gitkeep mt5installers/* !mt5installers/.gitkeep +# Host-managed backtest assets — referenced by /backtest via expert_name/set_name. +# Binaries are user-provided; only the layout (and any tracked docs) belong in git. +assets/experts/* +!assets/experts/.gitkeep +assets/sets/* +!assets/sets/.gitkeep config/config.yaml config/accounts.json config/terminals.json diff --git a/README.md b/README.md index 17e29e3..2fab663 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,9 @@ api_token: "paste-the-output-of-openssl-rand-hex-32-here" # VM auto-reboot every N minutes (flushes DWM/VirtIO-GPU state). 0 = disable. reboot_interval: 30 +# Default Strategy Tester timeout. POST /backtest can override per job. +backtest_timeout: "6h" + tailscale: auth_key: "" # tskey-auth-... — empty disables the tailscale sidecar login_server: "" # Headscale URL; empty = Tailscale cloud @@ -132,21 +135,32 @@ terminals: account: main port: 6542 utc_offset: "3h" + symbol_suffix: "" - broker: roboforex account: demo port: 6543 utc_offset: "3h" + symbol_suffix: "" + - broker: roboforex + account: tester + port: 6544 + utc_offset: "3h" + mode: backtest # don't auto-launch terminal64.exe; reserved for /backtest jobs + symbol_suffix: ".r" # optional explicit suffix for tester symbol remap ``` Per-field notes: - **`api_token`** — if set, every endpoint requires `Authorization: Bearer `. Empty = open. Generate with `openssl rand -hex 32`. - **`reboot_interval`** — minutes between scheduled VM reboots. `0` disables. +- **`backtest_timeout`** — default Strategy Tester timeout for `POST /backtest`. Accepts the same duration grammar as `utc_offset`: `"6h"`, `"30m"`, `"3h30m"`, `"90m"`, or a bare number interpreted as hours. Per-request form field `timeout` overrides it. - **`tailscale.auth_key`** / **`tailscale.login_server`** — see [Tailscale](#tailscale-optional). Empty `auth_key` skips the sidecar. - **`requirements`** — additional pip packages installed in the VM on every boot. - **`accounts..`** — `broker` must match the installer name (`mt5setup-.exe`) and the `broker` field in `terminals[]`. `account` must match the `account` field in `terminals[]`. - **`terminals[].port`** — container-internal port for this terminal's HTTP API. Only nginx and the mt5 container talk to it; not exposed to the host. - **`terminals[].utc_offset`** — broker server's UTC offset, used to normalize all timestamps to real UTC on the wire (see [Broker time vs real UTC](#broker-time-vs-real-utc) below). Optional — defaults to `0`. Accepts `"3h"`, `"3h30m"`, `"-2h"`, `"90m"`, or a bare number (interpreted as hours). Common values: RoboForex/FTMO `"3h"`, TeleTrade `"2h"`. +- **`terminals[].mode`** — `live` (default) or `backtest`. `live` keeps `terminal64.exe` running so the MT5 SDK stays initialized for live trading endpoints. `backtest` prepares the same portable directory but does **not** launch `terminal64.exe`, leaving the data dir free for the Strategy Tester subprocess to grab — see [Backtest](#backtest). MT5 is single-instance per portable data dir, so a backtest cannot run against a `live` terminal. +- **`terminals[].symbol_suffix`** — optional explicit symbol suffix for Strategy Tester remaps. If set, mt5-httpapi appends it when `[Tester].Symbol` does not already end with that suffix. Examples: `"p"`, `".p"`, `"-mini"`. Use `""` for no suffix. Each terminal installs to `/base/` and gets copied to `//` at startup so multiple accounts of the same broker don't step on each other. @@ -748,6 +762,152 @@ What comes back from POST/PUT/DELETE on orders and positions: `type`: 0 = buy, 1 = sell. `entry`: 0 = opening, 1 = closing. `profit` is 0 for entries, actual realized P&L for exits. +### Backtest + +Run MT5 Strategy Tester jobs over the HTTP API. Backtest endpoints are served +by every terminal, but they only **run** successfully on a terminal whose +config.yaml entry has `mode: backtest`. The reason is structural: MT5 is +single-instance per portable data directory, so if `terminal64.exe` is already +running to back the live SDK, a Strategy Tester subprocess against the same +directory exits silently with code `0` and produces no report. `mode: backtest` +skips the auto-launch and the live-mode SDK init, leaving the data dir free +for the tester. Pick the broker/account namespace whose credentials you want +injected into the run's `[Common]` section — e.g. a dedicated +`darwinex/tester` entry next to your live `darwinex/main`. + +If your broker uses suffixed symbols like `EURUSDp` or `EURUSD.p`, set +`terminals[].symbol_suffix` on that backtest terminal. If the broker uses plain +symbols, set `symbol_suffix: ""` explicitly. + +Two-stage flow: + +1. `POST /backtest/build-ini` — turns a small JSON spec into a fully formed + `tester.ini` (no credentials, no expert path resolution). Stateless helper; + you can also write the INI yourself. +2. `POST /backtest` — multipart upload of the INI plus the `.ex5` expert and + optional `.set` parameter file. Returns a `jobId`; poll for status; fetch + the HTML report and terminal log when complete. + +Only one tester runs at a time per API process (serialized by an internal lock); +additional submissions queue. + +#### Asset sources + +The expert and set file can be sent inline (preferred for ad-hoc runs) or +referenced by name from a host-managed pool: + +``` +assets/ + experts/ # *.ex5 — host-managed expert advisors (mounted read-only) + sets/ # *.set — host-managed parameter files +``` + +The `docker-compose.yml` mount `./assets:/shared/assets:ro` exposes them inside +the VM so the API can read them. Path traversal in `expert_name` / `set_name` +is rejected. + +#### `POST /backtest/build-ini` + +Body (JSON): + +| Field | Required | Notes | +| ------------------ | -------- | -------------------------------------------------- | +| `symbol` | yes | e.g. `NZDJPY` | +| `timeframe` | yes | `M1` `M5` `M15` `H1` `D1` … (21 standard values) | +| `expert` | yes | filename ending in `.ex5` | +| `fromDate`+`toDate`| one of | `YYYY-MM-DD` | +| `lastYears` | one of | integer; window ends today UTC | +| `lastDays` | one of | integer | +| `modelling` | no | `every-tick` `1m-ohlc` `open-prices` `real-ticks` | +| `latencyMs` | no | integer milliseconds → `ExecutionMode` | +| `deposit` | no | default `10000` | +| `currency` | no | default `USD` | +| `leverage` | no | default `100`, written as `1:N` | +| `expertParameters` | no | `.set` filename | +| `reportName` | no | default `backtest-report.htm` | + +Returns `text/plain` with the generated INI. + +#### `POST /backtest` + +Multipart form fields: + +| Field | Required | Notes | +| -------------- | -------- | -------------------------------------------------------------- | +| `ini` | yes | INI file (file upload) | +| `expert` | one of | `.ex5` upload | +| `expert_name` | one of | filename in `assets/experts/` | +| `set` | no | `.set` upload | +| `set_name` | no | filename in `assets/sets/` | +| `timeout` | no | Duration string override (`"30m"`, `"6h"`, `"3h30m"`). Defaults to `backtest_timeout` from `config.yaml`, then hardcoded `6h`. | + +Responds `202 Accepted` with `Retry-After` header and the queued job payload: + +```json +{ + "jobId": "b3f7…", + "status": "queued", + "broker": "darwinex", + "account": "live", + "submittedAt": "2026-05-12T10:00:00Z", + "statusUrl": "/backtest/b3f7…", + "reportUrl": "/backtest/b3f7…/report", + "logUrl": "/backtest/b3f7…/log", + "pollAfterSeconds": 60, + "queuePosition": 1 +} +``` + +`[Common]` `Login` / `Password` / `Server` in the uploaded INI are always +overwritten with the credentials from `config.yaml` for the request's +broker/account. The expert path is rewritten to `Uploaded\` and the +set file is namespaced per job to avoid collisions. + +#### `GET /backtest/` + +Status payload. `status` ∈ `queued` `running` `completed` `failed`. When +completed, includes a `summary` object parsed from the HTML report +(`netProfit`, `profitFactor`, `recoveryFactor`, `expectedPayoff`, `sharpeRatio`, +`maxDrawdown`, `totalTrades`, `profitTrades`, `lossTrades`, …). + +#### `GET /backtest//report` & `/log` + +Stream the raw HTML report and terminal log file. `404` until the job finishes. + +#### Worked example + +```bash +export URL=http://127.0.0.1:8888/darwinex/live +export TOK=changeme-mt5-httpapi-token + +# 1. Build INI for a 5-year NZDJPY M15 open-prices run with 5 ms latency. +curl -sS -X POST "$URL/backtest/build-ini" \ + -H "Authorization: Bearer $TOK" -H "Content-Type: application/json" \ + -d '{"symbol":"NZDJPY","timeframe":"M15","expert":"EA Studio NZDJPY M15 1615044595.ex5","lastYears":5,"modelling":"open-prices","latencyMs":5,"expertParameters":"ea studio nzdjpy m15 1615044595.set"}' \ + > tester.ini + +# 2. Submit using a host-managed expert + set already sitting in assets/. +JOB=$(curl -sS -X POST "$URL/backtest" \ + -H "Authorization: Bearer $TOK" \ + -F "ini=@tester.ini" \ + -F "expert_name=EA Studio NZDJPY M15 1615044595.ex5" \ + -F "set_name=ea studio nzdjpy m15 1615044595.set" \ + | jq -r .jobId) + +# 3. Poll until done. +while :; do + STATUS=$(curl -sS -H "Authorization: Bearer $TOK" "$URL/backtest/$JOB" | jq -r .status) + echo "$STATUS"; [[ "$STATUS" == completed || "$STATUS" == failed ]] && break + sleep 30 +done + +# 4. Fetch the report. +curl -sS -H "Authorization: Bearer $TOK" "$URL/backtest/$JOB/report" -o report.htm +``` + +If the API is restarted while a backtest is running, the orphaned job is marked +`failed` (`API restarted before completion`) on the next startup. + ## Examples ```bash diff --git a/assets/experts/.gitkeep b/assets/experts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/assets/sets/.gitkeep b/assets/sets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/config.yaml.example b/config/config.yaml.example index 2d47d90..9995268 100644 --- a/config/config.yaml.example +++ b/config/config.yaml.example @@ -9,6 +9,10 @@ api_token: "" # before sustained-load crashes wedge the MT5 SDK pipe. 0 = disabled. reboot_interval: 30 +# Default Strategy Tester timeout. Accepts the same duration strings as +# utc_offset ("6h", "30m", "3h30m"). POST /backtest can override per job. +backtest_timeout: "6h" + # Wickworks TA sidecar — used by POST /symbols//rates/ta. # Default URL points at the dockurr gateway IP so the Windows VM can reach # the wickworks container that shares mt5's net namespace. Override only @@ -43,12 +47,23 @@ accounts: # utc_offset: broker wall-clock offset from real UTC. # Accepts hours ("3"), duration strings ("3h30m", "-2h"), or "0". # MT5 disguises broker local time as UTC; this corrects it on the wire. +# mode: "live" (default) keeps terminal64.exe running for live SDK use. +# "backtest" prepares the portable dir but does NOT launch terminal64.exe, +# freeing the data dir so the tester can spawn it via /portable /config:. +# (MT5 is single-instance per portable data dir — a live terminal locks +# the dir and any second terminal64.exe exits silently with code 0.) +# symbol_suffix: optional broker-specific suffix appended to [Tester].Symbol +# when missing. Examples: "p", ".p", "-mini". Use "" for no suffix. terminals: - broker: ftmo account: tenkchallenge port: 5001 utc_offset: "0" + mode: live + symbol_suffix: "" - broker: roboforex account: main port: 5002 utc_offset: "3" + mode: backtest + symbol_suffix: ".r" diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 87bb6b1..7df6852 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -16,6 +16,7 @@ services: - ./data/storage:/storage - ./data/oem:/oem - ./data/shared:/shared + - ./assets:/shared/assets:ro - ./data/win.iso:/boot.iso deploy: resources: diff --git a/mt5api/backtest/__init__.py b/mt5api/backtest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mt5api/backtest/handler.py b/mt5api/backtest/handler.py new file mode 100644 index 0000000..50f077a --- /dev/null +++ b/mt5api/backtest/handler.py @@ -0,0 +1,520 @@ +"""Backtest Flask views: build-ini, run, status, report, log. + +Run flow: + 1. Validate multipart inputs (ini text + expert/.set bytes or names). + 2. Inject [Common] credentials from config.yaml. + 3. Stage files under logs/backtest-jobs//. + 4. Copy expert into MQL5\\Experts\\Uploaded\\ and set into + MQL5\\Profiles\\Tester\\, namespacing the .set with the jobId so + concurrent submissions cannot clobber each other. + 5. Write the final INI as UTF-16-LE+BOM (MT5 silently rejects [Tester] + Login under UTF-8 — verified during the prior backtester branch + work). + 6. Spawn a worker thread that holds RUN_LOCK while terminal64.exe runs. +""" +from __future__ import annotations + +import configparser +import io +import json +import os +import shutil +import subprocess +import threading +import time +import uuid + +from flask import Response, abort, jsonify, request, send_file + +from mt5api.backtest import ini_builder, jobs +from mt5api.config import ( + ACCOUNT, + BROKER, + LOG_DIR, + TERMINAL_DIR, + TERMINAL_PATH, + SYMBOL_SUFFIX, + SYMBOL_SUFFIX_CONFIGURED, + ASSETS_DIR, + BACKTEST_TIMEOUT_SECONDS, + BACKTEST_JOB_DIR, + load_yaml_config, + parse_duration_to_seconds, +) +from mt5api.logger import log + +RUN_LOCK = threading.Lock() +DIAGNOSTIC_TAIL_CHARS = 4000 + + +# ── INI builder route ─────────────────────────────────────────────── + + +def build_ini_route(): + if not request.is_json: + return jsonify({"error": "Content-Type must be application/json"}), 400 + try: + ini_text = ini_builder.build_ini(request.get_json(silent=True) or {}) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return Response(ini_text, mimetype="text/plain") + + +# ── Helpers ───────────────────────────────────────────────────────── + + +def _load_account_config(): + data = load_yaml_config() + accounts = (data.get("accounts") or {}).get(BROKER) + if not isinstance(accounts, dict): + raise ValueError(f"Broker not configured in config.yaml: {BROKER}") + creds = accounts.get(ACCOUNT) + if not isinstance(creds, dict): + raise ValueError(f"Account not configured in config.yaml: {BROKER}/{ACCOUNT}") + for field in ("login", "password", "server"): + if not creds.get(field): + raise ValueError(f"Missing account field '{field}' for {BROKER}/{ACCOUNT}") + return creds + + +def _safe_basename(name, field): + name = (name or "").strip() + if not name: + return "" + if name != os.path.basename(name) or name in ("..", "."): + raise ValueError(f"{field} must be a filename, not a path") + return name + + +def _read_submission(upload, asset_name, asset_subdir, field, *, required, required_ext): + """Resolve an expert/set input from either an upload or a host-managed name.""" + if upload is not None and upload.filename: + filename = _safe_basename(upload.filename, field) + data = upload.stream.read() + else: + filename = _safe_basename(asset_name, field) + if not filename: + if required: + raise ValueError(f"Missing backtest input: {field}") + return "", b"" + path = os.path.join(ASSETS_DIR, asset_subdir, filename) + if not os.path.isfile(path): + raise ValueError(f"{field} asset not found: {filename}") + with open(path, "rb") as handle: + data = handle.read() + if required_ext and not filename.lower().endswith(required_ext): + raise ValueError(f"{field} must be a {required_ext} file") + return filename, data + + +def _parse_ini(text): + parser = configparser.ConfigParser() + parser.optionxform = str + parser.read_string(text) + if "Tester" not in parser: + raise ValueError("INI missing [Tester] section") + return parser + + +def _override_credentials(parser, creds): + if "Common" not in parser: + parser["Common"] = {} + common = parser["Common"] + common["Login"] = str(creds["login"]) + common["Password"] = str(creds["password"]) + common["Server"] = str(creds["server"]) + + +def _ensure_report_path(parser): + tester = parser["Tester"] + raw = tester.get("Report", "").strip() + name = raw.split("\\")[-1].split("/")[-1].strip() or f"backtest-{uuid.uuid4().hex}.htm" + if not name.lower().endswith((".htm", ".html")): + name += ".htm" + tester["Report"] = f"Reports\\{name}" + tester["ReplaceReport"] = "1" + tester["ShutdownTerminal"] = "1" + return name + + +def _normalize_expert(parser, expert_filename): + base = expert_filename + if base.lower().endswith(".ex5"): + base = base[:-4] + parser["Tester"]["Expert"] = f"Uploaded\\{base}" + + +def _normalize_set(parser, set_filename): + if set_filename: + parser["Tester"]["ExpertParameters"] = set_filename + else: + parser["Tester"].pop("ExpertParameters", None) + + +def _normalize_symbol(parser): + tester = parser["Tester"] + symbol = tester.get("Symbol", "").strip() + if not symbol: + return + + suffix = SYMBOL_SUFFIX if SYMBOL_SUFFIX_CONFIGURED else "" + if not suffix: + return + + if symbol.endswith(suffix): + return + + remapped = f"{symbol}{suffix}" + tester["Symbol"] = remapped + log.info( + "backtest symbol remap broker=%s account=%s %s -> %s", + BROKER, + ACCOUNT, + symbol, + remapped, + ) + + +def _serialize_ini(parser): + buffer = io.StringIO() + parser.write(buffer, space_around_delimiters=False) + return buffer.getvalue() + + +def _write_utf16_ini(parser, path): + text = _serialize_ini(parser).replace("\n", "\r\n") + with open(path, "wb") as handle: + handle.write(b"\xff\xfe") + handle.write(text.encode("utf-16-le")) + + +def _read_text_best_effort(path): + try: + with open(path, "rb") as handle: + raw = handle.read() + except OSError: + return "" + if raw[:2] == b"\xff\xfe": + return raw.decode("utf-16-le", errors="replace") + return raw.decode("utf-8", errors="replace") + + +def _tail(text, limit=DIAGNOSTIC_TAIL_CHARS): + if not text: + return "" + return text if len(text) <= limit else text[-limit:] + + +def _tail_terminal_log(lines=20): + log_dir = os.path.join(TERMINAL_DIR, "logs") + if not os.path.isdir(log_dir): + return "" + + try: + candidates = sorted( + file_name for file_name in os.listdir(log_dir) if file_name.endswith(".log") + ) + except OSError: + return "" + + if not candidates: + return "" + + latest_path = os.path.join(log_dir, candidates[-1]) + try: + with open(latest_path, "r", encoding="utf-16-le", errors="replace") as handle: + content = handle.read() + except OSError: + return "" + + tail_lines = [line.strip() for line in content.splitlines() if line.strip()] + if not tail_lines: + return "" + return "\n".join(tail_lines[-lines:]) + + +# ── Submit route ──────────────────────────────────────────────────── + + +def run_backtest(): + if not os.path.exists(TERMINAL_PATH): + return jsonify({"error": f"Terminal not found: {TERMINAL_PATH}"}), 500 + + ini_upload = request.files.get("ini") + if ini_upload is None or not ini_upload.filename: + return jsonify({"error": "Missing form file: ini"}), 400 + + try: + ini_text = ini_upload.stream.read().decode("utf-8-sig") + except UnicodeDecodeError: + return jsonify({"error": "INI must be UTF-8 text"}), 400 + + try: + timeout_value = (request.form.get("timeout") or "").strip() + timeout_seconds = ( + parse_duration_to_seconds(timeout_value) + if timeout_value + else BACKTEST_TIMEOUT_SECONDS + ) + expert_filename, expert_bytes = _read_submission( + request.files.get("expert"), + request.form.get("expert_name", ""), + "experts", + "expert", + required=True, + required_ext=".ex5", + ) + set_filename, set_bytes = _read_submission( + request.files.get("set"), + request.form.get("set_name", ""), + "sets", + "set", + required=False, + required_ext=".set", + ) + parser = _parse_ini(ini_text) + creds = _load_account_config() + _override_credentials(parser, creds) + _normalize_symbol(parser) + _normalize_expert(parser, expert_filename) + report_name = _ensure_report_path(parser) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + + job_id = uuid.uuid4().hex + # Namespace the .set so concurrent jobs cannot clobber the same file in + # MQL5\Profiles\Tester\. + staged_set_filename = f"{job_id}__{set_filename}" if set_filename else "" + _normalize_set(parser, staged_set_filename) + + stage_dir = os.path.join(BACKTEST_JOB_DIR, job_id) + os.makedirs(stage_dir, exist_ok=True) + os.makedirs(LOG_DIR, exist_ok=True) + + staged_expert_path = os.path.join(stage_dir, expert_filename) + with open(staged_expert_path, "wb") as handle: + handle.write(expert_bytes) + staged_set_path = "" + if set_filename: + staged_set_path = os.path.join(stage_dir, staged_set_filename) + with open(staged_set_path, "wb") as handle: + handle.write(set_bytes) + + # Save the human-readable normalized INI for debugging. + debug_ini_path = os.path.join(stage_dir, "normalized.ini") + with open(debug_ini_path, "w", encoding="utf-8", newline="\n") as handle: + handle.write(_serialize_ini(parser)) + + report_path = os.path.join(TERMINAL_DIR, "Reports", report_name) + log_path = os.path.join(stage_dir, "run.log") + + job = { + "jobId": job_id, + "status": "queued", + "broker": BROKER, + "account": ACCOUNT, + "submittedAt": jobs.now_iso(), + "startedAt": None, + "finishedAt": None, + "durationSeconds": None, + "reportName": report_name, + "reportPath": report_path, + "logPath": log_path, + "debugIniPath": debug_ini_path, + "stageDir": stage_dir, + "expertFilename": expert_filename, + "setFilename": set_filename, + "stagedSetFilename": staged_set_filename, + "stagedExpertPath": staged_expert_path, + "stagedSetPath": staged_set_path, + "exitCode": None, + "error": None, + "summary": None, + "timeoutSeconds": timeout_seconds, + } + jobs.store_job(job) + + threading.Thread(target=_execute_job, args=(job_id,), daemon=True).start() + + response = jsonify(jobs.public_payload(job)) + response.status_code = 202 + response.headers["Retry-After"] = str(jobs.POLL_AFTER_SECONDS) + return response + + +# ── Worker ────────────────────────────────────────────────────────── + + +def _execute_job(job_id): + job = jobs.load_job(job_id) + if job is None: + return + + reports_dir = os.path.join(TERMINAL_DIR, "Reports") + experts_dir = os.path.join(TERMINAL_DIR, "MQL5", "Experts", "Uploaded") + sets_dir = os.path.join(TERMINAL_DIR, "MQL5", "Profiles", "Tester") + + try: + os.makedirs(reports_dir, exist_ok=True) + os.makedirs(experts_dir, exist_ok=True) + os.makedirs(sets_dir, exist_ok=True) + except OSError as exc: + jobs.update_job( + job_id, + status="failed", + error=f"Cannot prepare terminal directories: {exc}", + finishedAt=jobs.now_iso(), + ) + return + + expert_dest = os.path.join(experts_dir, job["expertFilename"]) + set_dest = ( + os.path.join(sets_dir, job["stagedSetFilename"]) + if job["stagedSetFilename"] + else "" + ) + if os.path.exists(job["reportPath"]): + try: + os.remove(job["reportPath"]) + except OSError: + pass + + started_at = jobs.now_iso() + start_time = time.time() + with RUN_LOCK: + jobs.update_job(job_id, status="running", startedAt=started_at) + try: + shutil.copyfile(job["stagedExpertPath"], expert_dest) + if set_dest: + shutil.copyfile(job["stagedSetPath"], set_dest) + parser = _parse_ini(_read_text_best_effort(job["debugIniPath"])) + ini_path = os.path.join(job["stageDir"], "tester.ini") + _write_utf16_ini(parser, ini_path) + + cmd = [TERMINAL_PATH, "/portable", f"/config:{ini_path}"] + log.info( + "backtest start broker=%s account=%s job=%s report=%s", + BROKER, ACCOUNT, job_id, job["reportName"], + ) + with open(job["logPath"], "w", encoding="utf-8") as log_handle: + try: + result = subprocess.run( + cmd, + cwd=TERMINAL_DIR, + stdout=log_handle, + stderr=subprocess.STDOUT, + timeout=job["timeoutSeconds"], + check=False, + ) + except subprocess.TimeoutExpired: + duration = round(time.time() - start_time, 3) + jobs.update_job( + job_id, + status="failed", + error=f"Backtest timed out after {job['timeoutSeconds']}s", + durationSeconds=duration, + finishedAt=jobs.now_iso(), + ) + return + except Exception as exc: + log.exception("backtest crashed broker=%s account=%s job=%s", BROKER, ACCOUNT, job_id) + jobs.update_job( + job_id, + status="failed", + error=f"Backtest crashed: {exc}", + durationSeconds=round(time.time() - start_time, 3), + finishedAt=jobs.now_iso(), + ) + return + finally: + # Clean up the per-job set file copy so MQL5\Profiles\Tester\ does + # not accumulate junk over time. Errors are non-fatal. + if set_dest and os.path.exists(set_dest): + try: + os.remove(set_dest) + except OSError: + pass + + duration = round(time.time() - start_time, 3) + if result.returncode != 0: + terminal_tail = _tail(_tail_terminal_log()) + error = f"terminal64.exe exited with code {result.returncode}" + if terminal_tail: + error = f"{error} | terminal log tail: {terminal_tail}" + jobs.update_job( + job_id, + status="failed", + error=error, + exitCode=result.returncode, + durationSeconds=duration, + finishedAt=jobs.now_iso(), + ) + return + + if not os.path.exists(job["reportPath"]): + jobs.update_job( + job_id, + status="failed", + error="Report not generated", + exitCode=result.returncode, + durationSeconds=duration, + finishedAt=jobs.now_iso(), + ) + return + + report_html = _read_text_best_effort(job["reportPath"]) + summary = jobs.parse_report_summary(report_html) + if jobs.is_empty_backtest_summary(summary): + terminal_tail = _tail(_tail_terminal_log()) + error = "Backtest produced empty report (Bars=0, Ticks=0, Symbols=0)" + if terminal_tail: + error = f"{error} | terminal log tail: {terminal_tail}" + jobs.update_job( + job_id, + status="failed", + error=error, + exitCode=result.returncode, + durationSeconds=duration, + finishedAt=jobs.now_iso(), + summary=summary, + ) + return + jobs.update_job( + job_id, + status="completed", + exitCode=result.returncode, + durationSeconds=duration, + finishedAt=jobs.now_iso(), + summary=summary, + ) + log.info("backtest done broker=%s account=%s job=%s duration=%.1fs", BROKER, ACCOUNT, job_id, duration) + + +# ── Status / artifacts ────────────────────────────────────────────── + + +def get_status(job_id): + job = jobs.load_job(job_id) + if job is None: + return jsonify({"error": f"Backtest job not found: {job_id}"}), 404 + return jsonify(jobs.public_payload(job)) + + +def get_report(job_id): + job = jobs.load_job(job_id) + if job is None: + return jsonify({"error": f"Backtest job not found: {job_id}"}), 404 + path = job.get("reportPath") + if not path or not os.path.exists(path): + return jsonify({"error": "Report not available yet"}), 404 + return send_file(path, mimetype="text/html", as_attachment=False, download_name=job["reportName"]) + + +def get_log(job_id): + job = jobs.load_job(job_id) + if job is None: + return jsonify({"error": f"Backtest job not found: {job_id}"}), 404 + path = job.get("logPath") + if not path or not os.path.exists(path): + return jsonify({"error": "Log not available yet"}), 404 + return send_file(path, mimetype="text/plain", as_attachment=False, download_name=f"{job_id}.log") diff --git a/mt5api/backtest/ini_builder.py b/mt5api/backtest/ini_builder.py new file mode 100644 index 0000000..83492af --- /dev/null +++ b/mt5api/backtest/ini_builder.py @@ -0,0 +1,234 @@ +"""Build an MT5 Strategy Tester INI from validated JSON parameters. + +Pure-Python, no MT5 SDK dependency. The runner (handler.py) is responsible +for injecting [Common] credentials from config.yaml; the builder leaves +[Common] empty so this module stays callable in isolation and from tests. + +The INI text returned here is the canonical UTF-8 form. The runner +re-encodes it as UTF-16-LE with BOM before MT5 reads it, because MT5's +Strategy Tester silently rejects [Tester] Login under UTF-8. +""" +from __future__ import annotations + +import io +from configparser import ConfigParser +from datetime import date, datetime, timedelta, timezone + +# MT5 Strategy Tester modelling modes. +# 0 = Every tick +# 1 = 1 minute OHLC +# 2 = Open prices only +# 4 = Every tick based on real ticks +MODELLING_MAP = { + "every-tick": 0, + "1m-ohlc": 1, + "open-prices": 2, + "real-ticks": 4, +} + +VALID_TIMEFRAMES = ( + "M1", "M2", "M3", "M4", "M5", "M6", "M10", "M12", "M15", "M20", + "M30", "H1", "H2", "H3", "H4", "H6", "H8", "H12", "D1", "W1", "MN1", +) + +# MT5 reads tester dates as "YYYY.MM.DD". +_DATE_FMT_OUT = "%Y.%m.%d" +_DATE_FMT_IN = "%Y-%m-%d" + + +def _today_utc() -> date: + return datetime.now(timezone.utc).date() + + +def _parse_iso_date(value, field): + if isinstance(value, date): + return value + if not isinstance(value, str): + raise ValueError(f"{field} must be a YYYY-MM-DD string") + try: + return datetime.strptime(value, _DATE_FMT_IN).date() + except ValueError as exc: + raise ValueError(f"{field} must be YYYY-MM-DD ({value!r})") from exc + + +def _resolve_dates(params): + has_explicit = "fromDate" in params or "toDate" in params + has_last_years = "lastYears" in params + has_last_days = "lastDays" in params + + set_count = sum([has_explicit, has_last_years, has_last_days]) + if set_count == 0: + raise ValueError( + "Date window required: provide fromDate+toDate, lastYears, or lastDays" + ) + if set_count > 1: + raise ValueError( + "Date window over-specified: pick one of fromDate+toDate, lastYears, lastDays" + ) + + if has_explicit: + if "fromDate" not in params or "toDate" not in params: + raise ValueError("fromDate and toDate must be provided together") + from_d = _parse_iso_date(params["fromDate"], "fromDate") + to_d = _parse_iso_date(params["toDate"], "toDate") + else: + to_d = _today_utc() + if has_last_years: + n = params["lastYears"] + if not isinstance(n, int) or n <= 0: + raise ValueError("lastYears must be a positive integer") + try: + from_d = to_d.replace(year=to_d.year - n) + except ValueError: + # e.g. Feb 29 minus N years where target year is non-leap. + from_d = to_d.replace(year=to_d.year - n, day=28) + else: + n = params["lastDays"] + if not isinstance(n, int) or n <= 0: + raise ValueError("lastDays must be a positive integer") + from_d = to_d - timedelta(days=n) + + if from_d > to_d: + raise ValueError(f"fromDate ({from_d}) must be on or before toDate ({to_d})") + return from_d, to_d + + +def _resolve_modelling(params): + raw = params.get("modelling", "open-prices") + if raw in MODELLING_MAP: + return MODELLING_MAP[raw] + if isinstance(raw, int) and raw in MODELLING_MAP.values(): + return raw + raise ValueError( + f"Invalid modelling: {raw!r}. Use one of {sorted(MODELLING_MAP)}" + ) + + +def _strip_ex5(name): + return name[:-4] if name.lower().endswith(".ex5") else name + + +def _require_filename(value, field): + import os + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} is required") + name = value.strip() + if name != os.path.basename(name): + raise ValueError(f"{field} must be a filename, not a path") + return name + + +def _validate_timeframe(value): + if value not in VALID_TIMEFRAMES: + raise ValueError( + f"Invalid timeframe: {value!r}. Use one of {list(VALID_TIMEFRAMES)}" + ) + return value + + +def _positive_number(value, field, *, allow_zero=False, kind=float): + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{field} must be a number") + if (allow_zero and value < 0) or (not allow_zero and value <= 0): + raise ValueError(f"{field} must be {'>= 0' if allow_zero else '> 0'}") + return kind(value) + + +def build_ini(params: dict) -> str: + """Return INI text for the MT5 Strategy Tester from JSON params. + + Required: symbol, timeframe, expert, and a date window + (fromDate+toDate OR lastYears OR lastDays). + Optional: modelling, latencyMs, deposit, currency, leverage, + expertParameters, reportName, optimization, forwardMode, visual. + """ + if not isinstance(params, dict): + raise ValueError("params must be an object") + + symbol = params.get("symbol") + if not isinstance(symbol, str) or not symbol.strip(): + raise ValueError("symbol is required") + symbol = symbol.strip() + + timeframe = _validate_timeframe(params.get("timeframe")) + expert_filename = _require_filename(params.get("expert"), "expert") + if not expert_filename.lower().endswith(".ex5"): + raise ValueError("expert must end with .ex5") + + set_filename = "" + if params.get("expertParameters"): + set_filename = _require_filename(params["expertParameters"], "expertParameters") + if not set_filename.lower().endswith(".set"): + raise ValueError("expertParameters must end with .set") + + from_d, to_d = _resolve_dates(params) + model = _resolve_modelling(params) + + deposit = _positive_number(params.get("deposit", 10000), "deposit", kind=float) + leverage = _positive_number(params.get("leverage", 100), "leverage", kind=int) + latency_ms = _positive_number( + params.get("latencyMs", 0), "latencyMs", allow_zero=True, kind=int + ) + currency = params.get("currency", "USD") + if not isinstance(currency, str) or not currency.strip(): + raise ValueError("currency must be a non-empty string") + currency = currency.strip().upper() + + report_name = params.get("reportName", "backtest-report.htm") + if not isinstance(report_name, str) or not report_name.strip(): + raise ValueError("reportName must be a non-empty string") + report_name = report_name.strip() + if not report_name.lower().endswith((".htm", ".html")): + report_name += ".htm" + + optimization = int(bool(params.get("optimization", 0))) + forward_mode = params.get("forwardMode", 0) + if forward_mode not in (0, 1, 2, 3, 4): + raise ValueError("forwardMode must be 0..4") + visual = int(bool(params.get("visual", 0))) + + parser = ConfigParser() + parser.optionxform = str # preserve key casing — MT5 is case-sensitive. + + parser["Common"] = { + # Login/Password/Server are injected by the runner from config.yaml. + "Login": "", + "Password": "", + "Server": "", + "KeepPrivate": "0", + "AutoTrading": "1", + "NewsEnable": "0", + } + parser["Experts"] = { + "AllowLiveTrading": "1", + "AllowDllImport": "1", + "Enabled": "1", + } + parser["Tester"] = { + "Expert": f"Uploaded\\{_strip_ex5(expert_filename)}", + "Symbol": symbol, + "Period": timeframe, + "Deposit": str(int(deposit)) if deposit.is_integer() else f"{deposit:.2f}", + "Currency": currency, + "Leverage": f"1:{leverage}", + "Model": str(model), + "ExecutionMode": str(latency_ms), + "Optimization": str(optimization), + "ForwardMode": str(forward_mode), + "FromDate": from_d.strftime(_DATE_FMT_OUT), + "ToDate": to_d.strftime(_DATE_FMT_OUT), + "Report": f"Reports\\{report_name}", + "ReplaceReport": "1", + "ShutdownTerminal": "1", + "Visual": str(visual), + "UseLocal": "1", + "UseRemote": "0", + "UseCloud": "0", + "AllowDll": "1", + } + if set_filename: + parser["Tester"]["ExpertParameters"] = set_filename + + buffer = io.StringIO() + parser.write(buffer, space_around_delimiters=False) + return buffer.getvalue() diff --git a/mt5api/backtest/jobs.py b/mt5api/backtest/jobs.py new file mode 100644 index 0000000..270ccd2 --- /dev/null +++ b/mt5api/backtest/jobs.py @@ -0,0 +1,232 @@ +"""Backtest job state: persistent JSON + in-memory cache + summary parser. + +Each job has one JSON file at logs/backtest-jobs/.json. The in-memory +dict (BACKTEST_JOBS) is a write-through cache guarded by JOB_LOCK. State files +survive API restarts; sweep_orphans() marks any in-flight job as failed at +startup so callers do not poll forever. +""" +from __future__ import annotations + +import json +import os +import re +import threading +from datetime import datetime, timezone + +from mt5api.config import BACKTEST_JOB_DIR +from mt5api.logger import log + +JOB_LOCK = threading.Lock() +BACKTEST_JOBS: dict = {} + +POLL_AFTER_SECONDS = 60 +TERMINAL_STATUSES = frozenset({"completed", "failed"}) +ACTIVE_STATUSES = frozenset({"queued", "running"}) + + +def now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0, tzinfo=None).isoformat() + "Z" + + +def _state_path(job_id: str) -> str: + os.makedirs(BACKTEST_JOB_DIR, exist_ok=True) + return os.path.join(BACKTEST_JOB_DIR, f"{job_id}.json") + + +def _write(job: dict) -> None: + path = _state_path(job["jobId"]) + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as handle: + json.dump(job, handle, indent=2, sort_keys=True) + os.replace(tmp, path) + + +def store_job(job: dict) -> None: + with JOB_LOCK: + BACKTEST_JOBS[job["jobId"]] = job + _write(job) + + +def update_job(job_id: str, **changes) -> dict: + with JOB_LOCK: + job = BACKTEST_JOBS.get(job_id) + if job is None: + path = _state_path(job_id) + if not os.path.exists(path): + raise KeyError(job_id) + with open(path, "r", encoding="utf-8") as handle: + job = json.load(handle) + BACKTEST_JOBS[job_id] = job + job.update(changes) + _write(job) + return dict(job) + + +def load_job(job_id: str): + with JOB_LOCK: + job = BACKTEST_JOBS.get(job_id) + if job is not None: + return dict(job) + path = _state_path(job_id) + if not os.path.exists(path): + return None + with open(path, "r", encoding="utf-8") as handle: + job = json.load(handle) + with JOB_LOCK: + BACKTEST_JOBS.setdefault(job_id, job) + return dict(job) + + +def queue_position(job_id: str): + with JOB_LOCK: + active = sorted( + (j for j in BACKTEST_JOBS.values() if j.get("status") in ACTIVE_STATUSES), + key=lambda j: j.get("submittedAt", ""), + ) + for index, job in enumerate(active): + if job["jobId"] == job_id: + return index + return None + + +def public_payload(job: dict) -> dict: + payload = { + "jobId": job["jobId"], + "status": job["status"], + "broker": job.get("broker"), + "account": job.get("account"), + "submittedAt": job.get("submittedAt"), + "startedAt": job.get("startedAt"), + "finishedAt": job.get("finishedAt"), + "durationSeconds": job.get("durationSeconds"), + "reportName": job.get("reportName"), + "reportUrl": f"/backtest/{job['jobId']}/report", + "logUrl": f"/backtest/{job['jobId']}/log", + "statusUrl": f"/backtest/{job['jobId']}", + "pollAfterSeconds": POLL_AFTER_SECONDS, + } + pos = queue_position(job["jobId"]) + if pos is not None: + payload["queuePosition"] = pos + if job.get("error"): + payload["error"] = job["error"] + if job.get("summary") is not None: + payload["summary"] = job["summary"] + if job.get("exitCode") is not None: + payload["exitCode"] = job["exitCode"] + return payload + + +def sweep_orphans() -> int: + """Mark any queued/running jobs on disk as failed. + + Called at API startup. Returns the number of jobs swept. + """ + if not os.path.isdir(BACKTEST_JOB_DIR): + return 0 + swept = 0 + for entry in sorted(os.listdir(BACKTEST_JOB_DIR)): + if not entry.endswith(".json"): + continue + path = os.path.join(BACKTEST_JOB_DIR, entry) + try: + with open(path, "r", encoding="utf-8") as handle: + job = json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + log.warning("backtest sweep: cannot read %s: %s", entry, exc) + continue + if job.get("status") not in ACTIVE_STATUSES: + continue + job["status"] = "failed" + job["error"] = "API restarted before completion" + job["finishedAt"] = now_iso() + try: + with open(path, "w", encoding="utf-8") as handle: + json.dump(job, handle, indent=2, sort_keys=True) + except OSError as exc: + log.warning("backtest sweep: cannot write %s: %s", entry, exc) + continue + swept += 1 + log.info("backtest sweep: marked %s as failed (was %s)", job.get("jobId"), entry) + return swept + + +# ── Report summary parser ─────────────────────────────────────────── +# MT5 HTML report layout varies by build; treat every field as best-effort +# and return None for anything we cannot parse. + +_NUM_RE = re.compile(r"-?\d[\d\s,.]*") + +_LABEL_TO_KEY = { + "Bars": "bars", + "Ticks": "ticks", + "Symbols": "symbols", + "Total Net Profit": "netProfit", + "Gross Profit": "grossProfit", + "Gross Loss": "grossLoss", + "Profit Factor": "profitFactor", + "Recovery Factor": "recoveryFactor", + "Expected Payoff": "expectedPayoff", + "Sharpe Ratio": "sharpeRatio", + "Balance Drawdown Maximal": "maxDrawdown", + "Balance Drawdown Absolute": "maxDrawdownAbsolute", + "Equity Drawdown Maximal": "maxEquityDrawdown", + "Total Trades": "totalTrades", + "Total Deals": "totalDeals", + "Profit Trades": "profitTrades", + "Loss Trades": "lossTrades", +} + + +def _to_number(text: str): + text = text.strip() + if not text: + return None + # Strip currency symbols and percent signs, keep sign and decimal. + cleaned = text.replace("\u00a0", " ").replace(" ", "").replace(",", "") + # Drop trailing '%' or trailing '(...)' like "1234.56 (12.34%)". + cleaned = cleaned.split("(")[0].rstrip("%").strip() + try: + if "." in cleaned: + return float(cleaned) + return int(cleaned) + except ValueError: + return None + + +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def parse_report_summary(html: str) -> dict: + """Best-effort extract a summary block from an MT5 HTML report. + + Returns a dict with all known fields; values are None when not found. + """ + summary = {key: None for key in _LABEL_TO_KEY.values()} + if not html: + return summary + text = _TAG_RE.sub(" ", html) + text = _WS_RE.sub(" ", text) + for label, key in _LABEL_TO_KEY.items(): + idx = text.find(label) + if idx < 0: + continue + tail = text[idx + len(label): idx + len(label) + 80] + m = _NUM_RE.search(tail) + if m: + summary[key] = _to_number(m.group(0)) + return summary + + +def is_empty_backtest_summary(summary: dict) -> bool: + """Return True when MT5 produced a semantically empty report. + + A valid no-trade backtest can still have non-zero Bars/Ticks/Symbols, so only + treat the report as empty when MT5 reports zero market data across all three. + """ + return ( + summary.get("bars") == 0 + and summary.get("ticks") == 0 + and summary.get("symbols") == 0 + ) diff --git a/mt5api/config.py b/mt5api/config.py index 337247e..d9db850 100644 --- a/mt5api/config.py +++ b/mt5api/config.py @@ -10,6 +10,8 @@ BASE_DIR = os.path.dirname(PACKAGE_DIR) CONFIG_YAML = os.path.join(BASE_DIR, "config", "config.yaml") BROKERS_DIR = os.path.join(BASE_DIR, "terminals") +ASSETS_DIR = os.path.join(BASE_DIR, "assets") +DEFAULT_BACKTEST_TIMEOUT = "6h" def load_yaml_config(): @@ -48,6 +50,15 @@ def _parse_args(): "UTC on the wire. Negative values are allowed for west-of-UTC " "brokers.", ) + parser.add_argument( + "--mode", + default=None, + choices=["live", "backtest"], + help="Terminal role. 'live' (default) initializes the MT5 SDK and " + "runs the monitor; 'backtest' skips both so this process can " + "spawn terminal64.exe /portable subprocesses against the same " + "data dir without hitting MT5's single-instance lock.", + ) args, _ = parser.parse_known_args() return args @@ -103,6 +114,20 @@ def load_terminal_config(): """ cfg = load_yaml_config() terms = cfg.get("terminals") or [] + if _args.broker or _args.account: + for t in terms: + if t.get("broker") != (_args.broker or t.get("broker")): + continue + if t.get("account", "") != (_args.account or t.get("account", "")): + continue + return { + "broker": t.get("broker", "default"), + "account": t.get("account", ""), + "port": t.get("port"), + "utc_offset": t.get("utc_offset", "0"), + "mode": (t.get("mode") or "live"), + "symbol_suffix": t.get("symbol_suffix"), + } if terms: t = terms[0] return { @@ -110,8 +135,10 @@ def load_terminal_config(): "account": t.get("account", ""), "port": t.get("port"), "utc_offset": t.get("utc_offset", "0"), + "mode": (t.get("mode") or "live"), + "symbol_suffix": t.get("symbol_suffix"), } - return {"broker": "default", "account": ""} + return {"broker": "default", "account": "", "mode": "live"} _args = _parse_args() @@ -124,6 +151,22 @@ def load_terminal_config(): UTC_OFFSET_RAW = _args.utc_offset if _args.utc_offset is not None else os.environ.get("UTC_OFFSET", "") UTC_OFFSET_SECONDS = parse_duration_to_seconds(UTC_OFFSET_RAW) UTC_OFFSET_HOURS = UTC_OFFSET_SECONDS / 3600.0 +_BACKTEST_TIMEOUT_ENV = os.environ.get("BACKTEST_TIMEOUT") +_BACKTEST_TIMEOUT_CONFIG = load_yaml_config().get("backtest_timeout") +BACKTEST_TIMEOUT_RAW = ( + _BACKTEST_TIMEOUT_ENV + if _BACKTEST_TIMEOUT_ENV not in (None, "") + else (_BACKTEST_TIMEOUT_CONFIG if _BACKTEST_TIMEOUT_CONFIG not in (None, "") else DEFAULT_BACKTEST_TIMEOUT) +) +BACKTEST_TIMEOUT = BACKTEST_TIMEOUT_RAW +BACKTEST_TIMEOUT_SECONDS = parse_duration_to_seconds(BACKTEST_TIMEOUT) +_MODE_RAW = (_args.mode or _terminal_config.get("mode") or os.environ.get("MT5_MODE") or "live") +MODE = str(_MODE_RAW).strip().lower() or "live" +if MODE not in ("live", "backtest"): + MODE = "live" +SYMBOL_SUFFIX_CONFIGURED = "symbol_suffix" in _terminal_config +_SYMBOL_SUFFIX_RAW = _terminal_config.get("symbol_suffix") +SYMBOL_SUFFIX = "" if _SYMBOL_SUFFIX_RAW is None else str(_SYMBOL_SUFFIX_RAW) # Wickworks TA sidecar — reachable only from the mt5 container's net namespace # (compose: network_mode: "service:mt5", no published ports). From inside the @@ -156,6 +199,7 @@ def load_terminal_config(): IDENTITY = f"{BROKER}/{ACCOUNT}" if ACCOUNT else BROKER LOG_DIR = os.path.join(BASE_DIR, "logs") FULL_LOG = os.path.join(LOG_DIR, "full.log") +BACKTEST_JOB_DIR = os.path.join(LOG_DIR, "backtest-jobs") TIMEFRAME_MAP = { "M1": mt5.TIMEFRAME_M1, diff --git a/mt5api/handlers/terminal.py b/mt5api/handlers/terminal.py index 2a09294..af437bc 100644 --- a/mt5api/handlers/terminal.py +++ b/mt5api/handlers/terminal.py @@ -1,6 +1,6 @@ import MetaTrader5 as mt5 from flask import jsonify -from mt5api.config import UTC_OFFSET_HOURS, UTC_OFFSET_SECONDS +from mt5api.config import MODE, UTC_OFFSET_HOURS, UTC_OFFSET_SECONDS from mt5api.mt5client import ( ensure_initialized, m, @@ -11,7 +11,7 @@ def ping(): - return jsonify({"status": "ok"}) + return jsonify({"status": "ok", "mode": MODE}) @with_mt5 diff --git a/mt5api/main.py b/mt5api/main.py index 5e5142f..7afdde5 100644 --- a/mt5api/main.py +++ b/mt5api/main.py @@ -5,7 +5,8 @@ from waitress import serve -from mt5api.config import ACCOUNT, BROKER, HOST, PORT +from mt5api.backtest import jobs as backtest_jobs +from mt5api.config import ACCOUNT, BROKER, HOST, MODE, PORT from mt5api.logger import log from mt5api.monitor import start_monitor from mt5api.mt5client import ( @@ -64,29 +65,40 @@ def _handle_signal(sig, _frame): def main(): - log.info("Starting — broker=%s account=%s port=%d", BROKER, ACCOUNT, PORT) + log.info("Starting — broker=%s account=%s port=%d mode=%s", BROKER, ACCOUNT, PORT, MODE) signal.signal(signal.SIGTERM, _handle_signal) signal.signal(signal.SIGINT, _handle_signal) - try: - with session(): - connected = init_mt5() - except Exception as e: - log.warning("Startup init session failed: %s", e) - connected = False - - if connected: - log.info("MT5 connected.") - else: - log.warning( - "MT5 not ready yet, retrying every %ds in background...", - RETRY_INTERVAL, + if MODE == "backtest": + log.info( + "mode=backtest — skipping MT5 SDK init and live health monitor " + "so terminal64.exe /portable can be spawned by the tester " + "without hitting MT5's single-instance lock on this data dir." ) - t = threading.Thread(target=_background_init, daemon=True) - t.start() + else: + try: + with session(): + connected = init_mt5() + except Exception as e: + log.warning("Startup init session failed: %s", e) + connected = False - start_monitor() + if connected: + log.info("MT5 connected.") + else: + log.warning( + "MT5 not ready yet, retrying every %ds in background...", + RETRY_INTERVAL, + ) + t = threading.Thread(target=_background_init, daemon=True) + t.start() + + start_monitor() + + swept = backtest_jobs.sweep_orphans() + if swept: + log.warning("Backtest sweep marked %d orphaned job(s) as failed.", swept) log.info( "HTTP API listening on %s:%d (waitress, threads=%d, conn_limit=%d, max_queue_depth=%d)", diff --git a/mt5api/server.py b/mt5api/server.py index acc154e..5e255be 100644 --- a/mt5api/server.py +++ b/mt5api/server.py @@ -3,6 +3,7 @@ from flask import Flask, abort, g, request from flask_compress import Compress +from mt5api.backtest import handler as backtest_handler from mt5api.config import API_TOKEN from mt5api.handlers import account, history, orders, positions, symbols, terminal from mt5api.logger import log @@ -92,3 +93,10 @@ def _end_request(response): # ── History ────────────────────────────────────────────────────── app.get("/history/orders")(history.get_orders) app.get("/history/deals")(history.get_deals) + +# ── Backtest ───────────────────────────────────────────────────── +app.post("/backtest/build-ini")(backtest_handler.build_ini_route) +app.post("/backtest")(backtest_handler.run_backtest) +app.get("/backtest/")(backtest_handler.get_status) +app.get("/backtest//report")(backtest_handler.get_report) +app.get("/backtest//log")(backtest_handler.get_log) diff --git a/run.sh b/run.sh index c50c90d..ebc0eaa 100755 --- a/run.sh +++ b/run.sh @@ -12,7 +12,7 @@ done LOG_FILE="${DIR}/run.log" exec > >(tee "${LOG_FILE}") 2>&1 -mkdir -p "${DIR}/data/storage" "${DIR}/data/shared/scripts" "${DIR}/data/shared/config" "${DIR}/data/shared/terminals" "${DIR}/data/oem" +mkdir -p "${DIR}/data/storage" "${DIR}/data/shared/scripts" "${DIR}/data/shared/config" "${DIR}/data/shared/terminals" "${DIR}/data/oem" "${DIR}/assets/experts" "${DIR}/assets/sets" # Bootstrap docker-compose.yml from example on first run; user owns the real file. if [ ! -f "${DIR}/docker-compose.yml" ]; then diff --git a/scripts/api_runner.bat b/scripts/api_runner.bat index 3eaf3dc..5768d62 100644 --- a/scripts/api_runner.bat +++ b/scripts/api_runner.bat @@ -1,5 +1,5 @@ @echo off -:: api_runner.bat broker account port token utc_offset +:: api_runner.bat broker account port token utc_offset mode :: Wraps the Python API process so start/exit/exitcode are logged. setlocal enabledelayedexpansion set "AR_BROKER=%~1" @@ -7,7 +7,9 @@ set "AR_ACCOUNT=%~2" set "AR_PORT=%~3" set "AR_TOKEN=%~4" set "AR_OFFSET=%~5" +set "AR_MODE=%~6" if "!AR_OFFSET!"=="" set "AR_OFFSET=0" +if "!AR_MODE!"=="" set "AR_MODE=live" set "SHARED=C:\Users\Docker\Desktop\Shared" set "LOGDIR=%SHARED%\logs" set "AR_LOG=%LOGDIR%\api-!AR_BROKER!-!AR_ACCOUNT!.log" @@ -16,11 +18,11 @@ set "PYDIR=C:\Program Files\Python312" mkdir "%LOGDIR%" 2>nul -echo [%DATE% %TIME%] [api:!AR_BROKER!/!AR_ACCOUNT!] === PROCESS STARTED on port !AR_PORT! (utc_offset=!AR_OFFSET!) === >> "!AR_LOG!" -echo [%DATE% %TIME%] [start] [api:!AR_BROKER!/!AR_ACCOUNT!] PROCESS STARTED on port !AR_PORT! utc_offset=!AR_OFFSET! >> "%FULL_LOG%" +echo [%DATE% %TIME%] [api:!AR_BROKER!/!AR_ACCOUNT!] === PROCESS STARTED on port !AR_PORT! (utc_offset=!AR_OFFSET! mode=!AR_MODE!) === >> "!AR_LOG!" +echo [%DATE% %TIME%] [start] [api:!AR_BROKER!/!AR_ACCOUNT!] PROCESS STARTED on port !AR_PORT! utc_offset=!AR_OFFSET! mode=!AR_MODE! >> "%FULL_LOG%" cd /d "%SHARED%" -"%PYDIR%\python.exe" -m mt5api --broker !AR_BROKER! --account !AR_ACCOUNT! --port !AR_PORT! --token "!AR_TOKEN!" --utc-offset "!AR_OFFSET!" >> "!AR_LOG!" 2>&1 +"%PYDIR%\python.exe" -m mt5api --broker !AR_BROKER! --account !AR_ACCOUNT! --port !AR_PORT! --token "!AR_TOKEN!" --utc-offset "!AR_OFFSET!" --mode "!AR_MODE!" >> "!AR_LOG!" 2>&1 set "AR_EC=!ERRORLEVEL!" echo [%DATE% %TIME%] [api:!AR_BROKER!/!AR_ACCOUNT!] === PROCESS EXITED exit_code=!AR_EC! === >> "!AR_LOG!" diff --git a/scripts/config_helper.py b/scripts/config_helper.py index 6e9b622..230d84f 100644 --- a/scripts/config_helper.py +++ b/scripts/config_helper.py @@ -45,7 +45,10 @@ def main(): for t in cfg.get("terminals") or []: utc = t.get("utc_offset") utc = "0" if utc is None else str(utc).replace(" ", "") - print(t["broker"], t["account"], t["port"], utc) + mode = (t.get("mode") or "live").strip().lower() or "live" + if mode not in ("live", "backtest"): + mode = "live" + print(t["broker"], t["account"], t["port"], utc, mode) elif cmd == "ports": ports = [t["port"] for t in (cfg.get("terminals") or [])] @@ -118,6 +121,8 @@ def main(): "http {\n" " server {\n" " listen 80;\n" + " client_max_body_size 25m;\n" + " client_body_timeout 120s;\n" + "\n".join(locs) + "\n" " location / { return 404 \"no route\\n\"; }\n" " }\n" diff --git a/scripts/start.bat b/scripts/start.bat index 3eeb2f6..a54cf42 100644 --- a/scripts/start.bat +++ b/scripts/start.bat @@ -47,7 +47,7 @@ call :log "%START_LOG%" "install.bat done." :: Install base deps first (pyyaml required for config_helper.py below). call :log "%START_LOG%" "Installing pip packages..." call :log "%PIP_LOG%" "Installing pip packages..." -"%PYDIR%\python.exe" -m pip install --quiet pyyaml MetaTrader5 flask waitress flask-compress >> "%PIP_LOG%" 2>&1 +"%PYDIR%\python.exe" -m pip install --quiet pyyaml MetaTrader5 flask waitress flask-compress psutil >> "%PIP_LOG%" 2>&1 if !errorlevel! neq 0 ( call :log "%START_LOG%" "ERROR: pip install (base) failed (exit code !errorlevel!), aborting." call :log "%PIP_LOG%" "ERROR: pip install (base) failed" @@ -105,7 +105,9 @@ if !errorlevel! neq 0 ( :: Configured via config.yaml reboot_interval (minutes). 0 = disabled. :: Default: 30. /f on schtasks is idempotent -- overwrites existing task. set "REBOOT_INTERVAL=30" -for /f "delims=" %%V in ('"%PYDIR%\python.exe" "%SCRIPTS%\config_helper.py" reboot_interval 2^>nul') do set "REBOOT_INTERVAL=%%V" +"%PYDIR%\python.exe" "%SCRIPTS%\config_helper.py" reboot_interval > "%SHARED%\mt5_ri.tmp" 2>nul +for /f "usebackq delims=" %%V in ("%SHARED%\mt5_ri.tmp") do set "REBOOT_INTERVAL=%%V" +del "%SHARED%\mt5_ri.tmp" 2>nul if "!REBOOT_INTERVAL!"=="0" ( schtasks /delete /tn "MT5AutoReboot" /f >nul 2>&1 call :log "%START_LOG%" "Auto-reboot disabled (reboot_interval=0)." @@ -182,10 +184,12 @@ goto status_loop :: ══════════════════════════════════════════════════════════════════ :launch_terminal -:: %1=broker %2=account %3=port %4=utc_offset (ignored here) +:: %1=broker %2=account %3=port %4=utc_offset %5=mode (live|backtest) set "LT_BROKER=%~1" set "LT_ACCOUNT=%~2" set "LT_PORT=%~3" +set "LT_MODE=%~5" +if "!LT_MODE!"=="" set "LT_MODE=live" set "LT_BASEDIR=%BROKERS%\!LT_BROKER!\base" set "LT_DIR=%BROKERS%\!LT_BROKER!\!LT_ACCOUNT!" @@ -216,6 +220,11 @@ if exist "!LT_LOGFILE!" ( for %%A in ("!LT_LOGFILE!") do set LT_LOGSIZE=%%~zA ) +if /i "!LT_MODE!"=="backtest" ( + call :log "%START_LOG%" " !LT_BROKER!/!LT_ACCOUNT! mode=backtest -- portable dir prepared, terminal NOT launched (tester will spawn it on demand)." + exit /b 0 +) + call :log "%START_LOG%" "Starting terminal: !LT_BROKER!/!LT_ACCOUNT! (port !LT_PORT!) [log offset !LT_LOGSIZE!]" powershell -Command "Start-Process '!LT_DIR!\terminal64.exe' -ArgumentList '/portable','/config:\"!LT_DIR!\mt5start.ini\"' -Verb RunAs -WindowStyle Normal" @@ -245,10 +254,12 @@ set "LA_BROKER=%~1" set "LA_ACCOUNT=%~2" set "LA_PORT=%~3" set "LA_OFFSET=%~4" +set "LA_MODE=%~5" if "!LA_OFFSET!"=="" set "LA_OFFSET=0" +if "!LA_MODE!"=="" set "LA_MODE=live" -call :log "%START_LOG%" "Starting API (bg): !LA_BROKER!/!LA_ACCOUNT! on port !LA_PORT! (utc_offset=!LA_OFFSET!)" -start "MT5 API !LA_BROKER!/!LA_ACCOUNT!" cmd /c ""%SCRIPTS%\api_runner.bat" !LA_BROKER! !LA_ACCOUNT! !LA_PORT! !API_TOKEN! !LA_OFFSET!" +call :log "%START_LOG%" "Starting API (bg): !LA_BROKER!/!LA_ACCOUNT! on port !LA_PORT! (utc_offset=!LA_OFFSET! mode=!LA_MODE!)" +start "MT5 API !LA_BROKER!/!LA_ACCOUNT!" cmd /c ""%SCRIPTS%\api_runner.bat" !LA_BROKER! !LA_ACCOUNT! !LA_PORT! !API_TOKEN! !LA_OFFSET! !LA_MODE!" exit /b 0 :: ══════════════════════════════════════════════════════════════════ diff --git a/tests/test_backtest_handler.py b/tests/test_backtest_handler.py new file mode 100644 index 0000000..2edb08b --- /dev/null +++ b/tests/test_backtest_handler.py @@ -0,0 +1,259 @@ +"""Validation-level tests for mt5api.backtest.handler. + +Subprocess (terminal64.exe) is monkeypatched out so the worker thread does +not actually launch MT5. We only test request validation, INI parsing, +asset resolution, credential injection, and the queued response shape. +""" +from __future__ import annotations + +import io +import os +from unittest.mock import patch + +import pytest + +from mt5api.backtest import handler, jobs +from mt5api.server import app + + +@pytest.fixture +def client(monkeypatch, tmp_path): + # Isolate filesystem state for every test. + terminal_dir = tmp_path / "terminal" + terminal_dir.mkdir() + terminal_path = terminal_dir / "terminal64.exe" + terminal_path.write_text("stub") + assets_dir = tmp_path / "assets" + (assets_dir / "experts").mkdir(parents=True) + (assets_dir / "sets").mkdir(parents=True) + log_dir = tmp_path / "logs" + log_dir.mkdir() + job_dir = log_dir / "backtest-jobs" + + monkeypatch.setattr(handler, "TERMINAL_DIR", str(terminal_dir)) + monkeypatch.setattr(handler, "TERMINAL_PATH", str(terminal_path)) + monkeypatch.setattr(handler, "ASSETS_DIR", str(assets_dir)) + monkeypatch.setattr(handler, "LOG_DIR", str(log_dir)) + monkeypatch.setattr(handler, "BACKTEST_JOB_DIR", str(job_dir)) + monkeypatch.setattr(handler, "BROKER", "testbroker") + monkeypatch.setattr(handler, "ACCOUNT", "testacct") + monkeypatch.setattr(jobs, "BACKTEST_JOB_DIR", str(job_dir)) + jobs.BACKTEST_JOBS.clear() + + monkeypatch.setattr(handler, "_load_account_config", lambda: { + "login": 12345678, + "password": "secret", + "server": "Test-Server", + }) + + # Don't actually spawn worker threads; tests inspect queued state only. + monkeypatch.setattr(handler.threading, "Thread", _NoopThread) + + # Disable auth so /backtest is reachable without a token. + monkeypatch.setattr("mt5api.server.API_TOKEN", "") + + app.config["TESTING"] = True + return app.test_client(), tmp_path + + +class _NoopThread: + def __init__(self, *_, **__): + pass + + def start(self): + pass + + +_VALID_INI = ( + "[Tester]\n" + "Expert=will be overridden\n" + "Symbol=NZDJPY\n" + "Period=M15\n" + "FromDate=2020.01.01\n" + "ToDate=2025.01.01\n" + "Model=2\n" +) + + +def _multipart(ini_text=_VALID_INI, expert_bytes=b"EX5BYTES", expert_filename="MyEA.ex5", + set_bytes=None, set_filename=None, expert_name=None, set_name=None, + include_ini=True): + fields = {} + if include_ini: + fields["ini"] = (io.BytesIO(ini_text.encode("utf-8")), "tester.ini") + if expert_bytes is not None: + fields["expert"] = (io.BytesIO(expert_bytes), expert_filename) + if set_bytes is not None: + fields["set"] = (io.BytesIO(set_bytes), set_filename) + if expert_name is not None: + fields["expert_name"] = expert_name + if set_name is not None: + fields["set_name"] = set_name + return fields + + +def test_missing_ini_returns_400(client): + c, _ = client + resp = c.post("/backtest", data=_multipart(include_ini=False, expert_bytes=b"x"), + content_type="multipart/form-data") + assert resp.status_code == 400 + assert "ini" in resp.get_json()["error"].lower() + + +def test_missing_expert_returns_400(client): + c, _ = client + resp = c.post("/backtest", data=_multipart(expert_bytes=None), + content_type="multipart/form-data") + assert resp.status_code == 400 + assert "expert" in resp.get_json()["error"].lower() + + +def test_path_traversal_in_expert_name_rejected(client): + c, _ = client + resp = c.post( + "/backtest", + data=_multipart(expert_bytes=None, expert_name="../etc/passwd.ex5"), + content_type="multipart/form-data", + ) + assert resp.status_code == 400 + assert "filename" in resp.get_json()["error"].lower() + + +def test_unknown_asset_name_returns_400(client): + c, _ = client + resp = c.post( + "/backtest", + data=_multipart(expert_bytes=None, expert_name="missing.ex5"), + content_type="multipart/form-data", + ) + assert resp.status_code == 400 + assert "not found" in resp.get_json()["error"].lower() + + +def test_ini_without_tester_section_returns_400(client): + c, _ = client + resp = c.post( + "/backtest", + data=_multipart(ini_text="[Common]\nLogin=1\n"), + content_type="multipart/form-data", + ) + assert resp.status_code == 400 + assert "tester" in resp.get_json()["error"].lower() + + +def test_happy_path_inline_upload_returns_202(client): + c, tmp = client + resp = c.post("/backtest", data=_multipart(), + content_type="multipart/form-data") + assert resp.status_code == 202, resp.get_data(as_text=True) + body = resp.get_json() + assert body["status"] == "queued" + assert body["broker"] == "testbroker" + assert body["account"] == "testacct" + assert body["jobId"] + assert body["statusUrl"].endswith(body["jobId"]) + assert resp.headers.get("Retry-After") + + # Stage dir + persisted files exist. + job_id = body["jobId"] + stage_dir = os.path.join(handler.BACKTEST_JOB_DIR, job_id) + assert os.path.exists(os.path.join(stage_dir, "MyEA.ex5")) + assert os.path.exists(os.path.join(stage_dir, "normalized.ini")) + + # Credentials were injected; expert was rewritten to Uploaded\. + norm = open(os.path.join(stage_dir, "normalized.ini"), encoding="utf-8").read() + assert "Login=12345678" in norm + assert "Server=Test-Server" in norm + assert "Expert=Uploaded\\MyEA" in norm + + +def test_backtest_timeout_form_override_is_stored(client): + c, _ = client + resp = c.post( + "/backtest", + data={**_multipart(), "timeout": "15m"}, + content_type="multipart/form-data", + ) + assert resp.status_code == 202, resp.get_data(as_text=True) + job = jobs.load_job(resp.get_json()["jobId"]) + assert job["timeoutSeconds"] == 900 + + +def test_invalid_backtest_timeout_returns_400(client): + c, _ = client + resp = c.post( + "/backtest", + data={**_multipart(), "timeout": "abc"}, + content_type="multipart/form-data", + ) + assert resp.status_code == 400 + assert "invalid duration" in resp.get_json()["error"].lower() + + +def test_host_managed_asset_resolves(client): + c, tmp = client + expert_path = tmp / "assets" / "experts" / "Hosted.ex5" + expert_path.write_bytes(b"hosted-bytes") + set_path = tmp / "assets" / "sets" / "hosted.set" + set_path.write_bytes(b"hosted-set") + + resp = c.post( + "/backtest", + data=_multipart(expert_bytes=None, expert_name="Hosted.ex5", set_name="hosted.set"), + content_type="multipart/form-data", + ) + assert resp.status_code == 202, resp.get_data(as_text=True) + job_id = resp.get_json()["jobId"] + stage_dir = os.path.join(handler.BACKTEST_JOB_DIR, job_id) + assert open(os.path.join(stage_dir, "Hosted.ex5"), "rb").read() == b"hosted-bytes" + # Set file is namespaced with the jobId to avoid concurrent collisions. + assert os.path.exists(os.path.join(stage_dir, f"{job_id}__hosted.set")) + + +def test_status_unknown_job_returns_404(client): + c, _ = client + resp = c.get("/backtest/does-not-exist") + assert resp.status_code == 404 + + +def test_report_and_log_404_when_not_ready(client): + c, _ = client + resp = c.post("/backtest", data=_multipart(), content_type="multipart/form-data") + job_id = resp.get_json()["jobId"] + # Worker is no-op so neither file exists. + assert c.get(f"/backtest/{job_id}/report").status_code == 404 + assert c.get(f"/backtest/{job_id}/log").status_code == 404 + + +def test_build_ini_route_returns_text(client): + c, _ = client + resp = c.post( + "/backtest/build-ini", + json={ + "symbol": "NZDJPY", + "timeframe": "M15", + "expert": "EA.ex5", + "lastYears": 5, + "modelling": "open-prices", + "latencyMs": 5, + }, + ) + assert resp.status_code == 200 + body = resp.get_data(as_text=True) + assert "[Tester]" in body + assert "Symbol=NZDJPY" in body + assert "ExecutionMode=5" in body + + +def test_build_ini_route_validation_error(client): + c, _ = client + resp = c.post("/backtest/build-ini", json={"symbol": "X", "timeframe": "BAD", + "expert": "EA.ex5", "lastDays": 1}) + assert resp.status_code == 400 + assert "timeframe" in resp.get_json()["error"].lower() + + +def test_build_ini_requires_json(client): + c, _ = client + resp = c.post("/backtest/build-ini", data="not json", content_type="text/plain") + assert resp.status_code == 400 diff --git a/tests/test_backtest_ini_builder.py b/tests/test_backtest_ini_builder.py new file mode 100644 index 0000000..5a778e5 --- /dev/null +++ b/tests/test_backtest_ini_builder.py @@ -0,0 +1,175 @@ +"""Unit tests for mt5api.backtest.ini_builder.""" +from __future__ import annotations + +import configparser +from datetime import date + +import pytest + +from mt5api.backtest import ini_builder + + +def _parse(text): + parser = configparser.ConfigParser() + parser.optionxform = str + parser.read_string(text) + return parser + + +def _base(**overrides): + params = { + "symbol": "NZDJPY", + "timeframe": "M15", + "expert": "MyEA.ex5", + "lastYears": 5, + } + params.update(overrides) + return params + + +def test_happy_path_minimum_params(): + text = ini_builder.build_ini(_base()) + parser = _parse(text) + tester = parser["Tester"] + assert tester["Symbol"] == "NZDJPY" + assert tester["Period"] == "M15" + assert tester["Expert"] == "Uploaded\\MyEA" + assert tester["Model"] == "2" # open-prices default + assert tester["ExecutionMode"] == "0" + assert tester["Optimization"] == "0" + assert tester["Visual"] == "0" + assert tester["ShutdownTerminal"] == "1" + assert tester["ReplaceReport"] == "1" + assert tester["UseLocal"] == "1" + assert tester["UseRemote"] == "0" + assert tester["UseCloud"] == "0" + assert tester["Report"].startswith("Reports\\") + assert "ExpertParameters" not in tester + # Common stays empty — runner injects credentials. + assert parser["Common"]["Login"] == "" + + +def test_last_years_window(): + text = ini_builder.build_ini(_base(lastYears=5)) + parser = _parse(text) + to_str = parser["Tester"]["ToDate"] + from_str = parser["Tester"]["FromDate"] + to_d = date.fromisoformat(to_str.replace(".", "-")) + from_d = date.fromisoformat(from_str.replace(".", "-")) + delta_years = to_d.year - from_d.year + assert delta_years == 5 + assert to_d == date.today() or (to_d - date.today()).days in (-1, 0, 1) + + +def test_explicit_dates(): + text = ini_builder.build_ini({ + "symbol": "EURUSD", + "timeframe": "H1", + "expert": "EA.ex5", + "fromDate": "2020-01-01", + "toDate": "2024-06-30", + }) + tester = _parse(text)["Tester"] + assert tester["FromDate"] == "2020.01.01" + assert tester["ToDate"] == "2024.06.30" + + +def test_modelling_enum_mapping(): + cases = { + "every-tick": "0", + "1m-ohlc": "1", + "open-prices": "2", + "real-ticks": "4", + } + for name, expected in cases.items(): + text = ini_builder.build_ini(_base(modelling=name)) + assert _parse(text)["Tester"]["Model"] == expected + + +def test_latency_and_set_file(): + text = ini_builder.build_ini(_base( + latencyMs=5, + expertParameters="myparams.set", + )) + tester = _parse(text)["Tester"] + assert tester["ExecutionMode"] == "5" + assert tester["ExpertParameters"] == "myparams.set" + + +def test_deposit_currency_leverage(): + text = ini_builder.build_ini(_base( + deposit=25000, + currency="eur", + leverage=200, + )) + tester = _parse(text)["Tester"] + assert tester["Deposit"] == "25000" + assert tester["Currency"] == "EUR" + assert tester["Leverage"] == "1:200" + + +def test_report_name_normalization(): + text = ini_builder.build_ini(_base(reportName="custom")) + assert _parse(text)["Tester"]["Report"] == "Reports\\custom.htm" + + +def test_missing_symbol_raises(): + with pytest.raises(ValueError, match="symbol"): + ini_builder.build_ini({"timeframe": "M15", "expert": "EA.ex5", "lastDays": 30}) + + +def test_invalid_timeframe_raises(): + with pytest.raises(ValueError, match="timeframe"): + ini_builder.build_ini(_base(timeframe="M7")) + + +def test_invalid_modelling_raises(): + with pytest.raises(ValueError, match="modelling"): + ini_builder.build_ini(_base(modelling="bogus")) + + +def test_no_date_window_raises(): + with pytest.raises(ValueError, match="Date window required"): + ini_builder.build_ini({"symbol": "EURUSD", "timeframe": "M15", "expert": "EA.ex5"}) + + +def test_overspecified_date_window_raises(): + with pytest.raises(ValueError, match="over-specified"): + ini_builder.build_ini({ + "symbol": "EURUSD", + "timeframe": "M15", + "expert": "EA.ex5", + "lastDays": 30, + "lastYears": 1, + }) + + +def test_inverted_dates_raises(): + with pytest.raises(ValueError, match="must be on or before"): + ini_builder.build_ini({ + "symbol": "EURUSD", + "timeframe": "M15", + "expert": "EA.ex5", + "fromDate": "2024-12-31", + "toDate": "2020-01-01", + }) + + +def test_expert_must_end_with_ex5(): + with pytest.raises(ValueError, match=".ex5"): + ini_builder.build_ini(_base(expert="MyEA.dll")) + + +def test_expert_path_traversal_rejected(): + with pytest.raises(ValueError, match="filename"): + ini_builder.build_ini(_base(expert="../escape.ex5")) + + +def test_set_must_end_with_set(): + with pytest.raises(ValueError, match=".set"): + ini_builder.build_ini(_base(expertParameters="params.txt")) + + +def test_negative_last_days_rejected(): + with pytest.raises(ValueError, match="positive"): + ini_builder.build_ini({"symbol": "X", "timeframe": "M15", "expert": "EA.ex5", "lastDays": -1}) diff --git a/tests/test_backtest_jobs.py b/tests/test_backtest_jobs.py new file mode 100644 index 0000000..6338fa0 --- /dev/null +++ b/tests/test_backtest_jobs.py @@ -0,0 +1,121 @@ +"""Unit tests for mt5api.backtest.jobs sweep + summary parser.""" +from __future__ import annotations + +import json +import os + +import pytest + +from mt5api.backtest import jobs + + +@pytest.fixture +def tmp_jobs_dir(monkeypatch, tmp_path): + monkeypatch.setattr(jobs, "BACKTEST_JOB_DIR", str(tmp_path)) + # Reset in-memory cache between tests. + jobs.BACKTEST_JOBS.clear() + return tmp_path + + +def _write(tmp_path, job_id, status, **extra): + payload = {"jobId": job_id, "status": status, "submittedAt": "2026-05-12T10:00:00Z"} + payload.update(extra) + (tmp_path / f"{job_id}.json").write_text(json.dumps(payload)) + + +def test_sweep_marks_running_and_queued_as_failed(tmp_jobs_dir): + _write(tmp_jobs_dir, "abc", "running") + _write(tmp_jobs_dir, "def", "queued") + swept = jobs.sweep_orphans() + assert swept == 2 + for jid in ("abc", "def"): + data = json.loads((tmp_jobs_dir / f"{jid}.json").read_text()) + assert data["status"] == "failed" + assert data["error"] == "API restarted before completion" + assert data["finishedAt"] + + +def test_sweep_leaves_terminal_jobs_alone(tmp_jobs_dir): + _write(tmp_jobs_dir, "ok", "completed", summary={"netProfit": 12.5}) + _write(tmp_jobs_dir, "bad", "failed", error="boom") + swept = jobs.sweep_orphans() + assert swept == 0 + assert json.loads((tmp_jobs_dir / "ok.json").read_text())["status"] == "completed" + assert json.loads((tmp_jobs_dir / "bad.json").read_text())["error"] == "boom" + + +def test_sweep_handles_corrupt_files(tmp_jobs_dir): + (tmp_jobs_dir / "broken.json").write_text("{not json") + _write(tmp_jobs_dir, "live", "running") + assert jobs.sweep_orphans() == 1 + + +def test_sweep_no_directory(monkeypatch, tmp_path): + monkeypatch.setattr(jobs, "BACKTEST_JOB_DIR", str(tmp_path / "missing")) + assert jobs.sweep_orphans() == 0 + + +def test_summary_parser_returns_all_keys_for_empty_html(): + summary = jobs.parse_report_summary("") + expected_keys = { + "netProfit", "grossProfit", "grossLoss", "profitFactor", + "recoveryFactor", "expectedPayoff", "sharpeRatio", + "maxDrawdown", "maxDrawdownAbsolute", "maxEquityDrawdown", + "totalTrades", "totalDeals", "profitTrades", "lossTrades", + "bars", "ticks", "symbols", + } + assert set(summary.keys()) == expected_keys + assert all(v is None for v in summary.values()) + + +def test_summary_parser_extracts_numbers(): + html = """ + + + + + +
Total Net Profit1 234.56
Profit Factor1.42
Total Trades87
Balance Drawdown Maximal234.50 (5.12%)
+ """ + summary = jobs.parse_report_summary(html) + assert summary["netProfit"] == 1234.56 + assert summary["profitFactor"] == 1.42 + assert summary["totalTrades"] == 87 + assert summary["maxDrawdown"] == 234.50 + + +def test_public_payload_shape(tmp_jobs_dir): + job = { + "jobId": "xyz", + "status": "completed", + "broker": "darwinex", + "account": "live", + "submittedAt": "2026-05-12T10:00:00Z", + "startedAt": "2026-05-12T10:00:01Z", + "finishedAt": "2026-05-12T10:30:00Z", + "durationSeconds": 1799.0, + "reportName": "r.htm", + "summary": {"netProfit": 1.0}, + "exitCode": 0, + } + payload = jobs.public_payload(job) + assert payload["jobId"] == "xyz" + assert payload["statusUrl"] == "/backtest/xyz" + assert payload["reportUrl"] == "/backtest/xyz/report" + assert payload["logUrl"] == "/backtest/xyz/log" + assert payload["summary"] == {"netProfit": 1.0} + assert payload["exitCode"] == 0 + assert "queuePosition" not in payload # completed → no position + + +def test_store_and_load_job_roundtrip(tmp_jobs_dir): + job = {"jobId": "rt1", "status": "queued", "submittedAt": "2026-05-12T11:00:00Z"} + jobs.store_job(job) + loaded = jobs.load_job("rt1") + assert loaded["status"] == "queued" + jobs.update_job("rt1", status="running") + assert jobs.load_job("rt1")["status"] == "running" + + +def test_load_job_missing_returns_none(tmp_jobs_dir): + assert jobs.load_job("nope") is None