From ba9638d189434c331e748fce41b222ab8772bc41 Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:31:53 +0100 Subject: [PATCH 1/8] feat(side-quests): add Seqera Platform Automation side quest Port the Platform Automation training module from the standalone m42-platform-advanced-training repo into the training collection as a single-page side quest, following the side_quests convention. - docs/en/docs/side_quests/platform_automation/index.md: the guide, with codespace badge, asset paths, and cd commands retargeted to nextflow-io/training and side-quests/platform_automation/. - side-quests/platform_automation/{terraform,seqerakit}: the Terraform (compute-env, pipeline) and seqerakit assets, verbatim aside from path/reference fixes and repaired code fences in the seqerakit README. - .editorconfig: add 2-space rule for *.tf/*.tfvars (terraform fmt style). Intentionally not added to the mkdocs nav or the side_quests index table. Passes pre-commit (prettier, editorconfig, whitespace, eof), the heading checker, terraform validate, and a non-strict mkdocs build. Generated by Claude Code --- .editorconfig | 4 + .../side_quests/platform_automation/index.md | 513 ++++++++++++++++++ .../platform_automation/seqerakit/README.md | 55 ++ .../seqerakit/add-rnaseq.yml | 19 + .../seqerakit/launch-rnaseq.yml | 22 + .../terraform/compute-env/README.md | 66 +++ .../terraform/compute-env/main.tf | 187 +++++++ .../compute-env/terraform.tfvars.example | 28 + .../terraform/compute-env/variables.tf | 83 +++ .../terraform/pipeline/README.md | 66 +++ .../terraform/pipeline/main.tf | 43 ++ .../pipeline/terraform.tfvars.example | 17 + .../terraform/pipeline/variables.tf | 39 ++ 13 files changed, 1142 insertions(+) create mode 100644 docs/en/docs/side_quests/platform_automation/index.md create mode 100644 side-quests/platform_automation/seqerakit/README.md create mode 100644 side-quests/platform_automation/seqerakit/add-rnaseq.yml create mode 100644 side-quests/platform_automation/seqerakit/launch-rnaseq.yml create mode 100644 side-quests/platform_automation/terraform/compute-env/README.md create mode 100644 side-quests/platform_automation/terraform/compute-env/main.tf create mode 100644 side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example create mode 100644 side-quests/platform_automation/terraform/compute-env/variables.tf create mode 100644 side-quests/platform_automation/terraform/pipeline/README.md create mode 100644 side-quests/platform_automation/terraform/pipeline/main.tf create mode 100644 side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example create mode 100644 side-quests/platform_automation/terraform/pipeline/variables.tf diff --git a/.editorconfig b/.editorconfig index 5e6d11b4b4..252370594c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,10 @@ indent_style = space [*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 +# Terraform's canonical `terraform fmt` style uses 2-space indentation +[*.{tf,tfvars}] +indent_size = 2 + # ignore python and markdown [*.{py,md}] indent_style = unset diff --git a/docs/en/docs/side_quests/platform_automation/index.md b/docs/en/docs/side_quests/platform_automation/index.md new file mode 100644 index 0000000000..7fc5800ecd --- /dev/null +++ b/docs/en/docs/side_quests/platform_automation/index.md @@ -0,0 +1,513 @@ +# Seqera Platform Automation + +The Seqera Platform does not run your work. It is an API and a control plane over your cloud. It hands jobs to a compute environment, the compute environment runs them on cloud VMs, and the Platform reads back state, logs, and exit codes. + +Everything the web UI does, it does by calling the Platform API. So everything is automatable: with the API, Terraform, and the CLI you can manage compute environments, pipelines, and runs as code, with no clicking. This side quest walks that programmatic surface from the most privileged role to the least. + +### Learning goals + +In this side quest, we'll drive the Platform across three workspace roles of decreasing permission: **Admin**, **Maintain**, and **Launch**. Each role does one job and hands an artifact to the next. You'll learn how to: + +- Create a compute environment two ways, with increasing control over the cloud: the UI (Batch Forge) and Terraform +- Add a pipeline to the Launchpad declaratively with Terraform, and see idempotency +- Launch a pipeline using the GUI, CLI and seqerakit. +- Tell declarative existence (Terraform) apart from imperative actions (the UI, seqerakit, `tw`) and learn when to use each + +### Prerequisites + +Before taking on this side quest, you should: + +- Have a Seqera Platform account on Seqera Cloud (`https://cloud.seqera.io`), or an Enterprise install +- Be a member of a workspace with a role: Admin, Maintain, or Launch +- Have credentials for your cloud provider already added to that workspace (not covered here) +- Ideally, a GitHub access token added too, to avoid GitHub rate limits +- Be comfortable with the command line and basic Nextflow concepts + +New to running pipelines on Seqera at all? Start with the gentler "Run pipelines on Seqera" module in the [Nextflow Triathlon](https://training.nextflow.io/) (sign up, launch in the UI, the `tw` CLI). This side quest is the automation layer on top of it: roles, Terraform, seqerakit, and Actions. + +--- + +## 0. Get started + +### Open the training codespace + +The Codespace contains all the tools you need (Terraform, `tw`, seqerakit); you +install nothing yourself. Open it now and read on while it builds. It ends with "Toolchain ready" in the terminal. + +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/nextflow-io/training?quickstart=1&ref=master) + +The assets for this side quest live under `side-quests/platform_automation/`: +`terraform/compute-env/`, `terraform/pipeline/`, and `seqerakit/`. Each section +below tells you where to `cd`. + +### Create an access token + +We authenticate to the Platform to tell it who we are. That is what grants the permissions of our role. Seqera uses a **personal access token**: a bearer token you create once and send with every API call, in an `Authorization: Bearer ` header. The token carries your identity and role, so the Platform knows who you are and what you can do. Anyone with the token can act as you, so keep it secret. In this training we save it as an environment variable. + +1. In the Platform, open the user menu (top right) and choose **Your tokens**. +2. Click **Add token**, name it (e.g. `platform-automation`), and click **Add**. +3. Copy the token now. The Platform shows it only once. +4. In the Codespace terminal, export it under both names: + +```bash +export TOWER_ACCESS_TOKEN= +export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN +``` + +Terraform, `tw`, and seqerakit all read the token from these variables. Check it worked: + +```bash +tw info +``` + +This prints the API endpoint and the authenticated user. If it errors, the token is wrong or not exported. + +!!! warning + + This only exists in a single terminal session, if you open a new terminal you will need to export them again. + +For Seqera Enterprise, also set `SEQERA_API_URL` (and the `server_url` Terraform variable) to your install's API URL. Everything else is identical. + +### Find your organization and workspace IDs + +Later steps need the **numeric workspace ID**. The web UI navigates by name, so the number is not in the URL; find it like this: + +1. In the Platform, click your organization name to open the organization page. +2. Open the **Workspaces** tab. Each workspace is listed with its numeric ID. That + number is the `workspace_id` the Terraform and API steps ask for. +3. You only need the numeric organization ID if you list workspaces over the API; if you read the workspace ID from the Workspaces tab, you can skip it. + +Or list everything your token can reach from the terminal: + +```bash +tw workspaces list +``` + +This prints a table with the workspace ID, the workspace name, and the organization name. + +Two forms of the same workspace turn up in this module. Terraform and the API want the **numeric** ID (`workspace_id`). `tw` and seqerakit accept either the numeric ID or the `Organization/Workspace` **name** (e.g. `my-org/platform-automation`). Export the numeric ID once so `tw` targets the shared workspace without a `--workspace` flag on every command: + +```bash +export TOWER_WORKSPACE_ID= +``` + +#### Know your role + +The work is split by role, from most to least privileged. Start at the highest tier your access allows: + +| Tier | Role | Can change | Produces | +| ------------------------------------------------------- | ------------- | ---------------------------------------- | -------------------- | +| [Admin](#1-admin-compute-environments-and-cloud) | Owner / Admin | Compute environments and cloud resources | a `compute_env_id` | +| [Maintain](#2-maintain-add-a-pipeline-to-the-launchpad) | Maintain | Pipelines on the Launchpad | a Launchpad pipeline | +| [Launch](#3-launch-run-a-pipeline) | Launch | Nothing; can only run | pipeline runs | + +Admin users create compute environments and assign roles to everyone else. If you only have Maintain, an Admin on your team hands you a `compute_env_id`; start at section 2. If you only have Launch, they hand you a Launchpad pipeline (and optionally an Action) to run; start at section 3. Each section opens with what it requires. + +--- + +## 1. Admin: compute environments and cloud + +**Requires:** Owner or Admin role, and permissions in the cloud environment. If you are using a Cloud provider, authenticate with them first via their CLI. + +The same compute environment can be created with increasing control over the cloud. We build it two ways. + +### 1.1. Click-ops a compute environment with Batch Forge + +Batch Forge is the hands-off option: the Platform reaches into your cloud and creates the resources for you. Most convenient, least control. + +In the side bar, open **Compute Environments** and click **Create compute environment**. Give it a name, pick your cloud platform, and select the credentials for that cloud. The rest of the form differs by provider; recommended settings: + +**AWS** + +- Region +- Work directory (S3 bucket) +- Wave, Fusion, Fast Instance should all be enabled +- Config mode should be Batch Forge +- Provisioning model should be Spot, using Spot instances for nodes which run Nextflow tasks. + +**Azure** + +- Location +- Work directory (Azure Storage container) +- Wave and Fusion should be enabled. +- Select "Separate head and worker pools" to create one pool for the Nextflow head job and a separate pool for the tasks. +- For the head pool VM type, select a VM you have sufficient quota for. Standard_D2s_v3 (2 CPUs) is a good starting choice. +- Set the head pool VM count to 1. The head pool runs a single Nextflow head job; it does not need more. +- For the worker pool, select a VM type you have sufficient quota for. Standard_D4s_v3 (4 CPUs) is a good starting choice. +- Set the worker pool VM count to 4 +- Leave autoscaling enabled for both pools + +Leave the other settings at their defaults. Keep **Dispose resources** enabled so the pool is torn down when you delete the compute environment. Click **Add** in the top right. + +Watch your cloud console and you will see Forge create the resources: an identity and roles for Nextflow (access to blob storage and the Batch service), a more limited role for the worker tasks (storage only), the pools, and their networking. Many moving parts, all handled for you. + +That convenience is the trade-off: Forge owns those resources and gives you little control. A lab that already runs on managed infrastructure wants the opposite, to describe the resources itself and wire the Platform to them. That is the Terraform path. + +### 1.2. Provision the cloud with Terraform + +Terraform manages cloud resources declaratively: you describe what should exist and Terraform makes it so. It does not run your work; it only creates the compute environment. `side-quests/platform_automation/terraform/compute-env/` does it in a single `main.tf`, two providers in one apply: + +- data sources: the existing Batch account, managed identity, and vnet/subnet, + referenced, not created. +- `azurerm`: creates a head pool and a worker pool on that account and subnet, + as that identity. +- `seqera`: stores the Azure credentials and creates the compute environment. + +The manual marker: `head_pool` is set to the pool Terraform just made and there is **no `forge` block**. That one difference is what makes the compute environment manual instead of Forge. `nextflow_config` routes tasks to the worker pool. + +Let's walk `terraform/compute-env/main.tf` from top to bottom. + +The first block declares the providers and pins their versions. The `seqera` provider is pinned to exactly `0.30.5`: + +```terraform +terraform { + required_version = ">= 1.9" + + required_providers { + seqera = { source = "seqeralabs/seqera", version = "0.30.5" } + azurerm = { source = "hashicorp/azurerm", version = ">= 3.0" } + random = { source = "hashicorp/random", version = ">= 3.0" } + } +} +``` + +Next we configure the providers. The `seqera` provider reads `TOWER_ACCESS_TOKEN` from the environment, so there is no token argument here: + +```terraform +provider "azurerm" { + features {} + subscription_id = var.subscription_id +} + +provider "seqera" { + server_url = var.server_url +} +``` + +Pool sizing and the VM image live in a `locals` block, so you edit them in one place rather than wiring every value through as a variable: + +```terraform +locals { + head_vm_size = "Standard_D4ds_v5" + worker_vm_size = "Standard_E16ds_v5" + worker_max_nodes = 8 + worker_max_tasks = 16 + node_agent_sku_id = "batch.node.ubuntu 22.04" +} +``` + +The cloud resources the lab already owns, the Batch account, the managed identity, and the subnet, are referenced with data sources. Terraform reads them; it does not create or manage them: + +```terraform +data "azurerm_batch_account" "existing" { + name = var.batch_account_name + resource_group_name = var.batch_account_rg +} + +data "azurerm_user_assigned_identity" "existing" { + name = var.managed_identity_name + resource_group_name = var.managed_identity_rg +} + +data "azurerm_subnet" "existing" { + name = var.subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_rg +} +``` + +Now the resources Terraform does create. The head pool runs the Nextflow head job, so one fixed node is enough: + +```terraform +resource "azurerm_batch_pool" "head" { + name = "rnaseq-head-${random_string.suffix.result}" + resource_group_name = var.batch_account_rg + account_name = var.batch_account_name + vm_size = local.head_vm_size + node_agent_sku_id = local.node_agent_sku_id + + fixed_scale { + target_dedicated_nodes = 1 + } + + storage_image_reference { + publisher = "microsoft-dsvm" + offer = "ubuntu-hpc" + sku = "2204" + version = "latest" + } + # identity, container, and network_configuration omitted for brevity +} +``` + +The worker pool runs the pipeline tasks, so it autoscales from zero up to `worker_max_nodes` based on the number of pending tasks: + +```terraform +resource "azurerm_batch_pool" "worker" { + name = "rnaseq-worker-${random_string.suffix.result}" + vm_size = local.worker_vm_size + node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.worker_max_tasks + # ...same account, image, identity, and network as the head pool + + auto_scale { + evaluation_interval = "PT5M" + formula = <<-FORMULA + pending = avg($PendingTasks.GetSample(180 * TimeInterval_Second)); + $TargetDedicatedNodes = min(${local.worker_max_nodes}, pending); + $NodeDeallocationOption = taskcompletion; + FORMULA + } +} +``` + +Both pools run the same `ubuntu-hpc` `2204` image, which is why `node_agent_sku_id` is `batch.node.ubuntu 22.04` for both. + +The `seqera` provider stores the Azure credentials in the Platform: + +```terraform +resource "seqera_azure_credential" "main" { + name = "azure-batch" + workspace_id = var.workspace_id + batch_name = var.batch_account_name + batch_key = var.azure_batch_key + storage_name = var.azure_storage_name + storage_key = var.azure_storage_key +} +``` + +Finally, the compute environment itself. This is the manual marker in code: `head_pool` points at the pool we just created, there is no `forge` block, and `nextflow_config` routes tasks to the worker pool's queue. Because it references `azurerm_batch_pool.head.name` and `azurerm_batch_pool.worker.name`, Terraform knows to create the pools first: + +```terraform +resource "seqera_compute_env" "main" { + workspace_id = var.workspace_id + + compute_env = { + name = "azure-batch-manual" + platform = "azure-batch" + credentials_id = seqera_azure_credential.main.credentials_id + + config = { + azure_batch = { + region = var.region + work_dir = var.work_dir + head_pool = azurerm_batch_pool.head.name + managed_identity_client_id = data.azurerm_user_assigned_identity.existing.client_id + nextflow_config = "process.queue = '${azurerm_batch_pool.worker.name}'\n" + } + } + } +} +``` + +The last block is the output the next tier needs: + +```terraform +output "compute_env_id" { + value = seqera_compute_env.main.compute_env_id +} +``` + +Terraform reads those references and works out the order itself: credentials and pools first, then the compute environment that depends on them. Run it: + +```bash +az login +cd side-quests/platform_automation/terraform/compute-env +terraform init +terraform plan +terraform apply +``` + +See `side-quests/platform_automation/terraform/compute-env/README.md` for the full variable list and the `terraform.tfvars` setup. + +### 1.3. See both sides + +The compute environment now exists in two places, and as an Admin with cloud access you can see both. + +On the Platform side, read the ID Terraform produced: + +```bash +terraform output compute_env_id +``` + +On the Azure side, open the Batch account in the portal. You will see the head and worker pools Terraform created, sitting idle with no jobs yet. Once you launch a pipeline (sections 2 and 3), jobs and tasks stack onto these pools, and you can drill into a task's logs and exit code. That is the whole point of the Admin tier: the Platform submits to Azure Batch, and Azure Batch runs the work. Maintain- and Launch-role users see only the compute environment in the workspace, not the cloud behind it. + +### Takeaway + +One compute environment, two levels of control: Forge in the UI (easiest, the Platform owns the pool) and Terraform (you own the pools and the wiring, end to end). The artifact you hand to the Maintain tier is a `compute_env_id`. + +--- + +## 2. Maintain: add a pipeline to the Launchpad + +**Requires:** Maintain role, and a `compute_env_id` (from section 1 or an Admin on your team) plus the numeric `workspace_id`. Maintain manages pipelines, not compute environments. That split is deliberate: Admin owns the compute environment, you own your pipelines. + +You add the same pipeline, `rnaseq-nf-$USER`, four ways. The first three are imperative, you run a command and it acts. The last, Terraform, is declarative. Watch what happens when you run each one twice. + +### 2.1. Add a pipeline via the UI + +In the workspace, open the **Launchpad** and click **Add pipeline**. Fill in the form: + +- **Name**: `rnaseq-nf-` (unique in a shared workspace). +- **Compute environment**: the one from section 1, e.g. `azure-batch-manual`. +- **Pipeline to launch**: `https://github.com/nextflow-io/rnaseq-nf`. +- **Revision**: `master`. +- **Work directory**: your Azure Blob work dir, e.g. `az://nf-work/work`. + +Click **Add**. The pipeline appears on the Launchpad with no run started. Adding a pipeline only saves a launch configuration; it does not run anything. + +### 2.2. Add a pipeline via `tw` + +The CLI does the same thing in one command: + +```bash +tw pipelines add \ + --name="rnaseq-nf-$USER" \ + --compute-env="azure-batch-manual" \ + --work-dir="az://nf-work/work" \ + --revision="master" \ + https://github.com/nextflow-io/rnaseq-nf +``` + +Run it again and `tw` errors: a pipeline with that name already exists. The command is imperative, so each invocation tries to add a pipeline; it has no notion of "already in the desired state". + +### 2.3. Add a pipeline with seqerakit + +`seqerakit` is a wrapper over `tw` that reads a YAML file and runs the underlying `tw` commands. It keeps the configuration as code, so a teammate can reproduce the exact same pipeline. `seqerakit/add-rnaseq.yml` describes the pipeline: + +```yaml +pipelines: + - name: "rnaseq-nf-${USERNAME}" + url: "https://github.com/nextflow-io/rnaseq-nf" + workspace: "my-org/platform-automation" + description: "rnaseq-nf for ${USERNAME}, added with seqerakit" + compute-env: "azure-batch-manual" + work-dir: "az://nf-work/work" + revision: "master" +``` + +`seqerakit` expands `${USERNAME}` from the environment, so set it first, then add the pipeline: + +```bash +cd side-quests/platform_automation/seqerakit +export USERNAME=$USER +seqerakit add-rnaseq.yml +``` + +If you run it again, it errors, the same way `tw` did, because the pipeline already exists: + +```console +ERROR: A pipeline with name 'rnaseq-nf-' already exists. +``` + +You can force it through with `seqerakit add-rnaseq.yml --overwrite`, which deletes and recreates the pipeline. But that is you telling it to repeat the action. By default, adding twice is an error. To make "add once, and leave it alone after that" the _default_ behaviour, we need a tool that manages existence rather than actions: Terraform. + +### 2.4. Add a pipeline with Terraform + +`side-quests/platform_automation/terraform/pipeline` adds the same pipeline declaratively. You describe the pipeline that should exist; Terraform makes the workspace match: + +```bash +cd side-quests/platform_automation/terraform/pipeline +terraform init +terraform plan -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" +terraform apply -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" +``` + +Tip: copy `terraform.tfvars.example` to `terraform.tfvars` so you stop passing `-var` flags. + +Open the Launchpad: `rnaseq-nf-$USER` is there, with no run. Now apply again: + +```bash +terraform apply -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" +``` + +```console +No changes. Your infrastructure matches the configuration. +``` + +That is the difference. `tw` and `seqerakit` errored on the second run because they perform an action every time. Terraform manages whether the pipeline **exists**: it is already there, so there is nothing to do. Apply ten times, still one pipeline. This is what keeps you from accumulating competing, half-duplicated resources in a shared workspace. To remove the pipeline, `terraform apply -destroy` with the same vars. + +### Takeaway + +Four ways to add one pipeline, two mental models. `tw` and `seqerakit` are imperative: each run is an action, and adding twice is an error. Terraform is declarative: it manages existence, so a second apply is a no-op. The artifact you hand to the Launch tier is the Launchpad pipeline `rnaseq-nf-$USER`. + +--- + +## 3. Launch: run a pipeline + +**Requires:** Launch role and a token with that role, plus a pipeline on the Launchpad (added by a Maintainer in section 2). Launch can only run things; it cannot create or modify compute environments, pipelines, or Actions. + +Everything in this section runs the pipeline the Maintainer already configured. None of it changes the pipeline; launching is an action, not a state. + +### 3.1. Use the GUI + +Open the **Launchpad**, select `rnaseq-nf-$USER`, and click **Launch**. The compute environment, revision, and work directory are already filled in by the Maintainer, so a Launch user just clicks the button. Submit it and the run appears under **Runs**. + +### 3.2. Use the `tw` CLI + +```bash +tw launch rnaseq-nf-$USER +``` + +One command, no flags: everything is pre-configured on the Launchpad entry. The CLI submits the run and prints its URL. + +For an automated launch, pin the parameters in a version-controlled file and pass it with `--params-file params.yaml`. That keeps each run reproducible, which is the whole reason to drive launches from code rather than the form. + +### 3.3. Use `seqerakit` + +`seqerakit` launches a pipeline that already exists on the Launchpad, filling in the `tw launch` command from `seqerakit/launch-rnaseq.yml`. Set `USERNAME`, dry run first, then launch: + +```bash +cd side-quests/platform_automation/seqerakit +export USERNAME=$USER + +seqerakit launch-rnaseq.yml --dryrun # prints the tw command, changes nothing +seqerakit launch-rnaseq.yml # launches for real +``` + +The dry run shows the underlying `tw` command. Run it twice and you get two runs. That is the imperative model: do the thing, now. It is the opposite of Terraform (section 2.4), which manages existence. See `side-quests/platform_automation/seqerakit/README.md`. + +### 3.4. Examine the cloud resources + +A run does not create one neatly named cloud job. Nextflow submits **many** Batch jobs and tasks, one per process invocation, so you will not find a single resource named after the Platform run. Open the run on the Platform (**Runs** → your run), which mirrors the underlying Batch service: the task table here is the same work you can see in the cloud console. + +If you have cloud access, the prefixes Nextflow uses in each Batch service are: + +- **Azure Batch**: jobs named `job--`, tasks named `nf-`. +- **AWS Batch**: job names of the form `_`, with unsupported characters stripped (max 128 chars). +- **GCP Batch**: job IDs of the form `nf--`. + +The relationship is the point: the Platform hands work to Batch, Batch runs it on the pool VMs, and the Platform reads back state, logs, and exit codes. + +### 3.5. Side note: launch Actions + +There is one more way to launch, built for automation rather than people. An **Action** is a saved launch configuration behind a single URL: call that URL and the pipeline runs, with no Launchpad and no `tw`. A Maintainer creates one in the UI under **Actions** → **Add action**, picks the pipeline and compute environment, and saves it. The Platform then shows a ready-made `curl` command for the Action's endpoint. + +The split mirrors the roles: _creating_ an Action needs the Maintain role, but _triggering_ it needs only a Launch token. That is the point of the stratified launch: a Maintainer hands launchers (or an automation system) a URL they can call to run the pipeline, and nothing more. + +### Takeaway + +Several ways to launch, all imperative: the GUI button, `tw launch`, `seqerakit`, and a launch Action's URL. Each does the thing, now; run it twice and you get two runs. That is the opposite of Terraform, which manages whether something exists. + +--- + +## Summary + +You drove the Seqera Platform programmatically across three roles, each handing an artifact to the next: Admin builds the compute environment and owns the cloud, Maintain adds pipelines and Launch triggers a run. + +### Key patterns + +- **Everything is one API.** The GUI, Terraform, `tw`, and seqerakit all call the same Platform API. Anything you can click, you can automate. +- **Use the right tool.** Terraform manages resources as state: what should exist, where. `tw` and `seqerakit` act on the Platform imperatively: they do the thing, now. + +| | Terraform | seqerakit / tw / Action | +| ------------ | ------------- | ----------------------- | +| Manages | existence | actions | +| Run twice | no-op | two runs | +| Mental model | desired state | do the thing, now | + +- **Roles stratify what each token can do.** A Maintain token defines pipelines and Actions; a Launch token can only trigger an Action and nothing else. The Action is the safe handoff from maintainers to launchers (and to automation). + +## What's next? + +- The AI half of the workshop is the CoScientist side quest in [nextflow-io/training](https://github.com/nextflow-io/training), published at training.nextflow.io. It uses the same `rnaseq-nf` pipeline and API endpoints, but interacts with them via AI agents. diff --git a/side-quests/platform_automation/seqerakit/README.md b/side-quests/platform_automation/seqerakit/README.md new file mode 100644 index 0000000000..f22e28d9f2 --- /dev/null +++ b/side-quests/platform_automation/seqerakit/README.md @@ -0,0 +1,55 @@ +# seqerakit + +The imperative asset, used in two tiers. See the Platform Automation side quest, §2 (Maintain) and §3 (Launch). seqerakit wraps `tw` and reads a YAML file: + +- `add-rnaseq.yml` (Maintain) adds `rnaseq-nf-$USERNAME` to the Launchpad. Re-running errors unless `--overwrite`, the imperative contrast to Terraform's idempotent `apply`. +- `launch-rnaseq.yml` (Launch) launches that pipeline. Run it twice and you get two runs. + +Terraform owns the pipeline's existence; seqerakit and `tw` act on it. This is the imperative contrast to the API Action and to Terraform. + +## Declarative vs imperative + +| | Terraform | seqerakit / tw | +| ------------ | ------------------------------------ | -------------------- | +| Manages | existence (does the pipeline exist?) | actions (run it now) | +| Run twice | no-op, still one pipeline | two runs | +| Mental model | desired state | do the thing, now | + +You added `rnaseq-nf-$USERNAME` with Terraform. Now you launch it. + +## Setup + +```bash +export TOWER_ACCESS_TOKEN= +export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN +export USERNAME=$USER +``` + +Edit `workspace` and `compute-env` in the YAML to match the workshop workspace if they differ. + +## Dry run first + +Always dry run first. `--dryrun` prints the `tw` command seqerakit would run and changes nothing: + +```bash +seqerakit launch-rnaseq.yml --dryrun +``` + +Then launch for real: + +```bash +seqerakit launch-rnaseq.yml +``` + +## The tw equivalent + +seqerakit is a wrapper over `tw`. The dry run shows you the underlying command. +It is the same as: + +```bash +tw launch rnaseq-nf-$USERNAME \ + --workspace=my-org/platform-automation \ + --compute-env=azure-batch-manual +``` + +This is the callback to the Azure mechanics: launching here submits jobs to the Platform, which hands them to Azure Batch, which stacks tasks onto the pool VMs. Same handoff the Admin tier showed in the portal. diff --git a/side-quests/platform_automation/seqerakit/add-rnaseq.yml b/side-quests/platform_automation/seqerakit/add-rnaseq.yml new file mode 100644 index 0000000000..4de2081787 --- /dev/null +++ b/side-quests/platform_automation/seqerakit/add-rnaseq.yml @@ -0,0 +1,19 @@ +# Add rnaseq-nf-$USERNAME to the Launchpad (the imperative contrast to Terraform). +# +# This ADDS a pipeline to the Launchpad. It does not launch it. Re-running errors +# unless you pass --overwrite, because the pipeline already exists. That is the +# imperative model: each run is an action, not a desired state. +# +# seqerakit expands ${USERNAME} from the environment, so set it first: +# +# export USERNAME=$USER +# +# Set the workspace and compute-env names to match your workshop workspace. +pipelines: + - name: "rnaseq-nf-${USERNAME}" + url: "https://github.com/nextflow-io/rnaseq-nf" + workspace: "my-org/platform-automation" + description: "rnaseq-nf for ${USERNAME}, added with seqerakit" + compute-env: "azure-batch-manual" + work-dir: "az://nf-work/work" + revision: "master" diff --git a/side-quests/platform_automation/seqerakit/launch-rnaseq.yml b/side-quests/platform_automation/seqerakit/launch-rnaseq.yml new file mode 100644 index 0000000000..afd1ebd0bf --- /dev/null +++ b/side-quests/platform_automation/seqerakit/launch-rnaseq.yml @@ -0,0 +1,22 @@ +# Launch rnaseq-nf-$USERNAME against the shared compute environment. +# +# This LAUNCHES a pipeline that already exists on the Launchpad. Terraform +# added it (see ../terraform/pipeline). seqerakit does not create it here. +# +# Declarative vs imperative: +# Terraform manages existence. apply twice = no second pipeline. +# seqerakit/tw do the thing, now. Run this twice = two runs. That is the +# point. Launching is an action, not a state. +# +# A bare `pipeline:` name (not a git URL) means "the saved Launchpad pipeline +# with this name". seqerakit expands ${USERNAME} from the environment, so set +# it first: +# +# export USERNAME=$USER +# +# Set the workspace and compute-env names to match your workshop workspace. +launch: + - name: "rnaseq-nf-${USERNAME}-run" + workspace: "my-org/platform-automation" + pipeline: "rnaseq-nf-${USERNAME}" + compute-env: "azure-batch-manual" diff --git a/side-quests/platform_automation/terraform/compute-env/README.md b/side-quests/platform_automation/terraform/compute-env/README.md new file mode 100644 index 0000000000..9a8f997fad --- /dev/null +++ b/side-quests/platform_automation/terraform/compute-env/README.md @@ -0,0 +1,66 @@ +# terraform/compute-env + +The Admin tier's compute-environment asset. See the Platform Automation side quest, §1 (Admin). You run this +only if you have the **Admin** (or Owner) role and permissions in the cloud +environment. In a workshop the presenter builds the shared compute environment +before the session; Maintain-role attendees do not run this and instead get a +`compute_env_id` to build pipelines against. + +## What it does + +The realistic, complex path. One config, two providers, end to end: + +1. **azurerm** creates a head pool and a worker pool on the customer's existing + Azure Batch account, on their existing subnet, running as their existing + managed identity. +2. **seqera** stores the Azure credentials and creates a manual Azure Batch + compute environment that uses the head pool and routes tasks to the worker + pool. + +This is what "manage your cloud as code" looks like for a lab that already +owns its Azure footprint. Terraform does not create the Batch account, the +network, or the identity. Those exist under change control and are referenced +with data sources (in `main.tf`). Terraform adds the pools and the Platform +wiring. + +## Three tiers, for context + +The session shows compute environments at three levels of control: + +1. **Batch Forge (UI)**: the Platform creates and scales the pool. Easiest. +2. **Manual CE pointing at an existing pool**: you already have a pool, the + Platform just submits to it. +3. **This config**: Terraform provisions the pools on your existing account and + network, then wires the Platform. Full infrastructure as code. + +## Manual vs Forge + +The compute environment is manual: `head_pool` is set and there is no `forge` +block. The Platform submits to the pools this config created. It never creates +or scales a pool itself. A `forge` block would flip it to Batch Forge. + +## Run it + +```bash +az login +export TOWER_ACCESS_TOKEN= +export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN + +cp terraform.tfvars.example terraform.tfvars +# edit terraform.tfvars: subscription, workspace, existing resource names, keys + +terraform init +terraform plan +terraform apply +``` + +After apply, note the `compute_env_id` output. The Maintain tier needs it for +`terraform/pipeline`. + +## The cloud side + +With Admin and cloud access you can also see the Azure side: the Batch account, +the pool VMs, jobs and tasks stacking onto them, a task's logs and exit code. +That is the whole point of the tier: the Platform submits to Azure Batch, which +runs the work. Maintain- and Launch-role users see only the compute environment +in the workspace, not the cloud behind it. diff --git a/side-quests/platform_automation/terraform/compute-env/main.tf b/side-quests/platform_automation/terraform/compute-env/main.tf new file mode 100644 index 0000000000..e90f9aeb3c --- /dev/null +++ b/side-quests/platform_automation/terraform/compute-env/main.tf @@ -0,0 +1,187 @@ +# Admin tier: Terraform creates the Azure Batch pools, then wires a manual +# Seqera compute environment to them. Manual = head_pool is set and there is no +# forge block, so the Platform submits to these pools but never creates or +# scales them. One config, two providers, end to end. +# +# Auth: run `az login`, then export TOWER_ACCESS_TOKEN (the seqera provider +# reads it from the environment). Sensitive keys come from terraform.tfvars or +# TF_VAR_ env vars; never commit them. + +terraform { + required_version = ">= 1.9" + + required_providers { + seqera = { source = "seqeralabs/seqera", version = "0.30.5" } + azurerm = { source = "hashicorp/azurerm", version = ">= 3.0" } + random = { source = "hashicorp/random", version = ">= 3.0" } + } +} + +provider "azurerm" { + features {} + subscription_id = var.subscription_id +} + +provider "seqera" { + server_url = var.server_url +} + +# Pool sizing and image. Edit here rather than exposing every value as a knob. +locals { + head_vm_size = "Standard_D4ds_v5" + worker_vm_size = "Standard_E16ds_v5" + worker_max_nodes = 8 + worker_max_tasks = 16 + node_agent_sku_id = "batch.node.ubuntu 22.04" +} + +# Keeps pool names unique across re-creates. +resource "random_string" "suffix" { + length = 6 + special = false + upper = false +} + +# Existing Azure resources the customer owns. Referenced, not managed. +data "azurerm_batch_account" "existing" { + name = var.batch_account_name + resource_group_name = var.batch_account_rg +} + +data "azurerm_user_assigned_identity" "existing" { + name = var.managed_identity_name + resource_group_name = var.managed_identity_rg +} + +data "azurerm_subnet" "existing" { + name = var.subnet_name + virtual_network_name = var.vnet_name + resource_group_name = var.vnet_rg +} + +# Let the managed identity read and write the work directory's storage. +resource "azurerm_role_assignment" "batch_data_contributor" { + scope = data.azurerm_batch_account.existing.id + role_definition_name = "Azure Batch Data Contributor" + principal_id = data.azurerm_user_assigned_identity.existing.principal_id +} + +# Head pool: runs the Nextflow head job. One node is enough. +resource "azurerm_batch_pool" "head" { + name = "rnaseq-head-${random_string.suffix.result}" + display_name = "rnaseq head pool" + resource_group_name = var.batch_account_rg + account_name = var.batch_account_name + vm_size = local.head_vm_size + node_agent_sku_id = local.node_agent_sku_id + + fixed_scale { + target_dedicated_nodes = 1 + } + + storage_image_reference { + publisher = "microsoft-dsvm" + offer = "ubuntu-hpc" + sku = "2204" + version = "latest" + } + + container_configuration { + type = "DockerCompatible" + } + + identity { + type = "UserAssigned" + identity_ids = [data.azurerm_user_assigned_identity.existing.id] + } + + network_configuration { + subnet_id = data.azurerm_subnet.existing.id + public_address_provisioning_type = "NoPublicIPAddresses" + } +} + +# Worker pool: runs the pipeline tasks. Autoscales 0..worker_max_nodes. +resource "azurerm_batch_pool" "worker" { + name = "rnaseq-worker-${random_string.suffix.result}" + display_name = "rnaseq worker pool" + resource_group_name = var.batch_account_rg + account_name = var.batch_account_name + vm_size = local.worker_vm_size + node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.worker_max_tasks + + auto_scale { + evaluation_interval = "PT5M" + formula = <<-FORMULA + pending = avg($PendingTasks.GetSample(180 * TimeInterval_Second)); + $TargetDedicatedNodes = min(${local.worker_max_nodes}, pending); + $NodeDeallocationOption = taskcompletion; + FORMULA + } + + storage_image_reference { + publisher = "microsoft-dsvm" + offer = "ubuntu-hpc" + sku = "2204" + version = "latest" + } + + container_configuration { + type = "DockerCompatible" + } + + identity { + type = "UserAssigned" + identity_ids = [data.azurerm_user_assigned_identity.existing.id] + } + + network_configuration { + subnet_id = data.azurerm_subnet.existing.id + public_address_provisioning_type = "NoPublicIPAddresses" + } + + # Terraform manages the pool, not its live node count. + lifecycle { + ignore_changes = [auto_scale] + } +} + +# Azure credentials stored in the Platform. Keys are write-only. +resource "seqera_azure_credential" "main" { + name = "azure-batch" + workspace_id = var.workspace_id + batch_name = var.batch_account_name + batch_key = var.azure_batch_key + storage_name = var.azure_storage_name + storage_key = var.azure_storage_key +} + +# Manual Azure Batch compute environment: uses the head pool, routes tasks to +# the worker pool. References to the pool names make Terraform create them first. +resource "seqera_compute_env" "main" { + workspace_id = var.workspace_id + + compute_env = { + name = "azure-batch-manual" + description = "Azure Batch CE on Terraform-managed pools" + platform = "azure-batch" + credentials_id = seqera_azure_credential.main.credentials_id + + config = { + azure_batch = { + region = var.region + work_dir = var.work_dir + head_pool = azurerm_batch_pool.head.name + managed_identity_client_id = data.azurerm_user_assigned_identity.existing.client_id + nextflow_config = "process.queue = '${azurerm_batch_pool.worker.name}'\n" + } + } + } +} + +# The Maintain tier (terraform/pipeline) needs this compute_env_id. +output "compute_env_id" { + description = "ID of the Azure Batch compute environment." + value = seqera_compute_env.main.compute_env_id +} diff --git a/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example b/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example new file mode 100644 index 0000000000..44e1a3bbc9 --- /dev/null +++ b/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example @@ -0,0 +1,28 @@ +# Copy to terraform.tfvars and fill in. terraform.tfvars is gitignored. +# Do NOT put your access token here. Export TOWER_ACCESS_TOKEN in the shell, +# and run `az login` first. + +subscription_id = "00000000-0000-0000-0000-000000000000" +workspace_id = 123456789 +region = "uaenorth" +work_dir = "az://nf-work/work" + +# Existing Azure resources (referenced, not created) +batch_account_name = "mybatch" +batch_account_rg = "batch-rg" +managed_identity_name = "nextflow-identity" +managed_identity_rg = "identity-rg" +vnet_name = "nf-vnet" +vnet_rg = "network-rg" +subnet_name = "batch-subnet" + +# Pool sizing lives in main.tf (locals), not here. + +# Seqera credential keys (sensitive). Prefer TF_VAR_azure_batch_key / +# TF_VAR_azure_storage_key environment variables over this file. +azure_storage_name = "mystorage" +azure_batch_key = "REPLACE_ME" +azure_storage_key = "REPLACE_ME" + +# server_url defaults to Seqera Cloud; uncomment only for Enterprise. +# server_url = "https://platform.example.com/api" diff --git a/side-quests/platform_automation/terraform/compute-env/variables.tf b/side-quests/platform_automation/terraform/compute-env/variables.tf new file mode 100644 index 0000000000..7fc99ffd1b --- /dev/null +++ b/side-quests/platform_automation/terraform/compute-env/variables.tf @@ -0,0 +1,83 @@ +# Inputs for the Admin tier compute-environment config. The Platform token is +# NOT here; it comes from TOWER_ACCESS_TOKEN. Sensitive Azure keys come from a +# gitignored terraform.tfvars or TF_VAR_ environment variables. + +variable "server_url" { + description = "Seqera Platform API endpoint. Cloud by default." + type = string + default = "https://api.cloud.seqera.io" +} + +variable "subscription_id" { + description = "Azure subscription ID." + type = string +} + +variable "workspace_id" { + description = "Numeric ID of the Platform workspace that owns the compute environment." + type = number +} + +# Existing Azure resources (referenced via data sources, not created). +variable "batch_account_name" { + description = "Name of the existing Azure Batch account." + type = string +} + +variable "batch_account_rg" { + description = "Resource group of the existing Azure Batch account." + type = string +} + +variable "managed_identity_name" { + description = "Name of the existing user-assigned managed identity for Batch nodes." + type = string +} + +variable "managed_identity_rg" { + description = "Resource group of the existing managed identity." + type = string +} + +variable "vnet_name" { + description = "Name of the existing virtual network the pools attach to." + type = string +} + +variable "vnet_rg" { + description = "Resource group of the existing virtual network." + type = string +} + +variable "subnet_name" { + description = "Name of the existing subnet the pool nodes join." + type = string +} + +variable "region" { + description = "Azure region code of the Batch account, e.g. uaenorth." + type = string +} + +variable "work_dir" { + description = "Azure Blob Storage work directory, e.g. az://nf-work/work." + type = string +} + +# Seqera credential keys (sensitive). +variable "azure_batch_key" { + description = "Azure Batch account access key." + type = string + sensitive = true +} + +variable "azure_storage_name" { + description = "Azure Storage account name for the work directory." + type = string +} + +variable "azure_storage_key" { + description = "Azure Storage account access key." + type = string + sensitive = true +} diff --git a/side-quests/platform_automation/terraform/pipeline/README.md b/side-quests/platform_automation/terraform/pipeline/README.md new file mode 100644 index 0000000000..4bde95d596 --- /dev/null +++ b/side-quests/platform_automation/terraform/pipeline/README.md @@ -0,0 +1,66 @@ +# terraform/pipeline + +The Maintain tier's pipeline asset. This adds your own pipeline, +`rnaseq-nf-$USERNAME`, to the Launchpad with Terraform. It adds it. It does not +launch it. See the Platform Automation side quest, §2 (Maintain). + +## Before you start + +You need the **Maintain** role and two values from the Admin tier (or your +presenter): + +- `workspace_id`: the shared workspace. +- `compute_env_id`: the shared Azure Batch compute environment. + +And create a Platform token (User menu, Your tokens, Add token), then: + +```bash +export TOWER_ACCESS_TOKEN= +export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN +``` + +## The flow + +```bash +cd side-quests/platform_automation/terraform/pipeline + +# 1. Set your inputs. Either copy the example file: +cp terraform.tfvars.example terraform.tfvars +# and edit it, or pass -var flags on each command below. + +# 2. Initialise (downloads the seqera 0.30.5 provider). +terraform init + +# 3. Preview. One resource to add. Nothing changes yet. +terraform plan -var="username=$USER" + +# 4. Add the pipeline to the Launchpad. +terraform apply -var="username=$USER" +``` + +Open the workspace Launchpad. `rnaseq-nf-$USERNAME` is there. No run has +started. + +## Idempotency + +Run apply again: + +```bash +terraform apply -var="username=$USER" +``` + +"No changes. Your infrastructure matches the configuration." Terraform manages +existence, not actions. Applying ten times does not launch ten runs. It +launches zero. To actually run the pipeline, use seqerakit/tw (see +`../../seqerakit`). That is the imperative half: do the thing, now. + +## Teardown + +Remove your pipeline when you are done: + +```bash +terraform apply -destroy -var="username=$USER" +``` + +It deletes only the pipeline you added. The shared compute environment and +workspace are untouched. diff --git a/side-quests/platform_automation/terraform/pipeline/main.tf b/side-quests/platform_automation/terraform/pipeline/main.tf new file mode 100644 index 0000000000..ff1eb26033 --- /dev/null +++ b/side-quests/platform_automation/terraform/pipeline/main.tf @@ -0,0 +1,43 @@ +# Maintain tier: add one rnaseq-nf pipeline to the Launchpad. This ADDS the +# pipeline; it does not launch it. Terraform manages existence, not runs: +# +# terraform apply creates rnaseq-nf- +# terraform apply (again) no-op; idempotent, never launches a run +# terraform apply -destroy removes it +# +# Launching is imperative, done with seqerakit/tw (see ../../seqerakit). +# +# Auth: export TOWER_ACCESS_TOKEN; the seqera provider reads it from the env. + +terraform { + required_version = ">= 1.6" + + required_providers { + seqera = { + source = "seqeralabs/seqera" + version = "0.30.5" + } + } +} + +provider "seqera" { + server_url = var.server_url +} + +resource "seqera_pipeline" "rnaseq_nf" { + name = "rnaseq-nf-${var.username}" + description = "rnaseq-nf for ${var.username}, added with Terraform" + workspace_id = var.workspace_id + + launch = { + pipeline = "https://github.com/nextflow-io/rnaseq-nf" + revision = "master" + compute_env_id = var.compute_env_id + work_dir = var.work_dir + } +} + +output "pipeline_name" { + description = "Launchpad pipeline name. Use this when you launch it." + value = seqera_pipeline.rnaseq_nf.name +} diff --git a/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example b/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example new file mode 100644 index 0000000000..ac5950cc13 --- /dev/null +++ b/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example @@ -0,0 +1,17 @@ +# Copy to terraform.tfvars and fill in. terraform.tfvars is gitignored. +# Do NOT put your access token here. Export TOWER_ACCESS_TOKEN in the shell. + +# Your username. Or pass it on the command line: terraform apply -var="username=$USER" +username = "your-username" + +# Shared workshop workspace ID (the presenter gives you this). +workspace_id = 123456789 + +# Shared compute environment ID (the presenter gives you this). +compute_env_id = "REPLACE_ME" + +# Optional: defaults to az://nf-work/work +# work_dir = "az://nf-work/work" + +# server_url defaults to Seqera Cloud; uncomment only for Enterprise. +# server_url = "https://platform.example.com/api" diff --git a/side-quests/platform_automation/terraform/pipeline/variables.tf b/side-quests/platform_automation/terraform/pipeline/variables.tf new file mode 100644 index 0000000000..f7ac7f2080 --- /dev/null +++ b/side-quests/platform_automation/terraform/pipeline/variables.tf @@ -0,0 +1,39 @@ +# Inputs for the Maintain tier pipeline config. + +variable "server_url" { + description = "Seqera Platform API endpoint. Cloud by default." + type = string + default = "https://api.cloud.seqera.io" +} + +# Your username makes the pipeline name unique in a shared workspace. No +# default on purpose, so you must set it. Set it with -var or in terraform.tfvars: +# +# terraform apply -var="username=$USER" +variable "username" { + description = "Your username. Makes the Launchpad pipeline name unique." + type = string + + validation { + condition = can(regex("^[a-zA-Z0-9._-]{2,80}$", var.username)) + error_message = "username must be 2-80 chars: letters, numbers, dot, dash, underscore." + } +} + +variable "workspace_id" { + description = "Numeric ID of the shared workshop workspace." + type = number +} + +# The Admin tier built the shared compute environment with terraform/compute-env +# and shared its ID. This is a string ID, not the numeric workspace ID. +variable "compute_env_id" { + description = "ID of the shared Azure Batch compute environment to launch against." + type = string +} + +variable "work_dir" { + description = "Azure Blob Storage work directory for runs of this pipeline." + type = string + default = "az://nf-work/work" +} From bd5bccaf528f3aae1a2716b578403abfc680ba06 Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:40:51 +0100 Subject: [PATCH 2/8] feat(devcontainer): add Terraform, tw, and seqerakit to the toolchain The platform_automation side quest needs Terraform, the Seqera CLI (tw), and seqerakit. Add them to the dev image the same way the rest of the toolchain is managed (features baked into the codespaces-dev/local-dev image), not via setup.sh: - terraform: official devcontainers/features/terraform feature, pinned to 1.9.8 with tflint/terragrunt disabled. - tw: new local-features/seqera-cli feature, mirroring tower-agent but arch-aware (x86_64/arm64) so the multi-arch image build stays green. - seqerakit: added to the existing uv-tools feature (pure-Python uv tool). - hashicorp.terraform VS Code extension added to all three configs. The tools land in the prebuilt image once docker-devcontainer rebuilds on merge to master; the production devcontainer pulls that image. Generated by Claude Code --- .../codespaces-dev/devcontainer.json | 9 ++++++- .devcontainer/devcontainer.json | 3 ++- .devcontainer/local-dev/devcontainer.json | 9 ++++++- .../seqera-cli/devcontainer-feature.json | 13 +++++++++ .../local-features/seqera-cli/install.sh | 27 +++++++++++++++++++ .../local-features/uv-tools/install.sh | 2 ++ 6 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/local-features/seqera-cli/devcontainer-feature.json create mode 100644 .devcontainer/local-features/seqera-cli/install.sh diff --git a/.devcontainer/codespaces-dev/devcontainer.json b/.devcontainer/codespaces-dev/devcontainer.json index 84812c4ccf..3628394fab 100644 --- a/.devcontainer/codespaces-dev/devcontainer.json +++ b/.devcontainer/codespaces-dev/devcontainer.json @@ -20,8 +20,14 @@ "version": "latest" }, "ghcr.io/va-h/devcontainers-features/uv": {}, + "ghcr.io/devcontainers/features/terraform:1": { + "version": "1.9.8", + "tflint": "none", + "terragrunt": "none" + }, "../local-features/apptainer": {}, "../local-features/tower-agent": {}, + "../local-features/seqera-cli": {}, "../local-features/nextflow": {}, "../local-features/conda-channels": {}, "../local-features/uv-tools": {} @@ -47,7 +53,8 @@ "vscode": { "extensions": [ "nf-core.nf-core-extensionpack", - "ms-vscode.live-server" + "ms-vscode.live-server", + "hashicorp.terraform" ], // Use Python from conda "settings": { diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fa99bfe859..01ddd2ea77 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -27,7 +27,8 @@ "vscode": { "extensions": [ "nf-core.nf-core-extensionpack", - "ms-vscode.live-server" + "ms-vscode.live-server", + "hashicorp.terraform" ], // Use Python from conda "settings": { diff --git a/.devcontainer/local-dev/devcontainer.json b/.devcontainer/local-dev/devcontainer.json index 130b120d66..b8bd743d95 100644 --- a/.devcontainer/local-dev/devcontainer.json +++ b/.devcontainer/local-dev/devcontainer.json @@ -20,8 +20,14 @@ "version": "latest" }, "ghcr.io/va-h/devcontainers-features/uv": {}, + "ghcr.io/devcontainers/features/terraform:1": { + "version": "1.9.8", + "tflint": "none", + "terragrunt": "none" + }, "../local-features/apptainer": {}, "../local-features/tower-agent": {}, + "../local-features/seqera-cli": {}, "../local-features/nextflow": {}, "../local-features/conda-channels": {}, "../local-features/uv-tools": {} @@ -49,7 +55,8 @@ "vscode": { "extensions": [ "nf-core.nf-core-extensionpack", - "ms-vscode.live-server" + "ms-vscode.live-server", + "hashicorp.terraform" ], // Use Python from conda "settings": { diff --git a/.devcontainer/local-features/seqera-cli/devcontainer-feature.json b/.devcontainer/local-features/seqera-cli/devcontainer-feature.json new file mode 100644 index 0000000000..14c7ee69b5 --- /dev/null +++ b/.devcontainer/local-features/seqera-cli/devcontainer-feature.json @@ -0,0 +1,13 @@ +{ + "id": "seqera-cli", + "name": "Install Seqera CLI (tw)", + "description": "Install the Seqera Platform CLI, tw", + "options": { + "version": { + "type": "string", + "proposals": ["latest", "0.32.0"], + "default": "0.32.0", + "description": "Select or enter a tower-cli version (without the leading v), or 'latest'." + } + } +} diff --git a/.devcontainer/local-features/seqera-cli/install.sh b/.devcontainer/local-features/seqera-cli/install.sh new file mode 100644 index 0000000000..6b6d3e4add --- /dev/null +++ b/.devcontainer/local-features/seqera-cli/install.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Install the Seqera Platform CLI (tw). +# Arch-aware so the multi-architecture image build works on both amd64 and arm64. +# Only downloads the binary (no execution), so it is safe under QEMU emulation. +set -eu + +VERSION="${VERSION:-0.32.0}" + +case "$(uname -m)" in + x86_64 | amd64) ARCH="x86_64" ;; + aarch64 | arm64) ARCH="arm64" ;; + *) + echo "Unsupported architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +if [ "${VERSION}" = "latest" ]; then + URL="https://github.com/seqeralabs/tower-cli/releases/latest/download/tw-linux-${ARCH}" +else + URL="https://github.com/seqeralabs/tower-cli/releases/download/v${VERSION}/tw-linux-${ARCH}" +fi + +echo "Installing Seqera CLI (tw) ${VERSION} for ${ARCH}..." +curl -fSL "${URL}" -o /usr/local/bin/tw +chmod +x /usr/local/bin/tw diff --git a/.devcontainer/local-features/uv-tools/install.sh b/.devcontainer/local-features/uv-tools/install.sh index 8537c04973..d6465a6be7 100644 --- a/.devcontainer/local-features/uv-tools/install.sh +++ b/.devcontainer/local-features/uv-tools/install.sh @@ -5,3 +5,5 @@ uv tool install pre-commit uv tool install nf-core==3.5.2 uv tool install "mkdocs-quiz>=1.5.2" +# seqerakit: used by the platform_automation side quest +uv tool install "seqerakit==0.5.7" From a3e2dab170bf01457623599b2189968cdef4ae2f Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:53:24 +0100 Subject: [PATCH 3/8] fix(platform_automation): make navigation coherent after the repo move The assets moved from the repo root into side-quests/platform_automation/, so the root-relative `cd` commands broke mid-sequence: after cd-ing into terraform/compute-env, a later `cd side-quests/platform_automation/seqerakit` fails. Make every `cd` absolute (/workspaces/training/...) so each step works from wherever the previous one left the learner, matching the absolute-path convention used elsewhere in the side quests. Also drop the stale "ends with 'Toolchain ready'" line: that string came from the standalone repo's setup.sh and is not printed by the training devcontainer. Generated by Claude Code --- docs/en/docs/side_quests/platform_automation/index.md | 11 ++++++----- .../platform_automation/terraform/pipeline/README.md | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/en/docs/side_quests/platform_automation/index.md b/docs/en/docs/side_quests/platform_automation/index.md index 7fc5800ecd..ab349a195a 100644 --- a/docs/en/docs/side_quests/platform_automation/index.md +++ b/docs/en/docs/side_quests/platform_automation/index.md @@ -32,7 +32,8 @@ New to running pipelines on Seqera at all? Start with the gentler "Run pipelines ### Open the training codespace The Codespace contains all the tools you need (Terraform, `tw`, seqerakit); you -install nothing yourself. Open it now and read on while it builds. It ends with "Toolchain ready" in the terminal. +install nothing yourself. Open it now and read on while it builds; it is ready +when the terminal returns to a prompt. [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/nextflow-io/training?quickstart=1&ref=master) @@ -312,7 +313,7 @@ Terraform reads those references and works out the order itself: credentials and ```bash az login -cd side-quests/platform_automation/terraform/compute-env +cd /workspaces/training/side-quests/platform_automation/terraform/compute-env terraform init terraform plan terraform apply @@ -389,7 +390,7 @@ pipelines: `seqerakit` expands `${USERNAME}` from the environment, so set it first, then add the pipeline: ```bash -cd side-quests/platform_automation/seqerakit +cd /workspaces/training/side-quests/platform_automation/seqerakit export USERNAME=$USER seqerakit add-rnaseq.yml ``` @@ -407,7 +408,7 @@ You can force it through with `seqerakit add-rnaseq.yml --overwrite`, which dele `side-quests/platform_automation/terraform/pipeline` adds the same pipeline declaratively. You describe the pipeline that should exist; Terraform makes the workspace match: ```bash -cd side-quests/platform_automation/terraform/pipeline +cd /workspaces/training/side-quests/platform_automation/terraform/pipeline terraform init terraform plan -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" terraform apply -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" @@ -458,7 +459,7 @@ For an automated launch, pin the parameters in a version-controlled file and pas `seqerakit` launches a pipeline that already exists on the Launchpad, filling in the `tw launch` command from `seqerakit/launch-rnaseq.yml`. Set `USERNAME`, dry run first, then launch: ```bash -cd side-quests/platform_automation/seqerakit +cd /workspaces/training/side-quests/platform_automation/seqerakit export USERNAME=$USER seqerakit launch-rnaseq.yml --dryrun # prints the tw command, changes nothing diff --git a/side-quests/platform_automation/terraform/pipeline/README.md b/side-quests/platform_automation/terraform/pipeline/README.md index 4bde95d596..6cae02825d 100644 --- a/side-quests/platform_automation/terraform/pipeline/README.md +++ b/side-quests/platform_automation/terraform/pipeline/README.md @@ -22,7 +22,7 @@ export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN ## The flow ```bash -cd side-quests/platform_automation/terraform/pipeline +cd /workspaces/training/side-quests/platform_automation/terraform/pipeline # 1. Set your inputs. Either copy the example file: cp terraform.tfvars.example terraform.tfvars From 626683d6a10d68456e966df2c2119d94a0dc2e82 Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:06:18 +0100 Subject: [PATCH 4/8] fix(platform_automation): gitignore terraform secrets, link CoScientist next step Add Terraform ignore rules so learner-generated terraform.tfvars and *.tfstate (both contain Azure credentials) are not committed; the committed *.tfvars.example files stay tracked. Point the "What's next?" cross-reference at the in-site Building with Seqera AI (CoScientist) side quest instead of a generic GitHub link. Generated by Claude Code --- .gitignore | 9 +++++++++ docs/en/docs/side_quests/platform_automation/index.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 68936ef169..7ba8953eab 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,12 @@ _scripts/.venv/ # Tutorial working directories (created by learners during exercises) hello-nf-core/core-hello/ + +# Terraform (platform_automation side quest) +**/.terraform/* +*.tfstate +*.tfstate.* +*.tfvars +!*.tfvars.example +crash.log +crash.*.log diff --git a/docs/en/docs/side_quests/platform_automation/index.md b/docs/en/docs/side_quests/platform_automation/index.md index ab349a195a..3dd605338f 100644 --- a/docs/en/docs/side_quests/platform_automation/index.md +++ b/docs/en/docs/side_quests/platform_automation/index.md @@ -511,4 +511,4 @@ You drove the Seqera Platform programmatically across three roles, each handing ## What's next? -- The AI half of the workshop is the CoScientist side quest in [nextflow-io/training](https://github.com/nextflow-io/training), published at training.nextflow.io. It uses the same `rnaseq-nf` pipeline and API endpoints, but interacts with them via AI agents. +- The AI half of the workshop is the [Building with Seqera AI (CoScientist)](../co_scientist/index.md) side quest. It uses the same `rnaseq-nf` pipeline and API endpoints, but drives them via AI agents. From f9a5ef7c574be554149b8740dbb4bfbc6b4756a4 Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:47:25 +0100 Subject: [PATCH 5/8] fix(devcontainer): install tw via JAR on arm64 tower-cli publishes no tw-linux-arm64 asset, so the multi-arch image build 404'd on the arm64 leg. Keep the native binary on x86_64 and fall back to the arch-independent tw-jar.jar (run via Java) on arm64. Generated by Claude Code --- .../local-features/seqera-cli/install.sh | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/.devcontainer/local-features/seqera-cli/install.sh b/.devcontainer/local-features/seqera-cli/install.sh index 6b6d3e4add..184af807eb 100644 --- a/.devcontainer/local-features/seqera-cli/install.sh +++ b/.devcontainer/local-features/seqera-cli/install.sh @@ -2,26 +2,38 @@ # Install the Seqera Platform CLI (tw). # Arch-aware so the multi-architecture image build works on both amd64 and arm64. -# Only downloads the binary (no execution), so it is safe under QEMU emulation. +# tower-cli only publishes a native Linux binary for x86_64; on arm64 we install +# the architecture-independent JAR (Java is provided by the java feature). +# Only downloads files (no execution), so it is safe under QEMU emulation. set -eu VERSION="${VERSION:-0.32.0}" +if [ "${VERSION}" = "latest" ]; then + BASE="https://github.com/seqeralabs/tower-cli/releases/latest/download" +else + BASE="https://github.com/seqeralabs/tower-cli/releases/download/v${VERSION}" +fi + case "$(uname -m)" in - x86_64 | amd64) ARCH="x86_64" ;; - aarch64 | arm64) ARCH="arm64" ;; + x86_64 | amd64) + echo "Installing Seqera CLI (tw) ${VERSION} native binary for x86_64..." + curl -fSL "${BASE}/tw-linux-x86_64" -o /usr/local/bin/tw + chmod +x /usr/local/bin/tw + ;; + aarch64 | arm64) + # No native Linux arm64 binary is published; use the JAR via Java. + echo "Installing Seqera CLI (tw) ${VERSION} JAR for arm64..." + mkdir -p /usr/local/lib/tw + curl -fSL "${BASE}/tw-jar.jar" -o /usr/local/lib/tw/tw.jar + cat > /usr/local/bin/tw <<'EOF' +#!/usr/bin/env bash +exec java -jar /usr/local/lib/tw/tw.jar "$@" +EOF + chmod +x /usr/local/bin/tw + ;; *) echo "Unsupported architecture: $(uname -m)" >&2 exit 1 ;; esac - -if [ "${VERSION}" = "latest" ]; then - URL="https://github.com/seqeralabs/tower-cli/releases/latest/download/tw-linux-${ARCH}" -else - URL="https://github.com/seqeralabs/tower-cli/releases/download/v${VERSION}/tw-linux-${ARCH}" -fi - -echo "Installing Seqera CLI (tw) ${VERSION} for ${ARCH}..." -curl -fSL "${URL}" -o /usr/local/bin/tw -chmod +x /usr/local/bin/tw From 7a190c7002a2d2511b1fc13866fa14fe0329f1bf Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Wed, 17 Jun 2026 12:25:43 +0100 Subject: [PATCH 6/8] docs(platform_automation): orient participants and cd into project dir Section 0 jumped from opening the Codespace straight to creating a token, never walking participants into the working directory or showing its contents. Add "Move into the project directory" (cd + code .) and "Review the materials" (directory tree) subsections, matching the debugging side quest pattern. Generated by Claude Code --- .../side_quests/platform_automation/index.md | 67 ++++++++++++++----- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/docs/en/docs/side_quests/platform_automation/index.md b/docs/en/docs/side_quests/platform_automation/index.md index 3dd605338f..eecf74e9a0 100644 --- a/docs/en/docs/side_quests/platform_automation/index.md +++ b/docs/en/docs/side_quests/platform_automation/index.md @@ -10,8 +10,8 @@ In this side quest, we'll drive the Platform across three workspace roles of dec - Create a compute environment two ways, with increasing control over the cloud: the UI (Batch Forge) and Terraform - Add a pipeline to the Launchpad declaratively with Terraform, and see idempotency -- Launch a pipeline using the GUI, CLI and seqerakit. -- Tell declarative existence (Terraform) apart from imperative actions (the UI, seqerakit, `tw`) and learn when to use each +- Launch a pipeline using the GUI, CLI and `seqerakit`. +- Tell declarative existence (Terraform) apart from imperative actions (the UI, `seqerakit`, `tw`) and learn when to use each ### Prerequisites @@ -23,7 +23,7 @@ Before taking on this side quest, you should: - Ideally, a GitHub access token added too, to avoid GitHub rate limits - Be comfortable with the command line and basic Nextflow concepts -New to running pipelines on Seqera at all? Start with the gentler "Run pipelines on Seqera" module in the [Nextflow Triathlon](https://training.nextflow.io/) (sign up, launch in the UI, the `tw` CLI). This side quest is the automation layer on top of it: roles, Terraform, seqerakit, and Actions. +New to running pipelines on Seqera at all? Start with the gentler "Run pipelines on Seqera" module in the [Nextflow Triathlon](https://training.nextflow.io/) (sign up, launch in the UI, the `tw` CLI). This side quest is the automation layer on top of it: roles, Terraform, `seqerakit`, and Actions. --- @@ -31,15 +31,50 @@ New to running pipelines on Seqera at all? Start with the gentler "Run pipelines ### Open the training codespace -The Codespace contains all the tools you need (Terraform, `tw`, seqerakit); you +The Codespace contains all the tools you need (Terraform, `tw`, `seqerakit`); you install nothing yourself. Open it now and read on while it builds; it is ready when the terminal returns to a prompt. [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/nextflow-io/training?quickstart=1&ref=master) -The assets for this side quest live under `side-quests/platform_automation/`: -`terraform/compute-env/`, `terraform/pipeline/`, and `seqerakit/`. Each section -below tells you where to `cd`. +### Move into the project directory + +The Codespace terminal opens at the repository root (`/workspaces/training`). All the assets for this side quest live under `side-quests/platform_automation/`, so move there now: + +```bash +cd /workspaces/training/side-quests/platform_automation +``` + +Focus VSCode on this directory so the file explorer shows the assets you'll edit: + +```bash +code . +``` + +### Review the materials + +The directory holds the Terraform and `seqerakit` configurations for each role. Each section below tells you which subdirectory to `cd` into: + +??? abstract "Directory contents" + + ```console + . + ├── terraform + │ ├── compute-env # section 1 (Admin): provision the cloud + compute environment + │ │ ├── main.tf + │ │ ├── variables.tf + │ │ ├── terraform.tfvars.example + │ │ └── README.md + │ └── pipeline # section 2 (Maintain): add a pipeline to the Launchpad + │ ├── main.tf + │ ├── variables.tf + │ ├── terraform.tfvars.example + │ └── README.md + └── seqerakit # sections 2 & 3: add and launch a pipeline + ├── add-rnaseq.yml + ├── launch-rnaseq.yml + └── README.md + ``` ### Create an access token @@ -55,7 +90,7 @@ export TOWER_ACCESS_TOKEN= export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN ``` -Terraform, `tw`, and seqerakit all read the token from these variables. Check it worked: +Terraform, `tw`, and `seqerakit` all read the token from these variables. Check it worked: ```bash tw info @@ -86,7 +121,7 @@ tw workspaces list This prints a table with the workspace ID, the workspace name, and the organization name. -Two forms of the same workspace turn up in this module. Terraform and the API want the **numeric** ID (`workspace_id`). `tw` and seqerakit accept either the numeric ID or the `Organization/Workspace` **name** (e.g. `my-org/platform-automation`). Export the numeric ID once so `tw` targets the shared workspace without a `--workspace` flag on every command: +Two forms of the same workspace turn up in this module. Terraform and the API want the **numeric** ID (`workspace_id`). `tw` and `seqerakit` accept either the numeric ID or the `Organization/Workspace` **name** (e.g. `my-org/platform-automation`). Export the numeric ID once so `tw` targets the shared workspace without a `--workspace` flag on every command: ```bash export TOWER_WORKSPACE_ID= @@ -372,7 +407,7 @@ tw pipelines add \ Run it again and `tw` errors: a pipeline with that name already exists. The command is imperative, so each invocation tries to add a pipeline; it has no notion of "already in the desired state". -### 2.3. Add a pipeline with seqerakit +### 2.3. Add a pipeline with `seqerakit` `seqerakit` is a wrapper over `tw` that reads a YAML file and runs the underlying `tw` commands. It keeps the configuration as code, so a teammate can reproduce the exact same pipeline. `seqerakit/add-rnaseq.yml` describes the pipeline: @@ -498,14 +533,14 @@ You drove the Seqera Platform programmatically across three roles, each handing ### Key patterns -- **Everything is one API.** The GUI, Terraform, `tw`, and seqerakit all call the same Platform API. Anything you can click, you can automate. +- **Everything is one API.** The GUI, Terraform, `tw`, and `seqerakit` all call the same Platform API. Anything you can click, you can automate. - **Use the right tool.** Terraform manages resources as state: what should exist, where. `tw` and `seqerakit` act on the Platform imperatively: they do the thing, now. -| | Terraform | seqerakit / tw / Action | -| ------------ | ------------- | ----------------------- | -| Manages | existence | actions | -| Run twice | no-op | two runs | -| Mental model | desired state | do the thing, now | +| | Terraform | `seqerakit` / `tw` / Action | +| ------------ | ------------- | --------------------------- | +| Manages | existence | actions | +| Run twice | no-op | two runs | +| Mental model | desired state | do the thing, now | - **Roles stratify what each token can do.** A Maintain token defines pipelines and Actions; a Launch token can only trigger an Action and nothing else. The Action is the safe handoff from maintainers to launchers (and to automation). From 09082f876f0e450dc871f922bac6511c6364d273 Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:31:15 +0100 Subject: [PATCH 7/8] docs(platform_automation): align doc with assets, fix exercise, unify workshop handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review pass on the Platform Automation side quest: - Reset terraform/pipeline revision to master so the §2.4 pin-a-tag exercise starts from the right state - Sync §1.3 walkthrough with compute-env/main.tf (head VM size, worker ignore_changes, random_string, a note on the azcopy/Fusion start task) - Replace the $USER/$USERNAME interpolation mess with a single, dot-free WORKSHOP_USER handle set once and reused across the UI, tw, seqerakit and Terraform; tighten the Terraform username validation to reject dots - Register the page in mkdocs nav; refresh the §0 directory tree - Add seqerakit/launch-rnaseq-multiple.yml; drop per-directory READMEs - gitignore agent/terraform local state (.omc, .claude/.local, lock files) Generated by Claude Code --- .gitignore | 5 + .../side_quests/platform_automation/index.md | 371 +++++++++++++----- .../platform_automation/seqerakit/README.md | 55 --- .../seqerakit/add-rnaseq.yml | 15 +- .../seqerakit/launch-rnaseq-multiple.yml | 24 ++ .../seqerakit/launch-rnaseq.yml | 16 +- .../terraform/compute-env/README.md | 66 ---- .../terraform/compute-env/main.tf | 148 ++++--- .../compute-env/terraform.tfvars.example | 24 +- .../terraform/compute-env/variables.tf | 52 +-- .../terraform/pipeline/README.md | 66 ---- .../terraform/pipeline/main.tf | 5 +- .../pipeline/terraform.tfvars.example | 9 +- .../terraform/pipeline/variables.tf | 13 +- 14 files changed, 449 insertions(+), 420 deletions(-) delete mode 100644 side-quests/platform_automation/seqerakit/README.md create mode 100644 side-quests/platform_automation/seqerakit/launch-rnaseq-multiple.yml delete mode 100644 side-quests/platform_automation/terraform/compute-env/README.md delete mode 100644 side-quests/platform_automation/terraform/pipeline/README.md diff --git a/.gitignore b/.gitignore index 7ba8953eab..9bf9616195 100644 --- a/.gitignore +++ b/.gitignore @@ -30,9 +30,14 @@ hello-nf-core/core-hello/ # Terraform (platform_automation side quest) **/.terraform/* +.terraform.lock.hcl *.tfstate *.tfstate.* *.tfvars !*.tfvars.example crash.log crash.*.log + +# Agent/tooling local state +.omc/ +**/.claude/.local/ diff --git a/docs/en/docs/side_quests/platform_automation/index.md b/docs/en/docs/side_quests/platform_automation/index.md index eecf74e9a0..659cb48781 100644 --- a/docs/en/docs/side_quests/platform_automation/index.md +++ b/docs/en/docs/side_quests/platform_automation/index.md @@ -63,17 +63,15 @@ The directory holds the Terraform and `seqerakit` configurations for each role. │ ├── compute-env # section 1 (Admin): provision the cloud + compute environment │ │ ├── main.tf │ │ ├── variables.tf - │ │ ├── terraform.tfvars.example - │ │ └── README.md + │ │ └── terraform.tfvars.example │ └── pipeline # section 2 (Maintain): add a pipeline to the Launchpad │ ├── main.tf │ ├── variables.tf - │ ├── terraform.tfvars.example - │ └── README.md + │ └── terraform.tfvars.example └── seqerakit # sections 2 & 3: add and launch a pipeline ├── add-rnaseq.yml ├── launch-rnaseq.yml - └── README.md + └── launch-rnaseq-multiple.yml ``` ### Create an access token @@ -127,6 +125,16 @@ Two forms of the same workspace turn up in this module. Terraform and the API wa export TOWER_WORKSPACE_ID= ``` +#### Pick a workshop handle + +In a shared workspace, every pipeline you add needs a name no one else is using. Pick a short handle and export it once; everything from section 2 onward reuses this one variable. Use only letters, numbers, and hyphens. Launchpad names cannot contain dots or spaces: + +```bash +export WORKSHOP_USER= # e.g. ada, jdoe, team-rocket +``` + +We set this by hand rather than reading `$USER`: in the Codespace everyone shares the same Linux user, so `$USER` is identical for all learners and would collide. + #### Know your role The work is split by role, from most to least privileged. Start at the highest tier your access allows: @@ -179,28 +187,92 @@ Watch your cloud console and you will see Forge create the resources: an identit That convenience is the trade-off: Forge owns those resources and gives you little control. A lab that already runs on managed infrastructure wants the opposite, to describe the resources itself and wire the Platform to them. That is the Terraform path. -### 1.2. Provision the cloud with Terraform +### 1.2. Read it back over the API with `tw` + +The form you just filled in did nothing special: it sent a configuration to the Platform API. +Ask the API for what you made, from the terminal, with nothing retyped. + +`tw` reads the same workspace the UI does. List the compute environments and the one you created in 1.1 is there: + +```bash +tw compute-envs list +``` + +Now export its full configuration to a file. +Use the name you gave it in 1.1: + +```bash +tw compute-envs export forge-ce.json --name="" +``` + +`tw` reads the workspace from `TOWER_WORKSPACE_ID` (set in section 0), so no `--workspace` flag is needed. +The file `forge-ce.json` is the compute environment as the API stores it: + +```json title="forge-ce.json" +{ + "computeEnv": { + "name": "azure-batch", + "platform": "azure-batch", + "config": { + "workDir": "az://work", + "region": "eastus", + "waveEnabled": true, + "fusion2Enabled": true, + "forge": { + "headPool": { + "vmType": "Standard_D2s_v3", + "vmCount": 1, + "autoScale": true + }, + "workerPool": { + "vmType": "Standard_D4s_v3", + "vmCount": 4, + "autoScale": true + }, + "disposeOnDeletion": true + } + } + } +} +``` + +The `forge` block is exactly what the checkboxes set in 1.1. Its presence is what tells the Platform to create and scale the pools for you. In the next section, the absence of this block is what makes the Terraform compute environment manual. + +Close the loop: import the file to recreate the compute environment, with nothing retyped. + +```bash +tw compute-envs import forge-ce.json --name="azure-batch-copy" +``` + +The UI sent this JSON to create the compute environment; `tw export` reads it back, and `tw import` sends it again. The GUI and the CLI are two clients of the same API. + +The exported JSON does not include a credentials ID. If your workspace has more than one credential for this cloud, `tw` cannot tell which to use and fails with `Multiple credentials match this compute environment`. Name one explicitly with `-c` (see `tw credentials list`): + +```bash +tw compute-envs import forge-ce.json --name="azure-batch-copy" --credentials="" +``` + +### 1.3. Provision the cloud with Terraform Terraform manages cloud resources declaratively: you describe what should exist and Terraform makes it so. It does not run your work; it only creates the compute environment. `side-quests/platform_automation/terraform/compute-env/` does it in a single `main.tf`, two providers in one apply: -- data sources: the existing Batch account, managed identity, and vnet/subnet, - referenced, not created. -- `azurerm`: creates a head pool and a worker pool on that account and subnet, - as that identity. -- `seqera`: stores the Azure credentials and creates the compute environment. +- data sources: the existing Batch account and the Azure credentials already + stored in the Platform, referenced, not created. +- `azurerm`: creates a head pool and a worker pool on that account. +- `seqera`: creates the compute environment pointing at those pools. The manual marker: `head_pool` is set to the pool Terraform just made and there is **no `forge` block**. That one difference is what makes the compute environment manual instead of Forge. `nextflow_config` routes tasks to the worker pool. -Let's walk `terraform/compute-env/main.tf` from top to bottom. +Let's walk the key blocks of `terraform/compute-env/main.tf`. -The first block declares the providers and pins their versions. The `seqera` provider is pinned to exactly `0.30.5`: +The first block declares the providers and pins their versions. The `seqera` provider is pinned to exactly `0.40.1`: ```terraform terraform { required_version = ">= 1.9" required_providers { - seqera = { source = "seqeralabs/seqera", version = "0.30.5" } + seqera = { source = "seqeralabs/seqera", version = "0.40.1" } azurerm = { source = "hashicorp/azurerm", version = ">= 3.0" } random = { source = "hashicorp/random", version = ">= 3.0" } } @@ -220,47 +292,56 @@ provider "seqera" { } ``` -Pool sizing and the VM image live in a `locals` block, so you edit them in one place rather than wiring every value through as a variable: +Next we can define some variables we can use throughout our deployment. In this case, we will set the node sizes and details once and reuse them throughout the configuration: ```terraform locals { - head_vm_size = "Standard_D4ds_v5" + head_vm_size = "Standard_E4ds_v5" worker_vm_size = "Standard_E16ds_v5" worker_max_nodes = 8 - worker_max_tasks = 16 - node_agent_sku_id = "batch.node.ubuntu 22.04" + node_agent_sku_id = "batch.node.ubuntu 24.04" + + # The Platform requires a pool's task slots to equal the node's vCPU count. + # Azure VM size names embed that count (Standard_E16ds_v5 -> 16), so derive it + # from the size rather than hardcoding a number that can drift out of sync. + head_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.head_vm_size)[0]) + worker_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.worker_vm_size)[0]) } ``` -The cloud resources the lab already owns, the Batch account, the managed identity, and the subnet, are referenced with data sources. Terraform reads them; it does not create or manage them: +Next we can refer to resources that already exist. These are called `data` in Terraform. Here we refer to the Azure Batch account: ```terraform +# Existing Azure resources the customer owns. Referenced, not managed. data "azurerm_batch_account" "existing" { name = var.batch_account_name resource_group_name = var.batch_account_rg } +``` -data "azurerm_user_assigned_identity" "existing" { - name = var.managed_identity_name - resource_group_name = var.managed_identity_rg -} +Now the resources Terraform does create. The pool names must stay unique across re-creates, so we first generate a short random suffix to append to them: -data "azurerm_subnet" "existing" { - name = var.subnet_name - virtual_network_name = var.vnet_name - resource_group_name = var.vnet_rg +```terraform +# Keeps pool names unique across re-creates. +resource "random_string" "suffix" { + length = 6 + special = false + upper = false } ``` -Now the resources Terraform does create. The head pool runs the Nextflow head job, so one fixed node is enough: +Then the node pools themselves. Terraform creates two pools in Azure Batch with the `azurerm_batch_pool` resource. You can add as many as you like, but the details must match the values the Azure provider expects. Here we start a fixed-size head pool and a dynamically sized worker pool: ```terraform +# Head pool: runs the Nextflow head job. One node is enough. resource "azurerm_batch_pool" "head" { name = "rnaseq-head-${random_string.suffix.result}" + display_name = "rnaseq head pool" resource_group_name = var.batch_account_rg account_name = var.batch_account_name vm_size = local.head_vm_size node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.head_vm_cores fixed_scale { target_dedicated_nodes = 1 @@ -269,22 +350,24 @@ resource "azurerm_batch_pool" "head" { storage_image_reference { publisher = "microsoft-dsvm" offer = "ubuntu-hpc" - sku = "2204" + sku = "2404" version = "latest" } - # identity, container, and network_configuration omitted for brevity -} -``` -The worker pool runs the pipeline tasks, so it autoscales from zero up to `worker_max_nodes` based on the number of pending tasks: + container_configuration { + type = "DockerCompatible" + } +} -```terraform +# Worker pool: runs the pipeline tasks. Autoscales 0..worker_max_nodes. resource "azurerm_batch_pool" "worker" { - name = "rnaseq-worker-${random_string.suffix.result}" - vm_size = local.worker_vm_size - node_agent_sku_id = local.node_agent_sku_id - max_tasks_per_node = local.worker_max_tasks - # ...same account, image, identity, and network as the head pool + name = "rnaseq-worker-${random_string.suffix.result}" + display_name = "rnaseq worker pool" + resource_group_name = var.batch_account_rg + account_name = var.batch_account_name + vm_size = local.worker_vm_size + node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.worker_vm_cores auto_scale { evaluation_interval = "PT5M" @@ -294,25 +377,44 @@ resource "azurerm_batch_pool" "worker" { $NodeDeallocationOption = taskcompletion; FORMULA } + + storage_image_reference { + publisher = "microsoft-dsvm" + offer = "ubuntu-hpc" + sku = "2404" + version = "latest" + } + + container_configuration { + type = "DockerCompatible" + } + + # Terraform manages the pool, not its live scale. + lifecycle { + ignore_changes = [auto_scale, fixed_scale] + } } ``` -Both pools run the same `ubuntu-hpc` `2204` image, which is why `node_agent_sku_id` is `batch.node.ubuntu 22.04` for both. +Each pool in `main.tf` also carries a `start_task` (elided above) that runs once per node at startup. Forge adds an equivalent task to the pools it creates; manual pools get nothing, so we replicate the essential parts: install `azcopy` (Nextflow's Azure Batch executor uses it to stage data) and register the AppArmor profile Fusion needs on Ubuntu 24.04 nodes. See the `node_start_task_script` local in the file for the full script. -The `seqera` provider stores the Azure credentials in the Platform: +Once we've built the Azure Batch node pools, we add the Seqera Platform compute environment that points to them. The compute environment needs Azure credentials to talk to your cloud. We added those to the workspace earlier (see Prerequisites), so we look them up by name rather than storing keys here. The `seqera_credentials` data source lists the credentials in the workspace, and we pick out the one whose name matches: ```terraform -resource "seqera_azure_credential" "main" { - name = "azure-batch" +# Azure credentials already stored in the Platform, looked up by name. +data "seqera_credentials" "all" { workspace_id = var.workspace_id - batch_name = var.batch_account_name - batch_key = var.azure_batch_key - storage_name = var.azure_storage_name - storage_key = var.azure_storage_key +} + +locals { + azure_credentials_id = one([ + for c in data.seqera_credentials.all.credentials : c.id + if c.name == var.azure_credential_name + ]) } ``` -Finally, the compute environment itself. This is the manual marker in code: `head_pool` points at the pool we just created, there is no `forge` block, and `nextflow_config` routes tasks to the worker pool's queue. Because it references `azurerm_batch_pool.head.name` and `azurerm_batch_pool.worker.name`, Terraform knows to create the pools first: +Finally, the compute environment itself. This is the manual marker in code: `head_pool` points at the pool we just created and `nextflow_config` routes tasks to the worker pool's queue. `enable_wave` and `enable_fusion` turn on Wave (container provisioning) and the Fusion file system, the same checkboxes you would tick in the UI; with Fusion enabled, Wave must be too. Because it references `azurerm_batch_pool.head.name` and `azurerm_batch_pool.worker.name`, Terraform knows to create the pools first: ```terraform resource "seqera_compute_env" "main" { @@ -320,16 +422,18 @@ resource "seqera_compute_env" "main" { compute_env = { name = "azure-batch-manual" + description = "Azure Batch CE on Terraform-managed pools" platform = "azure-batch" - credentials_id = seqera_azure_credential.main.credentials_id + credentials_id = local.azure_credentials_id config = { azure_batch = { - region = var.region - work_dir = var.work_dir - head_pool = azurerm_batch_pool.head.name - managed_identity_client_id = data.azurerm_user_assigned_identity.existing.client_id - nextflow_config = "process.queue = '${azurerm_batch_pool.worker.name}'\n" + region = var.region + work_dir = var.work_dir + head_pool = azurerm_batch_pool.head.name + enable_wave = true + enable_fusion = true + nextflow_config = "process.queue = '${azurerm_batch_pool.worker.name}'\n" } } } @@ -344,19 +448,26 @@ output "compute_env_id" { } ``` -Terraform reads those references and works out the order itself: credentials and pools first, then the compute environment that depends on them. Run it: +Terraform reads those references and works out the order itself: pools first, then the compute environment that depends on them. + +Now, we can run Terraform to apply these changes. ```bash -az login -cd /workspaces/training/side-quests/platform_automation/terraform/compute-env +# you may need to log in to the cloud provider with `az login` terraform init -terraform plan terraform apply ``` -See `side-quests/platform_automation/terraform/compute-env/README.md` for the full variable list and the `terraform.tfvars` setup. +After you run `terraform apply`, it will ask you to fill in the details defined in `variables.tf` + +!!! tip "Tearing it down" + + You can save them as a .tfvars file and Terraform will read them automatically, so you don't have to type them each time. See `terraform.tfvars.example` for the full variable reference. + Alternatively, you can save them as an environment variable preceded by `TF_VAR_`, for example `TF_VAR_subscription_id=00000000-0000-0000-0000-000000000000`. -### 1.3. See both sides +Terraform will show you a preview, if it looks accurate, type `yes` to apply the changes. It will create the pools and the compute environment, and print the `compute_env_id` you hand to the Maintain tier. + +### 1.4. See both sides The compute environment now exists in two places, and as an Admin with cloud access you can see both. @@ -370,7 +481,11 @@ On the Azure side, open the Batch account in the portal. You will see the head a ### Takeaway -One compute environment, two levels of control: Forge in the UI (easiest, the Platform owns the pool) and Terraform (you own the pools and the wiring, end to end). The artifact you hand to the Maintain tier is a `compute_env_id`. +One compute environment, three levels of control: Forge in the UI (easiest, the Platform owns the pool), `tw` (export and import the same config over the API), and Terraform (you own the pools and the wiring, end to end). The artifact you hand to the Maintain tier is a `compute_env_id`. + +!!! note "Tearing it down" + + Terraform manages the cloud resources it created, so it can remove them too. From `terraform/compute-env`, run `terraform apply -destroy` to delete the compute environment and the Batch pools in one step. The existing Batch account and credentials are referenced, not managed, so they are left untouched. --- @@ -378,83 +493,91 @@ One compute environment, two levels of control: Forge in the UI (easiest, the Pl **Requires:** Maintain role, and a `compute_env_id` (from section 1 or an Admin on your team) plus the numeric `workspace_id`. Maintain manages pipelines, not compute environments. That split is deliberate: Admin owns the compute environment, you own your pipelines. -You add the same pipeline, `rnaseq-nf-$USER`, four ways. The first three are imperative, you run a command and it acts. The last, Terraform, is declarative. Watch what happens when you run each one twice. +You add the same pipeline, `rnaseq-nf-$WORKSHOP_USER`, four ways. The first three are imperative, you run a command and it acts. The last, Terraform, is declarative. Watch what happens when you run each one twice. ### 2.1. Add a pipeline via the UI In the workspace, open the **Launchpad** and click **Add pipeline**. Fill in the form: -- **Name**: `rnaseq-nf-` (unique in a shared workspace). +- **Name**: `rnaseq-nf-` followed by your handle, e.g. `rnaseq-nf-ada` (the handle you exported as `WORKSHOP_USER` in section 0). - **Compute environment**: the one from section 1, e.g. `azure-batch-manual`. - **Pipeline to launch**: `https://github.com/nextflow-io/rnaseq-nf`. - **Revision**: `master`. -- **Work directory**: your Azure Blob work dir, e.g. `az://nf-work/work`. +- **Work directory**: your Azure Blob work dir, e.g. `az://work`. Click **Add**. The pipeline appears on the Launchpad with no run started. Adding a pipeline only saves a launch configuration; it does not run anything. +To remove from the workspace, you can click the hamburger menu on the top right and click **Delete**. + ### 2.2. Add a pipeline via `tw` The CLI does the same thing in one command: ```bash tw pipelines add \ - --name="rnaseq-nf-$USER" \ + --name="rnaseq-nf-$WORKSHOP_USER" \ --compute-env="azure-batch-manual" \ - --work-dir="az://nf-work/work" \ + --work-dir="az://work" \ --revision="master" \ https://github.com/nextflow-io/rnaseq-nf ``` Run it again and `tw` errors: a pipeline with that name already exists. The command is imperative, so each invocation tries to add a pipeline; it has no notion of "already in the desired state". +To remove the pipeline, you can use the `tw` command line again: + +```bash +tw pipelines delete --name="rnaseq-nf-$WORKSHOP_USER" +``` + ### 2.3. Add a pipeline with `seqerakit` `seqerakit` is a wrapper over `tw` that reads a YAML file and runs the underlying `tw` commands. It keeps the configuration as code, so a teammate can reproduce the exact same pipeline. `seqerakit/add-rnaseq.yml` describes the pipeline: ```yaml pipelines: - - name: "rnaseq-nf-${USERNAME}" + - name: "rnaseq-nf-${WORKSHOP_USER}" url: "https://github.com/nextflow-io/rnaseq-nf" - workspace: "my-org/platform-automation" - description: "rnaseq-nf for ${USERNAME}, added with seqerakit" + workspace: "${TOWER_WORKSPACE_ID}" + description: "Added with seqerakit" compute-env: "azure-batch-manual" - work-dir: "az://nf-work/work" + work-dir: "az://work" revision: "master" ``` -`seqerakit` expands `${USERNAME}` from the environment, so set it first, then add the pipeline: +`seqerakit` expands variables like `${TOWER_WORKSPACE_ID}` and `${WORKSHOP_USER}` from the environment. Both were set in section 0, so just add the pipeline: ```bash cd /workspaces/training/side-quests/platform_automation/seqerakit -export USERNAME=$USER seqerakit add-rnaseq.yml ``` If you run it again, it errors, the same way `tw` did, because the pipeline already exists: ```console -ERROR: A pipeline with name 'rnaseq-nf-' already exists. +The pipelines resource already exists and will not be created. Please set 'on_exists: overwrite' to replace the resource or set 'on_exists: ignore' to ignore this error. ``` You can force it through with `seqerakit add-rnaseq.yml --overwrite`, which deletes and recreates the pipeline. But that is you telling it to repeat the action. By default, adding twice is an error. To make "add once, and leave it alone after that" the _default_ behaviour, we need a tool that manages existence rather than actions: Terraform. ### 2.4. Add a pipeline with Terraform -`side-quests/platform_automation/terraform/pipeline` adds the same pipeline declaratively. You describe the pipeline that should exist; Terraform makes the workspace match: +`side-quests/platform_automation/terraform/pipeline` adds the same pipeline declaratively. You describe the pipeline that should exist; Terraform makes the workspace match. Like the imperative methods above, the config tracks the `master` branch; you will pin it to a release tag below to see Terraform update the pipeline in place. ```bash cd /workspaces/training/side-quests/platform_automation/terraform/pipeline terraform init -terraform plan -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" -terraform apply -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" +terraform apply -var="username=$WORKSHOP_USER" -var="workspace_id=" -var="compute_env_id=" ``` -Tip: copy `terraform.tfvars.example` to `terraform.tfvars` so you stop passing `-var` flags. +!!! Tip + + copy `terraform.tfvars.example` to `terraform.tfvars` and fill it in so you stop passing `-var` flags. -Open the Launchpad: `rnaseq-nf-$USER` is there, with no run. Now apply again: +Open the Launchpad: `rnaseq-nf-$WORKSHOP_USER` is there, with no run. Now apply again: ```bash -terraform apply -var="username=$USER" -var="workspace_id=" -var="compute_env_id=" +terraform apply -var="username=$WORKSHOP_USER" -var="workspace_id=" -var="compute_env_id=" ``` ```console @@ -463,9 +586,59 @@ No changes. Your infrastructure matches the configuration. That is the difference. `tw` and `seqerakit` errored on the second run because they perform an action every time. Terraform manages whether the pipeline **exists**: it is already there, so there is nothing to do. Apply ten times, still one pipeline. This is what keeps you from accumulating competing, half-duplicated resources in a shared workspace. To remove the pipeline, `terraform apply -destroy` with the same vars. +**Update in place.** Terraform does more than create-or-skip; it reconciles the pipeline to whatever the config says. The config tracks the `master` branch, which moves as the pipeline gets new commits. Pin it to a fixed release tag instead, so every launch runs the same code. Edit one line: in `main.tf`, line 35 of the `launch` block, change the revision from the `master` branch to the `v2.4` release tag. + +=== "After" + + ```hcl title="main.tf" hl_lines="4" linenums="32" + launch = { + pipeline = "https://github.com/nextflow-io/rnaseq-nf" + # The master branch moves; pin a release tag (e.g. v2.4) for reproducible launches. + revision = "v2.4" + compute_env_id = var.compute_env_id + work_dir = var.work_dir + } + ``` + +=== "Before" + + ```hcl title="main.tf" hl_lines="4" linenums="32" + launch = { + pipeline = "https://github.com/nextflow-io/rnaseq-nf" + # The master branch moves; pin a release tag (e.g. v2.4) for reproducible launches. + revision = "master" + compute_env_id = var.compute_env_id + work_dir = var.work_dir + } + ``` + +Then apply: + +```console +Terraform will perform the following actions: + + # seqera_pipeline.rnaseq_nf will be updated in-place + ~ resource "seqera_pipeline" "rnaseq_nf" { + ~ launch = { + ~ revision = "master" -> "v2.4" + # (4 unchanged attributes hidden) + } + } + +Plan: 0 to add, 1 to change, 0 to destroy. +``` + +One pipeline, modified; not a second pipeline, and not an error. That is the third declarative behaviour, alongside create and no-op: update to match. Moving from a branch to a release tag is exactly the kind of change you want under version control: the pinned revision is now recorded in `main.tf`. + +!!! note "Pinning a revision vs. Launchpad versions" + + The Platform also has a separate concept of **Launchpad pipeline versions**: several saved launch configurations on one pipeline, with one marked as the default (most recent). You can use this to maintain and control pipeline versions within one concept of a "pipeline". + + The Terraform provider can also publish and promote those versions, and flag drift if someone changes the default in the UI. See the provider's [pipeline versioning guide](https://github.com/seqeralabs/terraform-provider-seqera/blob/master/docs/guides/pipeline-versioning.md). + ### Takeaway -Four ways to add one pipeline, two mental models. `tw` and `seqerakit` are imperative: each run is an action, and adding twice is an error. Terraform is declarative: it manages existence, so a second apply is a no-op. The artifact you hand to the Launch tier is the Launchpad pipeline `rnaseq-nf-$USER`. +Four ways to add one pipeline, two mental models. `tw` and `seqerakit` are imperative: each run is an action, and adding twice is an error. Terraform is declarative: it manages existence, so a second apply is a no-op or an update-in-place. The artifact you hand to the Launch tier is the Launchpad pipeline `rnaseq-nf-$WORKSHOP_USER`. --- @@ -477,35 +650,53 @@ Everything in this section runs the pipeline the Maintainer already configured. ### 3.1. Use the GUI -Open the **Launchpad**, select `rnaseq-nf-$USER`, and click **Launch**. The compute environment, revision, and work directory are already filled in by the Maintainer, so a Launch user just clicks the button. Submit it and the run appears under **Runs**. +Open the **Launchpad**, select `rnaseq-nf-$WORKSHOP_USER`, and click **Launch**. The compute environment, revision, and work directory are already filled in by the Maintainer, so a Launch user just clicks the button. Submit it and the run appears under **Runs**. ### 3.2. Use the `tw` CLI ```bash -tw launch rnaseq-nf-$USER +tw launch rnaseq-nf-$WORKSHOP_USER +``` + +It should show something like: + +```console + Workflow 2ZXaU1AzEn7Onk submitted at [Organization / Workspace] workspace. + + https://cloud.seqera.io/orgs/Organization/workspaces/Workspace/watch/2ZXaU1AzEn7Onk ``` -One command, no flags: everything is pre-configured on the Launchpad entry. The CLI submits the run and prints its URL. +You can monitor the run by clicking the provided URL. -For an automated launch, pin the parameters in a version-controlled file and pass it with `--params-file params.yaml`. That keeps each run reproducible, which is the whole reason to drive launches from code rather than the form. +!!! Note Using a params.yml + + If you wish to provide parameters to the pipeline, you can do so with the `--params-file` flag. ### 3.3. Use `seqerakit` -`seqerakit` launches a pipeline that already exists on the Launchpad, filling in the `tw launch` command from `seqerakit/launch-rnaseq.yml`. Set `USERNAME`, dry run first, then launch: +`seqerakit` launches a pipeline that already exists on the Launchpad, filling in the `tw launch` command from `seqerakit/launch-rnaseq.yml`. `WORKSHOP_USER` is set from section 0; set the compute environment name, dry run first, then launch: ```bash cd /workspaces/training/side-quests/platform_automation/seqerakit -export USERNAME=$USER +export COMPUTE_ENVIRONMENT= seqerakit launch-rnaseq.yml --dryrun # prints the tw command, changes nothing seqerakit launch-rnaseq.yml # launches for real ``` -The dry run shows the underlying `tw` command. Run it twice and you get two runs. That is the imperative model: do the thing, now. It is the opposite of Terraform (section 2.4), which manages existence. See `side-quests/platform_automation/seqerakit/README.md`. +The dry run shows the underlying `tw` command. Run it twice and you get two runs. That is the imperative model: do the thing, now. + +One advantage of using Seqerakit is the YAML forms a template of actions to perform. Because of this we can do two more things: firstly, we can save it and re-use it later. Secondly, we can launch the same pipeline several times by adding more launch blocks to the YAML file. This is useful if you want to launch the same pipeline with different parameters or on different compute environments. Let's try and launch the pipeline now with + +```bash +seqerakit launch-rnaseq-multiple.yml +``` + +### 3.4. Side note: Examine the cloud resources -### 3.4. Examine the cloud resources +It's worth taking a second to examine the cloud resources here. -A run does not create one neatly named cloud job. Nextflow submits **many** Batch jobs and tasks, one per process invocation, so you will not find a single resource named after the Platform run. Open the run on the Platform (**Runs** → your run), which mirrors the underlying Batch service: the task table here is the same work you can see in the cloud console. +A run does not create one neatly named cloud job. The platform submits a single job to the compute environment who's only job is to configure and run Nextflow. The Nextflow running within this job submits **many** Batch jobs and tasks, one per process invocation. Overall, this means you will not find a single resource named after the Platform run. Open the run on the Platform (**Runs** → your run), which mirrors the underlying Batch service: the task table here is the same work you can see in the cloud console. If you have cloud access, the prefixes Nextflow uses in each Batch service are: @@ -517,7 +708,7 @@ The relationship is the point: the Platform hands work to Batch, Batch runs it o ### 3.5. Side note: launch Actions -There is one more way to launch, built for automation rather than people. An **Action** is a saved launch configuration behind a single URL: call that URL and the pipeline runs, with no Launchpad and no `tw`. A Maintainer creates one in the UI under **Actions** → **Add action**, picks the pipeline and compute environment, and saves it. The Platform then shows a ready-made `curl` command for the Action's endpoint. +There is one more way to launch, built for automation rather than people. An **Action** is a saved launch configuration behind a single URL: call that URL and the pipeline runs, with no Launchpad and no `tw`. A Maintainer creates one in the UI under **Actions** → **Add action**, picks the pipeline, compute environment and other settings, and saves it as a configuration. The Platform then shows a ready-made `curl` command for the Action's endpoint. The split mirrors the roles: _creating_ an Action needs the Maintain role, but _triggering_ it needs only a Launch token. That is the point of the stratified launch: a Maintainer hands launchers (or an automation system) a URL they can call to run the pipeline, and nothing more. diff --git a/side-quests/platform_automation/seqerakit/README.md b/side-quests/platform_automation/seqerakit/README.md deleted file mode 100644 index f22e28d9f2..0000000000 --- a/side-quests/platform_automation/seqerakit/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# seqerakit - -The imperative asset, used in two tiers. See the Platform Automation side quest, §2 (Maintain) and §3 (Launch). seqerakit wraps `tw` and reads a YAML file: - -- `add-rnaseq.yml` (Maintain) adds `rnaseq-nf-$USERNAME` to the Launchpad. Re-running errors unless `--overwrite`, the imperative contrast to Terraform's idempotent `apply`. -- `launch-rnaseq.yml` (Launch) launches that pipeline. Run it twice and you get two runs. - -Terraform owns the pipeline's existence; seqerakit and `tw` act on it. This is the imperative contrast to the API Action and to Terraform. - -## Declarative vs imperative - -| | Terraform | seqerakit / tw | -| ------------ | ------------------------------------ | -------------------- | -| Manages | existence (does the pipeline exist?) | actions (run it now) | -| Run twice | no-op, still one pipeline | two runs | -| Mental model | desired state | do the thing, now | - -You added `rnaseq-nf-$USERNAME` with Terraform. Now you launch it. - -## Setup - -```bash -export TOWER_ACCESS_TOKEN= -export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN -export USERNAME=$USER -``` - -Edit `workspace` and `compute-env` in the YAML to match the workshop workspace if they differ. - -## Dry run first - -Always dry run first. `--dryrun` prints the `tw` command seqerakit would run and changes nothing: - -```bash -seqerakit launch-rnaseq.yml --dryrun -``` - -Then launch for real: - -```bash -seqerakit launch-rnaseq.yml -``` - -## The tw equivalent - -seqerakit is a wrapper over `tw`. The dry run shows you the underlying command. -It is the same as: - -```bash -tw launch rnaseq-nf-$USERNAME \ - --workspace=my-org/platform-automation \ - --compute-env=azure-batch-manual -``` - -This is the callback to the Azure mechanics: launching here submits jobs to the Platform, which hands them to Azure Batch, which stacks tasks onto the pool VMs. Same handoff the Admin tier showed in the portal. diff --git a/side-quests/platform_automation/seqerakit/add-rnaseq.yml b/side-quests/platform_automation/seqerakit/add-rnaseq.yml index 4de2081787..804d3a3819 100644 --- a/side-quests/platform_automation/seqerakit/add-rnaseq.yml +++ b/side-quests/platform_automation/seqerakit/add-rnaseq.yml @@ -1,19 +1,18 @@ -# Add rnaseq-nf-$USERNAME to the Launchpad (the imperative contrast to Terraform). +# Add rnaseq-nf-$WORKSHOP_USER to the Launchpad (the imperative contrast to Terraform). # # This ADDS a pipeline to the Launchpad. It does not launch it. Re-running errors # unless you pass --overwrite, because the pipeline already exists. That is the # imperative model: each run is an action, not a desired state. # -# seqerakit expands ${USERNAME} from the environment, so set it first: -# -# export USERNAME=$USER +# seqerakit expands ${WORKSHOP_USER} from the environment. Export it first +# (see section 0): a short, unique, dot-free handle for the shared workspace. # # Set the workspace and compute-env names to match your workshop workspace. pipelines: - - name: "rnaseq-nf-${USERNAME}" + - name: "rnaseq-nf-${WORKSHOP_USER}" url: "https://github.com/nextflow-io/rnaseq-nf" - workspace: "my-org/platform-automation" - description: "rnaseq-nf for ${USERNAME}, added with seqerakit" + workspace: "${TOWER_WORKSPACE_ID}" + description: "Added with seqerakit" compute-env: "azure-batch-manual" - work-dir: "az://nf-work/work" + work-dir: "az://work" revision: "master" diff --git a/side-quests/platform_automation/seqerakit/launch-rnaseq-multiple.yml b/side-quests/platform_automation/seqerakit/launch-rnaseq-multiple.yml new file mode 100644 index 0000000000..20c4928a03 --- /dev/null +++ b/side-quests/platform_automation/seqerakit/launch-rnaseq-multiple.yml @@ -0,0 +1,24 @@ +# Launch rnaseq-nf-$WORKSHOP_USER against the shared compute environment. +# +# This LAUNCHES a pipeline that already exists on the Launchpad. Terraform +# added it (see ../terraform/pipeline). seqerakit does not create it here. +# +# Declarative vs imperative: +# Terraform manages existence. apply twice = no second pipeline. +# seqerakit/tw do the thing, now. Run this twice = two runs. That is the +# point. Launching is an action, not a state. +# +# A bare `pipeline:` name (not a git URL) means "the saved Launchpad pipeline +# with this name". seqerakit expands ${WORKSHOP_USER} from the environment; +# export it first (see section 0). +# +# Set the workspace and compute-env names to match your workshop workspace. +launch: + - name: "rnaseq-nf-${WORKSHOP_USER}-run-1" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" + - name: "rnaseq-nf-${WORKSHOP_USER}-run-2" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" diff --git a/side-quests/platform_automation/seqerakit/launch-rnaseq.yml b/side-quests/platform_automation/seqerakit/launch-rnaseq.yml index afd1ebd0bf..9253657b1e 100644 --- a/side-quests/platform_automation/seqerakit/launch-rnaseq.yml +++ b/side-quests/platform_automation/seqerakit/launch-rnaseq.yml @@ -1,4 +1,4 @@ -# Launch rnaseq-nf-$USERNAME against the shared compute environment. +# Launch rnaseq-nf-$WORKSHOP_USER against the shared compute environment. # # This LAUNCHES a pipeline that already exists on the Launchpad. Terraform # added it (see ../terraform/pipeline). seqerakit does not create it here. @@ -9,14 +9,12 @@ # point. Launching is an action, not a state. # # A bare `pipeline:` name (not a git URL) means "the saved Launchpad pipeline -# with this name". seqerakit expands ${USERNAME} from the environment, so set -# it first: -# -# export USERNAME=$USER +# with this name". seqerakit expands ${WORKSHOP_USER} from the environment; +# export it first (see section 0). # # Set the workspace and compute-env names to match your workshop workspace. launch: - - name: "rnaseq-nf-${USERNAME}-run" - workspace: "my-org/platform-automation" - pipeline: "rnaseq-nf-${USERNAME}" - compute-env: "azure-batch-manual" + - name: "rnaseq-nf-${WORKSHOP_USER}-run" + workspace: "${TOWER_WORKSPACE_ID}" + pipeline: "rnaseq-nf-${WORKSHOP_USER}" + compute-env: "${COMPUTE_ENVIRONMENT}" diff --git a/side-quests/platform_automation/terraform/compute-env/README.md b/side-quests/platform_automation/terraform/compute-env/README.md deleted file mode 100644 index 9a8f997fad..0000000000 --- a/side-quests/platform_automation/terraform/compute-env/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# terraform/compute-env - -The Admin tier's compute-environment asset. See the Platform Automation side quest, §1 (Admin). You run this -only if you have the **Admin** (or Owner) role and permissions in the cloud -environment. In a workshop the presenter builds the shared compute environment -before the session; Maintain-role attendees do not run this and instead get a -`compute_env_id` to build pipelines against. - -## What it does - -The realistic, complex path. One config, two providers, end to end: - -1. **azurerm** creates a head pool and a worker pool on the customer's existing - Azure Batch account, on their existing subnet, running as their existing - managed identity. -2. **seqera** stores the Azure credentials and creates a manual Azure Batch - compute environment that uses the head pool and routes tasks to the worker - pool. - -This is what "manage your cloud as code" looks like for a lab that already -owns its Azure footprint. Terraform does not create the Batch account, the -network, or the identity. Those exist under change control and are referenced -with data sources (in `main.tf`). Terraform adds the pools and the Platform -wiring. - -## Three tiers, for context - -The session shows compute environments at three levels of control: - -1. **Batch Forge (UI)**: the Platform creates and scales the pool. Easiest. -2. **Manual CE pointing at an existing pool**: you already have a pool, the - Platform just submits to it. -3. **This config**: Terraform provisions the pools on your existing account and - network, then wires the Platform. Full infrastructure as code. - -## Manual vs Forge - -The compute environment is manual: `head_pool` is set and there is no `forge` -block. The Platform submits to the pools this config created. It never creates -or scales a pool itself. A `forge` block would flip it to Batch Forge. - -## Run it - -```bash -az login -export TOWER_ACCESS_TOKEN= -export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN - -cp terraform.tfvars.example terraform.tfvars -# edit terraform.tfvars: subscription, workspace, existing resource names, keys - -terraform init -terraform plan -terraform apply -``` - -After apply, note the `compute_env_id` output. The Maintain tier needs it for -`terraform/pipeline`. - -## The cloud side - -With Admin and cloud access you can also see the Azure side: the Batch account, -the pool VMs, jobs and tasks stacking onto them, a task's logs and exit code. -That is the whole point of the tier: the Platform submits to Azure Batch, which -runs the work. Maintain- and Launch-role users see only the compute environment -in the workspace, not the cloud behind it. diff --git a/side-quests/platform_automation/terraform/compute-env/main.tf b/side-quests/platform_automation/terraform/compute-env/main.tf index e90f9aeb3c..79d9ced4e5 100644 --- a/side-quests/platform_automation/terraform/compute-env/main.tf +++ b/side-quests/platform_automation/terraform/compute-env/main.tf @@ -11,7 +11,7 @@ terraform { required_version = ">= 1.9" required_providers { - seqera = { source = "seqeralabs/seqera", version = "0.30.5" } + seqera = { source = "seqeralabs/seqera", version = "0.40.1" } azurerm = { source = "hashicorp/azurerm", version = ">= 3.0" } random = { source = "hashicorp/random", version = ">= 3.0" } } @@ -28,11 +28,16 @@ provider "seqera" { # Pool sizing and image. Edit here rather than exposing every value as a knob. locals { - head_vm_size = "Standard_D4ds_v5" + head_vm_size = "Standard_E4ds_v5" worker_vm_size = "Standard_E16ds_v5" worker_max_nodes = 8 - worker_max_tasks = 16 - node_agent_sku_id = "batch.node.ubuntu 22.04" + node_agent_sku_id = "batch.node.ubuntu 24.04" + + # The Platform requires a pool's task slots to equal the node's vCPU count. + # Azure VM size names embed that count (Standard_E16ds_v5 -> 16), so derive it + # from the size rather than hardcoding a number that can drift out of sync. + head_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.head_vm_size)[0]) + worker_vm_cores = tonumber(regex("^Standard_[A-Z]+([0-9]+)", local.worker_vm_size)[0]) } # Keeps pool names unique across re-creates. @@ -42,30 +47,52 @@ resource "random_string" "suffix" { upper = false } +# Manual pools (head_pool set, no forge block) don't receive the node start task +# that Forge adds to auto-created pools, so we replicate its essential parts: +# 1. Install azcopy on the shared node path. Nextflow's Azure Batch executor +# uses it to stage data; manual pools must provide it. +# 2. Register the AppArmor profile Fusion needs on Ubuntu 24.04+ nodes. Without +# it, container init fails with "unable to apply apparmor profile" +# (COMP-1248, moby/moby#50013, launchpad #2111105). +locals { + node_start_task_script = <<-SCRIPT + set -euo pipefail + + # azcopy: Nextflow's Azure Batch executor uses it to transfer files. + curl -sL https://aka.ms/downloadazcopy-v10-linux -o /tmp/azcopy.tgz + tar -xzf /tmp/azcopy.tgz --strip-components=1 -C /tmp + mkdir -p "$AZ_BATCH_NODE_SHARED_DIR/bin/" + cp /tmp/azcopy "$AZ_BATCH_NODE_SHARED_DIR/bin/" + + # AppArmor profile for Fusion containers on Ubuntu 24.04+. + mkdir -p /etc/apparmor.d/containers + printf '%s\n' \ + 'abi ,' \ + 'include ' \ + '' \ + 'profile seqera-fusionfs-container flags=(default_allow) {' \ + ' userns,' \ + ' mount fstype=fuse.fusion -> /fusion/,' \ + ' mount fstype=fuse.fusion -> /fusion/**,' \ + ' umount,' \ + ' include ' \ + ' include ' \ + ' include if exists ' \ + '}' > /etc/apparmor.d/containers/seqera-fusionfs-container + apparmor_parser -r /etc/apparmor.d/containers/seqera-fusionfs-container + SCRIPT + + # Batch parses the command line itself (no shell), so multi-line scripts with + # quoting don't survive. Base64-encode the script and decode it on the node. + pool_start_command = "/bin/bash -c 'echo ${base64encode(local.node_start_task_script)} | base64 -d | bash'" +} + # Existing Azure resources the customer owns. Referenced, not managed. data "azurerm_batch_account" "existing" { name = var.batch_account_name resource_group_name = var.batch_account_rg } -data "azurerm_user_assigned_identity" "existing" { - name = var.managed_identity_name - resource_group_name = var.managed_identity_rg -} - -data "azurerm_subnet" "existing" { - name = var.subnet_name - virtual_network_name = var.vnet_name - resource_group_name = var.vnet_rg -} - -# Let the managed identity read and write the work directory's storage. -resource "azurerm_role_assignment" "batch_data_contributor" { - scope = data.azurerm_batch_account.existing.id - role_definition_name = "Azure Batch Data Contributor" - principal_id = data.azurerm_user_assigned_identity.existing.principal_id -} - # Head pool: runs the Nextflow head job. One node is enough. resource "azurerm_batch_pool" "head" { name = "rnaseq-head-${random_string.suffix.result}" @@ -74,6 +101,7 @@ resource "azurerm_batch_pool" "head" { account_name = var.batch_account_name vm_size = local.head_vm_size node_agent_sku_id = local.node_agent_sku_id + max_tasks_per_node = local.head_vm_cores fixed_scale { target_dedicated_nodes = 1 @@ -82,7 +110,7 @@ resource "azurerm_batch_pool" "head" { storage_image_reference { publisher = "microsoft-dsvm" offer = "ubuntu-hpc" - sku = "2204" + sku = "2404" version = "latest" } @@ -90,14 +118,18 @@ resource "azurerm_batch_pool" "head" { type = "DockerCompatible" } - identity { - type = "UserAssigned" - identity_ids = [data.azurerm_user_assigned_identity.existing.id] - } + # Install azcopy and load the Fusion AppArmor profile on each node at startup. + start_task { + command_line = local.pool_start_command + task_retry_maximum = 1 + wait_for_success = true - network_configuration { - subnet_id = data.azurerm_subnet.existing.id - public_address_provisioning_type = "NoPublicIPAddresses" + user_identity { + auto_user { + elevation_level = "Admin" + scope = "Pool" + } + } } } @@ -109,7 +141,7 @@ resource "azurerm_batch_pool" "worker" { account_name = var.batch_account_name vm_size = local.worker_vm_size node_agent_sku_id = local.node_agent_sku_id - max_tasks_per_node = local.worker_max_tasks + max_tasks_per_node = local.worker_vm_cores auto_scale { evaluation_interval = "PT5M" @@ -123,7 +155,7 @@ resource "azurerm_batch_pool" "worker" { storage_image_reference { publisher = "microsoft-dsvm" offer = "ubuntu-hpc" - sku = "2204" + sku = "2404" version = "latest" } @@ -131,30 +163,37 @@ resource "azurerm_batch_pool" "worker" { type = "DockerCompatible" } - identity { - type = "UserAssigned" - identity_ids = [data.azurerm_user_assigned_identity.existing.id] - } + # Install azcopy and load the Fusion AppArmor profile on each node at startup. + start_task { + command_line = local.pool_start_command + task_retry_maximum = 1 + wait_for_success = true - network_configuration { - subnet_id = data.azurerm_subnet.existing.id - public_address_provisioning_type = "NoPublicIPAddresses" + user_identity { + auto_user { + elevation_level = "Admin" + scope = "Pool" + } + } } - # Terraform manages the pool, not its live node count. + # Terraform manages the pool, not its live scale. lifecycle { - ignore_changes = [auto_scale] + ignore_changes = [auto_scale, fixed_scale] } } -# Azure credentials stored in the Platform. Keys are write-only. -resource "seqera_azure_credential" "main" { - name = "azure-batch" +# Azure credentials already stored in the Platform. We look them up by name +# rather than creating them, so no keys appear in this config. +data "seqera_credentials" "all" { workspace_id = var.workspace_id - batch_name = var.batch_account_name - batch_key = var.azure_batch_key - storage_name = var.azure_storage_name - storage_key = var.azure_storage_key +} + +locals { + azure_credentials_id = one([ + for c in data.seqera_credentials.all.credentials : c.id + if c.name == var.azure_credential_name + ]) } # Manual Azure Batch compute environment: uses the head pool, routes tasks to @@ -166,15 +205,16 @@ resource "seqera_compute_env" "main" { name = "azure-batch-manual" description = "Azure Batch CE on Terraform-managed pools" platform = "azure-batch" - credentials_id = seqera_azure_credential.main.credentials_id + credentials_id = local.azure_credentials_id config = { azure_batch = { - region = var.region - work_dir = var.work_dir - head_pool = azurerm_batch_pool.head.name - managed_identity_client_id = data.azurerm_user_assigned_identity.existing.client_id - nextflow_config = "process.queue = '${azurerm_batch_pool.worker.name}'\n" + region = var.region + work_dir = var.work_dir + head_pool = azurerm_batch_pool.head.name + enable_wave = true + enable_fusion = true + nextflow_config = "process.queue = '${azurerm_batch_pool.worker.name}'\n" } } } diff --git a/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example b/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example index 44e1a3bbc9..532f993e13 100644 --- a/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example +++ b/side-quests/platform_automation/terraform/compute-env/terraform.tfvars.example @@ -4,25 +4,17 @@ subscription_id = "00000000-0000-0000-0000-000000000000" workspace_id = 123456789 -region = "uaenorth" -work_dir = "az://nf-work/work" +region = "eastus" +work_dir = "az://work" -# Existing Azure resources (referenced, not created) -batch_account_name = "mybatch" -batch_account_rg = "batch-rg" -managed_identity_name = "nextflow-identity" -managed_identity_rg = "identity-rg" -vnet_name = "nf-vnet" -vnet_rg = "network-rg" -subnet_name = "batch-subnet" +# Existing Azure Batch account (referenced, not created). +batch_account_name = "mybatch" +batch_account_rg = "batch-rg" -# Pool sizing lives in main.tf (locals), not here. +# Azure credentials already stored in the Platform workspace, referenced by name. +azure_credential_name = "my-azure-credentials" -# Seqera credential keys (sensitive). Prefer TF_VAR_azure_batch_key / -# TF_VAR_azure_storage_key environment variables over this file. -azure_storage_name = "mystorage" -azure_batch_key = "REPLACE_ME" -azure_storage_key = "REPLACE_ME" +# Pool sizing lives in main.tf (locals), not here. # server_url defaults to Seqera Cloud; uncomment only for Enterprise. # server_url = "https://platform.example.com/api" diff --git a/side-quests/platform_automation/terraform/compute-env/variables.tf b/side-quests/platform_automation/terraform/compute-env/variables.tf index 7fc99ffd1b..c6ff497d28 100644 --- a/side-quests/platform_automation/terraform/compute-env/variables.tf +++ b/side-quests/platform_automation/terraform/compute-env/variables.tf @@ -1,6 +1,6 @@ # Inputs for the Admin tier compute-environment config. The Platform token is -# NOT here; it comes from TOWER_ACCESS_TOKEN. Sensitive Azure keys come from a -# gitignored terraform.tfvars or TF_VAR_ environment variables. +# NOT here; it comes from TOWER_ACCESS_TOKEN. The Azure credentials are not here +# either: they are already stored in the Platform and referenced by name. variable "server_url" { description = "Seqera Platform API endpoint. Cloud by default." @@ -18,7 +18,7 @@ variable "workspace_id" { type = number } -# Existing Azure resources (referenced via data sources, not created). +# Existing Azure Batch account (referenced via a data source, not created). variable "batch_account_name" { description = "Name of the existing Azure Batch account." type = string @@ -29,28 +29,10 @@ variable "batch_account_rg" { type = string } -variable "managed_identity_name" { - description = "Name of the existing user-assigned managed identity for Batch nodes." - type = string -} - -variable "managed_identity_rg" { - description = "Resource group of the existing managed identity." - type = string -} - -variable "vnet_name" { - description = "Name of the existing virtual network the pools attach to." - type = string -} - -variable "vnet_rg" { - description = "Resource group of the existing virtual network." - type = string -} - -variable "subnet_name" { - description = "Name of the existing subnet the pool nodes join." +# Azure credentials already stored in the Platform (added to the workspace +# beforehand). Referenced by name; the keys never appear in this config. +variable "azure_credential_name" { + description = "Name of the existing Azure credentials in the Platform workspace." type = string } @@ -60,24 +42,6 @@ variable "region" { } variable "work_dir" { - description = "Azure Blob Storage work directory, e.g. az://nf-work/work." - type = string -} - -# Seqera credential keys (sensitive). -variable "azure_batch_key" { - description = "Azure Batch account access key." - type = string - sensitive = true -} - -variable "azure_storage_name" { - description = "Azure Storage account name for the work directory." - type = string -} - -variable "azure_storage_key" { - description = "Azure Storage account access key." + description = "Azure Blob Storage work directory, e.g. az://work." type = string - sensitive = true } diff --git a/side-quests/platform_automation/terraform/pipeline/README.md b/side-quests/platform_automation/terraform/pipeline/README.md deleted file mode 100644 index 6cae02825d..0000000000 --- a/side-quests/platform_automation/terraform/pipeline/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# terraform/pipeline - -The Maintain tier's pipeline asset. This adds your own pipeline, -`rnaseq-nf-$USERNAME`, to the Launchpad with Terraform. It adds it. It does not -launch it. See the Platform Automation side quest, §2 (Maintain). - -## Before you start - -You need the **Maintain** role and two values from the Admin tier (or your -presenter): - -- `workspace_id`: the shared workspace. -- `compute_env_id`: the shared Azure Batch compute environment. - -And create a Platform token (User menu, Your tokens, Add token), then: - -```bash -export TOWER_ACCESS_TOKEN= -export SEQERA_ACCESS_TOKEN=$TOWER_ACCESS_TOKEN -``` - -## The flow - -```bash -cd /workspaces/training/side-quests/platform_automation/terraform/pipeline - -# 1. Set your inputs. Either copy the example file: -cp terraform.tfvars.example terraform.tfvars -# and edit it, or pass -var flags on each command below. - -# 2. Initialise (downloads the seqera 0.30.5 provider). -terraform init - -# 3. Preview. One resource to add. Nothing changes yet. -terraform plan -var="username=$USER" - -# 4. Add the pipeline to the Launchpad. -terraform apply -var="username=$USER" -``` - -Open the workspace Launchpad. `rnaseq-nf-$USERNAME` is there. No run has -started. - -## Idempotency - -Run apply again: - -```bash -terraform apply -var="username=$USER" -``` - -"No changes. Your infrastructure matches the configuration." Terraform manages -existence, not actions. Applying ten times does not launch ten runs. It -launches zero. To actually run the pipeline, use seqerakit/tw (see -`../../seqerakit`). That is the imperative half: do the thing, now. - -## Teardown - -Remove your pipeline when you are done: - -```bash -terraform apply -destroy -var="username=$USER" -``` - -It deletes only the pipeline you added. The shared compute environment and -workspace are untouched. diff --git a/side-quests/platform_automation/terraform/pipeline/main.tf b/side-quests/platform_automation/terraform/pipeline/main.tf index ff1eb26033..196ceb1652 100644 --- a/side-quests/platform_automation/terraform/pipeline/main.tf +++ b/side-quests/platform_automation/terraform/pipeline/main.tf @@ -15,7 +15,7 @@ terraform { required_providers { seqera = { source = "seqeralabs/seqera" - version = "0.30.5" + version = "0.40.1" } } } @@ -30,7 +30,8 @@ resource "seqera_pipeline" "rnaseq_nf" { workspace_id = var.workspace_id launch = { - pipeline = "https://github.com/nextflow-io/rnaseq-nf" + pipeline = "https://github.com/nextflow-io/rnaseq-nf" + # The master branch moves; pin a release tag (e.g. v2.4) for reproducible launches. revision = "master" compute_env_id = var.compute_env_id work_dir = var.work_dir diff --git a/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example b/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example index ac5950cc13..c8cca42dec 100644 --- a/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example +++ b/side-quests/platform_automation/terraform/pipeline/terraform.tfvars.example @@ -1,8 +1,9 @@ # Copy to terraform.tfvars and fill in. terraform.tfvars is gitignored. # Do NOT put your access token here. Export TOWER_ACCESS_TOKEN in the shell. -# Your username. Or pass it on the command line: terraform apply -var="username=$USER" -username = "your-username" +# Your workshop handle (letters, numbers, dash, underscore; no dots). +# Or pass it on the command line: terraform apply -var="username=$WORKSHOP_USER" +username = "your-handle" # Shared workshop workspace ID (the presenter gives you this). workspace_id = 123456789 @@ -10,8 +11,8 @@ workspace_id = 123456789 # Shared compute environment ID (the presenter gives you this). compute_env_id = "REPLACE_ME" -# Optional: defaults to az://nf-work/work -# work_dir = "az://nf-work/work" +# Optional: defaults to az://work +# work_dir = "az://work" # server_url defaults to Seqera Cloud; uncomment only for Enterprise. # server_url = "https://platform.example.com/api" diff --git a/side-quests/platform_automation/terraform/pipeline/variables.tf b/side-quests/platform_automation/terraform/pipeline/variables.tf index f7ac7f2080..1bfdd9835c 100644 --- a/side-quests/platform_automation/terraform/pipeline/variables.tf +++ b/side-quests/platform_automation/terraform/pipeline/variables.tf @@ -6,17 +6,18 @@ variable "server_url" { default = "https://api.cloud.seqera.io" } -# Your username makes the pipeline name unique in a shared workspace. No +# Your workshop handle makes the pipeline name unique in a shared workspace. No # default on purpose, so you must set it. Set it with -var or in terraform.tfvars: # -# terraform apply -var="username=$USER" +# terraform apply -var="username=$WORKSHOP_USER" variable "username" { - description = "Your username. Makes the Launchpad pipeline name unique." + description = "Your workshop handle. Makes the Launchpad pipeline name unique." type = string + # No dots: Launchpad pipeline names reject them. validation { - condition = can(regex("^[a-zA-Z0-9._-]{2,80}$", var.username)) - error_message = "username must be 2-80 chars: letters, numbers, dot, dash, underscore." + condition = can(regex("^[a-zA-Z0-9_-]{2,80}$", var.username)) + error_message = "username must be 2-80 chars: letters, numbers, dash, underscore (no dots)." } } @@ -35,5 +36,5 @@ variable "compute_env_id" { variable "work_dir" { description = "Azure Blob Storage work directory for runs of this pipeline." type = string - default = "az://nf-work/work" + default = "az://work" } From 9e8dc32a58f4180a840a85a7ef25b6c394d90b64 Mon Sep 17 00:00:00 2001 From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:51:34 +0100 Subject: [PATCH 8/8] docs(platform_automation): fix two malformed admonitions - !!! Tip -> !!! tip (lowercase type) - !!! Note Using a params.yml -> !!! note "Using a params file" (the unquoted title was parsed as CSS classes and dropped) Generated by Claude Code --- docs/en/docs/side_quests/platform_automation/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/docs/side_quests/platform_automation/index.md b/docs/en/docs/side_quests/platform_automation/index.md index 659cb48781..fb4dc0a371 100644 --- a/docs/en/docs/side_quests/platform_automation/index.md +++ b/docs/en/docs/side_quests/platform_automation/index.md @@ -570,9 +570,9 @@ terraform init terraform apply -var="username=$WORKSHOP_USER" -var="workspace_id=" -var="compute_env_id=" ``` -!!! Tip +!!! tip - copy `terraform.tfvars.example` to `terraform.tfvars` and fill it in so you stop passing `-var` flags. + Copy `terraform.tfvars.example` to `terraform.tfvars` and fill it in so you stop passing `-var` flags. Open the Launchpad: `rnaseq-nf-$WORKSHOP_USER` is there, with no run. Now apply again: @@ -668,7 +668,7 @@ It should show something like: You can monitor the run by clicking the provided URL. -!!! Note Using a params.yml +!!! note "Using a params file" If you wish to provide parameters to the pipeline, you can do so with the `--params-file` flag.