From 0514262ea9ebb034379527f13b4238c3787868da Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 15:29:01 +0200 Subject: [PATCH 01/15] ci: mount ReFS Dev Drive on Windows to speed up small-file I/O Acceptance tests on Windows spend most of their wall-clock on small-file writes: each terraform init copies providers into a per-test `.terraform/` under `$TEMP`, and the go build/module caches see similar churn. The default C: drive on GitHub-hosted and Databricks-protected Windows runners is backed by remote block storage (~4.3k IOPS); a ReFS Dev Drive is ~127k IOPS on comparable benchmarks. This step creates a 20GB dynamic VHDX, mounts it as Z:, formats it ReFS (with the `-DevDrive` flag where the host supports it, falling back to plain ReFS otherwise), and redirects TEMP/TMP + GOCACHE/GOMODCACHE/ GOTMPDIR onto it. Checkout stays on C: -- moving it would be invasive (acceptance test output normalization) for little further gain. Placed at the top of the composite action so it applies to every caller (test, test-exp-aitools, test-exp-ssh, test-pipelines). No-op on non-Windows runners via `runner.os == 'Windows'`. Co-authored-by: Isaac --- .../setup-build-environment/action.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 3fd492c70f..0e02bc1653 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,6 +9,35 @@ inputs: runs: using: 'composite' steps: + # Mount a ReFS Dev Drive on Windows and point TEMP/TMP + Go's cache dirs at + # it. Small-file I/O (per-test `.terraform/providers/`, go build cache, etc.) + # is roughly ~30x faster than the default C: drive on GH runners. The + # `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to plain + # ReFS which is still markedly faster than C:. Must run before setup-go so + # GOCACHE/GOMODCACHE are rooted on the drive. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" + New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null + $disk = Mount-VHD -Path $vhdPath -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z | Out-Null + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } + New-Item -ItemType Directory -Path Z:\tmp,Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null + "TEMP=Z:\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TMP=Z:\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=Z:\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=Z:\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=Z:\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 80ad9a4a1609c1a295eabe23785ef2710eefd6e2 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 16:32:51 +0200 Subject: [PATCH 02/15] Drop TEMP/TMP redirect from Dev Drive step Redirecting TEMP to Z: puts `t.TempDir()` (and therefore each test's bundle cwd) on Z:, while the checkout and uv's Python package cache stay on C:. Under `bundle/python/*` tests with older databricks-bundles versions (e.g. PYDAB_VERSION=0.266.0), the Python mutator calls `os.path.commonpath([os.getcwd(), path])` which raises `ValueError: Paths don't have the same drive`. Six tests regressed: experimental-compatibility{,-both-equal}, resource-loading, unicode-support, restricted-execution, resolve-variable. Keep only GOCACHE/GOMODCACHE/GOTMPDIR on the Dev Drive -- those benefit Go compilation I/O without spanning drive boundaries at the Python level. Per-test `.terraform/` speedup is lost; recover it in a follow-up by plumbing a test-framework-specific tmpdir that callers can keep on the same drive as the checkout. Co-authored-by: Isaac --- .../actions/setup-build-environment/action.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 0e02bc1653..3091a22468 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,12 +9,18 @@ inputs: runs: using: 'composite' steps: - # Mount a ReFS Dev Drive on Windows and point TEMP/TMP + Go's cache dirs at - # it. Small-file I/O (per-test `.terraform/providers/`, go build cache, etc.) - # is roughly ~30x faster than the default C: drive on GH runners. The + # Mount a ReFS Dev Drive on Windows and route Go's cache dirs onto it. + # The default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. + # Go build compilation is small-file-heavy and benefits most. The # `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to plain # ReFS which is still markedly faster than C:. Must run before setup-go so - # GOCACHE/GOMODCACHE are rooted on the drive. + # GOCACHE/GOMODCACHE save/restore lands on the drive. + # + # Note: TEMP/TMP are intentionally NOT redirected. Doing so puts + # `t.TempDir()` on Z: while the checkout and uv's Python package cache stay + # on C:, which triggers `ValueError: Paths don't have the same drive` in + # older databricks-bundles versions under `bundle/python/*` tests + # (`os.path.commonpath` can't span drives). - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh @@ -31,9 +37,7 @@ runs: Format-Volume -DriveLetter Z -FileSystem ReFS ` -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null } - New-Item -ItemType Directory -Path Z:\tmp,Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null - "TEMP=Z:\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "TMP=Z:\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + New-Item -ItemType Directory -Path Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null "GOCACHE=Z:\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOMODCACHE=Z:\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOTMPDIR=Z:\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV From 3e9064801f81777c3e87e7f3e28f947cb670f87f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 16:40:10 +0200 Subject: [PATCH 03/15] Junction the workspace onto the Dev Drive on Windows Rather than redirecting individual env vars (TEMP/TMP/GO*) and running into cross-drive path issues (Python mutator's `os.path.commonpath([cwd, path])` fails when cwd and the uv-cached module live on different drives), we can just relocate the entire workspace. Create a directory junction from `$GITHUB_WORKSPACE` (still on C:) to `Z:\workspace`. From every tool's point of view the path is unchanged -- it starts with `C:\` and `commonpath` works. Physically, all reads and writes go to the ReFS volume. Flow: - Mount VHDX at Z: - Wipe the workflow's prior checkout at $GITHUB_WORKSPACE - Create junction to Z:\workspace - The composite's own `actions/checkout` step re-populates the workspace via the junction (so setup-jfrog etc. find their files) No Go / TEMP env gymnastics needed. bundle/python/* tests stay happy. Co-authored-by: Isaac --- .../setup-build-environment/action.yml | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 3091a22468..2a679ca2b2 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,22 +9,25 @@ inputs: runs: using: 'composite' steps: - # Mount a ReFS Dev Drive on Windows and route Go's cache dirs onto it. + # Mount a ReFS Dev Drive on Windows and junction the workspace onto it so + # every checkout / build / test file physically lives on the fast volume. # The default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Go build compilation is small-file-heavy and benefits most. The - # `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to plain - # ReFS which is still markedly faster than C:. Must run before setup-go so - # GOCACHE/GOMODCACHE save/restore lands on the drive. + # The `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to + # plain ReFS which is still markedly faster than C:. # - # Note: TEMP/TMP are intentionally NOT redirected. Doing so puts - # `t.TempDir()` on Z: while the checkout and uv's Python package cache stay - # on C:, which triggers `ValueError: Paths don't have the same drive` in - # older databricks-bundles versions under `bundle/python/*` tests - # (`os.path.commonpath` can't span drives). - - name: Mount ReFS Dev Drive (Windows) + # We use a directory junction (not a Z: cwd) so that path-inspecting tools + # -- e.g. the Python mutator's `os.path.commonpath` in older + # databricks-bundles -- still see the original C:\ path and don't trip the + # "Paths don't have the same drive" error. + # + # The junction happens before the composite's own `actions/checkout` step + # re-populates the workspace (via the junction, physically on Z:). + - name: Mount ReFS Dev Drive and junction workspace (Windows) if: runner.os == 'Windows' shell: pwsh run: | + Set-Location C:\ + $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null $disk = Mount-VHD -Path $vhdPath -Passthru | @@ -37,6 +40,12 @@ runs: Format-Volume -DriveLetter Z -FileSystem ReFS ` -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null } + + $ws = $env:GITHUB_WORKSPACE + Remove-Item -Recurse -Force -LiteralPath $ws + New-Item -ItemType Directory -Path Z:\workspace | Out-Null + New-Item -ItemType Junction -Path $ws -Target Z:\workspace | Out-Null + New-Item -ItemType Directory -Path Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null "GOCACHE=Z:\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOMODCACHE=Z:\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV From ecaeaa7582aba6154dbb71835497378499f6123a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 16:58:20 +0200 Subject: [PATCH 04/15] Revert "Junction the workspace onto the Dev Drive on Windows" This reverts commit 931c8a948c64217031f6dc6180c2b11fedf60f80. --- .../setup-build-environment/action.yml | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 2a679ca2b2..3091a22468 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,25 +9,22 @@ inputs: runs: using: 'composite' steps: - # Mount a ReFS Dev Drive on Windows and junction the workspace onto it so - # every checkout / build / test file physically lives on the fast volume. + # Mount a ReFS Dev Drive on Windows and route Go's cache dirs onto it. # The default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # The `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to - # plain ReFS which is still markedly faster than C:. + # Go build compilation is small-file-heavy and benefits most. The + # `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to plain + # ReFS which is still markedly faster than C:. Must run before setup-go so + # GOCACHE/GOMODCACHE save/restore lands on the drive. # - # We use a directory junction (not a Z: cwd) so that path-inspecting tools - # -- e.g. the Python mutator's `os.path.commonpath` in older - # databricks-bundles -- still see the original C:\ path and don't trip the - # "Paths don't have the same drive" error. - # - # The junction happens before the composite's own `actions/checkout` step - # re-populates the workspace (via the junction, physically on Z:). - - name: Mount ReFS Dev Drive and junction workspace (Windows) + # Note: TEMP/TMP are intentionally NOT redirected. Doing so puts + # `t.TempDir()` on Z: while the checkout and uv's Python package cache stay + # on C:, which triggers `ValueError: Paths don't have the same drive` in + # older databricks-bundles versions under `bundle/python/*` tests + # (`os.path.commonpath` can't span drives). + - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh run: | - Set-Location C:\ - $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null $disk = Mount-VHD -Path $vhdPath -Passthru | @@ -40,12 +37,6 @@ runs: Format-Volume -DriveLetter Z -FileSystem ReFS ` -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null } - - $ws = $env:GITHUB_WORKSPACE - Remove-Item -Recurse -Force -LiteralPath $ws - New-Item -ItemType Directory -Path Z:\workspace | Out-Null - New-Item -ItemType Junction -Path $ws -Target Z:\workspace | Out-Null - New-Item -ItemType Directory -Path Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null "GOCACHE=Z:\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOMODCACHE=Z:\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV From 2caaa88e02982deac3abcc554808cca649816f83 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 17:35:26 +0200 Subject: [PATCH 05/15] Redirect TEMP + Go caches via C:\a\_fast junction Go-caches-only on Z: left the big Windows test jobs effectively flat (windows/terraform 32m34s vs 32m33s baseline) because the dominant cost is per-test `.terraform/` churn under TEMP, not Go compilation. Moving TEMP onto the Dev Drive was the missing piece. The first TEMP-on-Z: attempt broke `bundle/python/*` tests (older databricks-bundles calls `os.path.commonpath([cwd, uv_cache_path])` and chokes when the two live on different drives). Fix: create a directory junction at `C:\a\_fast` (sibling to `C:\a\cli\cli`, not inside the repo) pointing at `Z:\fast`. Path strings stay `C:\...`, so `commonpath` is happy; I/O physically lands on Z:. Junction is outside the checkout to avoid `git status` pollution, `git clean` interactions after `actions/checkout`, and unintended traversal by repo-walking tools. Co-authored-by: Isaac --- .../setup-build-environment/action.yml | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 3091a22468..bdf4a5a0ca 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,19 +9,27 @@ inputs: runs: using: 'composite' steps: - # Mount a ReFS Dev Drive on Windows and route Go's cache dirs onto it. + # Mount a ReFS Dev Drive on Windows and redirect TEMP, TMP, and the Go + # cache dirs through a `C:\a\_fast` junction that physically lives on Z:. # The default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Go build compilation is small-file-heavy and benefits most. The - # `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to plain - # ReFS which is still markedly faster than C:. Must run before setup-go so - # GOCACHE/GOMODCACHE save/restore lands on the drive. + # Per-test `.terraform/` dirs (under TEMP via `t.TempDir()`) dominate + # `make test` wall-clock on Windows, so moving TEMP onto the fast volume + # is the key lever. # - # Note: TEMP/TMP are intentionally NOT redirected. Doing so puts - # `t.TempDir()` on Z: while the checkout and uv's Python package cache stay - # on C:, which triggers `ValueError: Paths don't have the same drive` in - # older databricks-bundles versions under `bundle/python/*` tests - # (`os.path.commonpath` can't span drives). - - name: Mount ReFS Dev Drive (Windows) + # We use a junction at `C:\a\_fast` -> `Z:\fast` (not a direct `Z:\tmp`) + # so that path strings keep the `C:\` prefix. That matters for older + # databricks-bundles (`bundle/python/*` tests with PYDAB_VERSION=0.266.0) + # which call `os.path.commonpath([cwd, uv_cache_path])`; mixing C: and Z: + # in those string paths raises `ValueError: Paths don't have the same + # drive`. The junction is transparent to path-inspecting code. + # + # The junction is placed outside the checkout (`C:\a\_fast`, sibling to + # `C:\a\cli\cli`) so it doesn't pollute `git status`, trip `git clean` + # after `actions/checkout`, or show up in repo-walking tools. + # + # The `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall through + # to plain ReFS which is still markedly faster than C:. + - name: Mount ReFS Dev Drive and redirect I/O-hot dirs (Windows) if: runner.os == 'Windows' shell: pwsh run: | @@ -37,10 +45,17 @@ runs: Format-Volume -DriveLetter Z -FileSystem ReFS ` -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null } - New-Item -ItemType Directory -Path Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null - "GOCACHE=Z:\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOMODCACHE=Z:\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOTMPDIR=Z:\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + + New-Item -ItemType Directory -Path Z:\fast | Out-Null + New-Item -ItemType Junction -Path C:\a\_fast -Target Z:\fast | Out-Null + New-Item -ItemType Directory -Path ` + C:\a\_fast\tmp, C:\a\_fast\go-build, C:\a\_fast\go-mod, C:\a\_fast\go-tmp | Out-Null + + "TEMP=C:\a\_fast\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TMP=C:\a\_fast\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\a\_fast\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\a\_fast\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\a\_fast\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From e602ba361873758735a332bff3fb51cc49a9506b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 17:51:26 +0200 Subject: [PATCH 06/15] Use directory symlink instead of junction for Dev Drive `bundle.TestRootLookup` (unit test) called `filepath.EvalSymlinks` on its `t.TempDir()` path, which landed under the `C:\a\_fast` directory junction. Go's stdlib EvalSymlinks on Windows returns "cannot find the path specified" for `IO_REPARSE_TAG_MOUNT_POINT` (junctions) rooted at a newly mounted ReFS VHDX, but handles `IO_REPARSE_TAG_SYMLINK` (directory symlinks) correctly. Switching `New-Item -ItemType Junction` to `cmd /c mklink /D` (directory symbolic link) to dodge the quirk. Symlinks require Developer Mode, which is the default on GitHub-hosted Windows runners. Co-authored-by: Isaac --- .github/actions/setup-build-environment/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index bdf4a5a0ca..57be6ef0fb 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -47,7 +47,12 @@ runs: } New-Item -ItemType Directory -Path Z:\fast | Out-Null - New-Item -ItemType Junction -Path C:\a\_fast -Target Z:\fast | Out-Null + # Use a directory symbolic link (not a junction). Go's + # `filepath.EvalSymlinks` handles dir symlinks correctly but returns + # "cannot find the path specified" on junctions rooted at a newly + # mounted VHDX (hits `bundle.TestRootLookup` in the unit test suite). + # Requires Developer Mode, which is enabled on GH Windows runners. + cmd /c "mklink /D C:\a\_fast Z:\fast" | Out-Null New-Item -ItemType Directory -Path ` C:\a\_fast\tmp, C:\a\_fast\go-build, C:\a\_fast\go-mod, C:\a\_fast\go-tmp | Out-Null From 942125d4e3dae7d9d0252534b84f7279447af13c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 18:27:36 +0200 Subject: [PATCH 07/15] Revert "Use directory symlink instead of junction for Dev Drive" This reverts commit fce8d2d4892ddc38f98d97a8b14f019c7c320741. --- .github/actions/setup-build-environment/action.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 57be6ef0fb..bdf4a5a0ca 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -47,12 +47,7 @@ runs: } New-Item -ItemType Directory -Path Z:\fast | Out-Null - # Use a directory symbolic link (not a junction). Go's - # `filepath.EvalSymlinks` handles dir symlinks correctly but returns - # "cannot find the path specified" on junctions rooted at a newly - # mounted VHDX (hits `bundle.TestRootLookup` in the unit test suite). - # Requires Developer Mode, which is enabled on GH Windows runners. - cmd /c "mklink /D C:\a\_fast Z:\fast" | Out-Null + New-Item -ItemType Junction -Path C:\a\_fast -Target Z:\fast | Out-Null New-Item -ItemType Directory -Path ` C:\a\_fast\tmp, C:\a\_fast\go-build, C:\a\_fast\go-mod, C:\a\_fast\go-tmp | Out-Null From 164569bbfc0081b423942ca2254ffede682a11e4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 22 Apr 2026 18:27:36 +0200 Subject: [PATCH 08/15] Revert "Redirect TEMP + Go caches via C:\a\_fast junction" This reverts commit 9908fe7d184525b455258eb4b90ceae5178441ca. --- .../setup-build-environment/action.yml | 45 +++++++------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index bdf4a5a0ca..3091a22468 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,27 +9,19 @@ inputs: runs: using: 'composite' steps: - # Mount a ReFS Dev Drive on Windows and redirect TEMP, TMP, and the Go - # cache dirs through a `C:\a\_fast` junction that physically lives on Z:. + # Mount a ReFS Dev Drive on Windows and route Go's cache dirs onto it. # The default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Per-test `.terraform/` dirs (under TEMP via `t.TempDir()`) dominate - # `make test` wall-clock on Windows, so moving TEMP onto the fast volume - # is the key lever. + # Go build compilation is small-file-heavy and benefits most. The + # `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to plain + # ReFS which is still markedly faster than C:. Must run before setup-go so + # GOCACHE/GOMODCACHE save/restore lands on the drive. # - # We use a junction at `C:\a\_fast` -> `Z:\fast` (not a direct `Z:\tmp`) - # so that path strings keep the `C:\` prefix. That matters for older - # databricks-bundles (`bundle/python/*` tests with PYDAB_VERSION=0.266.0) - # which call `os.path.commonpath([cwd, uv_cache_path])`; mixing C: and Z: - # in those string paths raises `ValueError: Paths don't have the same - # drive`. The junction is transparent to path-inspecting code. - # - # The junction is placed outside the checkout (`C:\a\_fast`, sibling to - # `C:\a\cli\cli`) so it doesn't pollute `git status`, trip `git clean` - # after `actions/checkout`, or show up in repo-walking tools. - # - # The `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall through - # to plain ReFS which is still markedly faster than C:. - - name: Mount ReFS Dev Drive and redirect I/O-hot dirs (Windows) + # Note: TEMP/TMP are intentionally NOT redirected. Doing so puts + # `t.TempDir()` on Z: while the checkout and uv's Python package cache stay + # on C:, which triggers `ValueError: Paths don't have the same drive` in + # older databricks-bundles versions under `bundle/python/*` tests + # (`os.path.commonpath` can't span drives). + - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh run: | @@ -45,17 +37,10 @@ runs: Format-Volume -DriveLetter Z -FileSystem ReFS ` -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null } - - New-Item -ItemType Directory -Path Z:\fast | Out-Null - New-Item -ItemType Junction -Path C:\a\_fast -Target Z:\fast | Out-Null - New-Item -ItemType Directory -Path ` - C:\a\_fast\tmp, C:\a\_fast\go-build, C:\a\_fast\go-mod, C:\a\_fast\go-tmp | Out-Null - - "TEMP=C:\a\_fast\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "TMP=C:\a\_fast\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOCACHE=C:\a\_fast\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOMODCACHE=C:\a\_fast\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOTMPDIR=C:\a\_fast\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + New-Item -ItemType Directory -Path Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null + "GOCACHE=Z:\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=Z:\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=Z:\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 0ebe78141c90d9e92849da765ec6a3335dac6012 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 7 May 2026 12:00:57 +0200 Subject: [PATCH 09/15] ci: mount Dev Drive at C:\dd folder, junction workspace + redirect TEMP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mount the ReFS volume at C:\dd via Add-PartitionAccessPath instead of assigning drive letter Z:. Junction $GITHUB_WORKSPACE onto C:\dd\ws and point TMP/TEMP at C:\dd\tmp. Every path the test process sees now starts with C:\ — file I/O still lands on the Dev Drive, but os.path.commonpath won't trip on cross-drive paths in older databricks-bundles under bundle/python/*. Co-authored-by: Isaac --- .../setup-build-environment/action.yml | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 3091a22468..6d5e8c0e20 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,27 +9,28 @@ inputs: runs: using: 'composite' steps: - # Mount a ReFS Dev Drive on Windows and route Go's cache dirs onto it. - # The default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Go build compilation is small-file-heavy and benefits most. The - # `-DevDrive` flag is Win11 22621+ / WS2025; older hosts fall back to plain - # ReFS which is still markedly faster than C:. Must run before setup-go so - # GOCACHE/GOMODCACHE save/restore lands on the drive. + # Mount a ReFS Dev Drive on Windows and route Go's cache dirs, TEMP, and + # the workspace onto it. The default C: drive on GH runners is ~4.3k IOPS; + # a Dev Drive is ~127k. The `-DevDrive` flag is Win11 22621+ / WS2025; + # older hosts fall back to plain ReFS which is still markedly faster than C:. + # Must run before setup-go so GOCACHE/GOMODCACHE save/restore lands on the + # drive, and before checkout so the workspace junction is in place. # - # Note: TEMP/TMP are intentionally NOT redirected. Doing so puts - # `t.TempDir()` on Z: while the checkout and uv's Python package cache stay - # on C:, which triggers `ValueError: Paths don't have the same drive` in - # older databricks-bundles versions under `bundle/python/*` tests - # (`os.path.commonpath` can't span drives). + # The volume is mounted at C:\dd (no drive letter) so every path the test + # process sees starts with C:\. Same-drive paths are required so older + # databricks-bundles tests under bundle/python/* don't trip + # `ValueError: Paths don't have the same drive` from os.path.commonpath. - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh run: | + $ErrorActionPreference = 'Stop' + $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null $disk = Mount-VHD -Path $vhdPath -Passthru | Initialize-Disk -PartitionStyle GPT -Passthru - New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z | Out-Null + $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z try { Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null @@ -37,10 +38,34 @@ runs: Format-Volume -DriveLetter Z -FileSystem ReFS ` -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null } - New-Item -ItemType Directory -Path Z:\go-build,Z:\go-mod,Z:\go-tmp | Out-Null - "GOCACHE=Z:\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOMODCACHE=Z:\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOTMPDIR=Z:\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + + # Junction the workspace onto the Dev Drive. Both paths start with C:\ + # so os.path.commonpath stays happy while file I/O lands on ReFS. + $ws = $env:GITHUB_WORKSPACE + if (Test-Path $ws) { + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; refusing to junction over it." + } + Remove-Item -Force $ws + } + New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null + + fsutil devdrv query $mount - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From c6deaa7ef4e3fba2fbc38f111a1ad4abe01e3fb9 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 7 May 2026 12:38:48 +0200 Subject: [PATCH 10/15] ci: handle pre-existing workspace when junctioning to Dev Drive The calling workflows run actions/checkout before invoking this composite action, so the workspace is already populated when the Mount step runs. Robocopy /MOVE the contents to C:\dd\ws before placing the junction so file I/O lands on the Dev Drive. Co-authored-by: Isaac --- .github/actions/setup-build-environment/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 6d5e8c0e20..80c4f21e8d 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -56,10 +56,13 @@ runs: # Junction the workspace onto the Dev Drive. Both paths start with C:\ # so os.path.commonpath stays happy while file I/O lands on ReFS. + # The calling workflow has already done actions/checkout, so the + # workspace is non-empty: move its contents onto the Dev Drive first. $ws = $env:GITHUB_WORKSPACE if (Test-Path $ws) { if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { - throw "GITHUB_WORKSPACE ($ws) is not empty; refusing to junction over it." + robocopy $ws C:\dd\ws /E /MOVE /NFL /NDL /NJH /NJS /NP | Out-Null + if ($LASTEXITCODE -ge 8) { throw "robocopy failed with exit $LASTEXITCODE" } } Remove-Item -Force $ws } From 211541cac7ff42c38863f3ae5a59580c12902f40 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 7 May 2026 13:26:07 +0200 Subject: [PATCH 11/15] ci: move Dev Drive mount into push.yml jobs before actions/checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous attempt called the mount step from the setup-build-environment composite, which runs after the workflow's actions/checkout — moving 17k workspace files from C: to ReFS while Defender scans every read takes forever and the step hung past 40 min on every Windows job. Inline the mount step into each test* job in push.yml so it runs before checkout. The junction is placed over an empty GITHUB_WORKSPACE and the checkout writes directly onto ReFS — no copy needed. Local composite actions can't be referenced before checkout, so the PowerShell is duplicated across the four test* jobs that use Windows. Co-authored-by: Isaac --- .../setup-build-environment/action.yml | 61 ------ .github/workflows/push.yml | 200 ++++++++++++++++++ 2 files changed, 200 insertions(+), 61 deletions(-) diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 80c4f21e8d..3fd492c70f 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -9,67 +9,6 @@ inputs: runs: using: 'composite' steps: - # Mount a ReFS Dev Drive on Windows and route Go's cache dirs, TEMP, and - # the workspace onto it. The default C: drive on GH runners is ~4.3k IOPS; - # a Dev Drive is ~127k. The `-DevDrive` flag is Win11 22621+ / WS2025; - # older hosts fall back to plain ReFS which is still markedly faster than C:. - # Must run before setup-go so GOCACHE/GOMODCACHE save/restore lands on the - # drive, and before checkout so the workspace junction is in place. - # - # The volume is mounted at C:\dd (no drive letter) so every path the test - # process sees starts with C:\. Same-drive paths are required so older - # databricks-bundles tests under bundle/python/* don't trip - # `ValueError: Paths don't have the same drive` from os.path.commonpath. - - name: Mount ReFS Dev Drive (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - $ErrorActionPreference = 'Stop' - - $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" - New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null - $disk = Mount-VHD -Path $vhdPath -Passthru | - Initialize-Disk -PartitionStyle GPT -Passthru - $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z - try { - Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null - } catch { - Format-Volume -DriveLetter Z -FileSystem ReFS ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null - } - - $mount = 'C:\dd' - New-Item -ItemType Directory -Force -Path $mount | Out-Null - Add-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath $mount - Remove-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' - - New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null - - "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - - # Junction the workspace onto the Dev Drive. Both paths start with C:\ - # so os.path.commonpath stays happy while file I/O lands on ReFS. - # The calling workflow has already done actions/checkout, so the - # workspace is non-empty: move its contents onto the Dev Drive first. - $ws = $env:GITHUB_WORKSPACE - if (Test-Path $ws) { - if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { - robocopy $ws C:\dd\ws /E /MOVE /NFL /NDL /NJH /NJS /NP | Out-Null - if ($LASTEXITCODE -ge 8) { throw "robocopy failed with exit $LASTEXITCODE" } - } - Remove-Item -Force $ws - } - New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null - - fsutil devdrv query $mount - - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2be5811c93..0ac35110e9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -129,6 +129,56 @@ jobs: name: macos steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. Routes TEMP and Go's caches there too. The + # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. + # Mounted at C:\dd (no drive letter) so every path the test process sees + # starts with C:\ — older databricks-bundles tests under bundle/python/* + # call os.path.commonpath across TEMP and the workspace, which raises + # `ValueError: Paths don't have the same drive` for cross-drive paths. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" + New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null + $disk = Mount-VHD -Path $vhdPath -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + + $ws = $env:GITHUB_WORKSPACE + if (Test-Path $ws) { + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Remove-Item -Force $ws + } + New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -208,6 +258,56 @@ jobs: # labels: windows-server-latest-large steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. Routes TEMP and Go's caches there too. The + # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. + # Mounted at C:\dd (no drive letter) so every path the test process sees + # starts with C:\ — older databricks-bundles tests under bundle/python/* + # call os.path.commonpath across TEMP and the workspace, which raises + # `ValueError: Paths don't have the same drive` for cross-drive paths. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" + New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null + $disk = Mount-VHD -Path $vhdPath -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + + $ws = $env:GITHUB_WORKSPACE + if (Test-Path $ws) { + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Remove-Item -Force $ws + } + New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -260,6 +360,56 @@ jobs: labels: macos-latest steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. Routes TEMP and Go's caches there too. The + # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. + # Mounted at C:\dd (no drive letter) so every path the test process sees + # starts with C:\ — older databricks-bundles tests under bundle/python/* + # call os.path.commonpath across TEMP and the workspace, which raises + # `ValueError: Paths don't have the same drive` for cross-drive paths. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" + New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null + $disk = Mount-VHD -Path $vhdPath -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + + $ws = $env:GITHUB_WORKSPACE + if (Test-Path $ws) { + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Remove-Item -Force $ws + } + New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -312,6 +462,56 @@ jobs: labels: macos-latest steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. Routes TEMP and Go's caches there too. The + # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. + # Mounted at C:\dd (no drive letter) so every path the test process sees + # starts with C:\ — older databricks-bundles tests under bundle/python/* + # call os.path.commonpath across TEMP and the workspace, which raises + # `ValueError: Paths don't have the same drive` for cross-drive paths. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" + New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null + $disk = Mount-VHD -Path $vhdPath -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $disk.Number ` + -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + + $ws = $env:GITHUB_WORKSPACE + if (Test-Path $ws) { + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Remove-Item -Force $ws + } + New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 6c119db4866a8a9212a017af460616d9bd47f48a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 7 May 2026 14:02:57 +0200 Subject: [PATCH 12/15] ci: cd out of GITHUB_WORKSPACE before deleting it on Windows The pwsh shell starts with CWD set to GITHUB_WORKSPACE, so Remove-Item on that path fails with "because it is in use". Switch to C:\ first. Co-authored-by: Isaac --- .github/workflows/push.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0ac35110e9..01855fa757 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -175,6 +175,8 @@ jobs: if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." } + # The shell starts with CWD inside $ws; cd out before removing it. + Set-Location C:\ Remove-Item -Force $ws } New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null @@ -304,6 +306,8 @@ jobs: if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." } + # The shell starts with CWD inside $ws; cd out before removing it. + Set-Location C:\ Remove-Item -Force $ws } New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null @@ -406,6 +410,8 @@ jobs: if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." } + # The shell starts with CWD inside $ws; cd out before removing it. + Set-Location C:\ Remove-Item -Force $ws } New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null @@ -508,6 +514,8 @@ jobs: if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." } + # The shell starts with CWD inside $ws; cd out before removing it. + Set-Location C:\ Remove-Item -Force $ws } New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null From 73845ad2e6d96c98824a2c187585d6d876758f2f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 7 May 2026 14:42:14 +0200 Subject: [PATCH 13/15] ci: mount Dev Drive volumes directly via Add-PartitionAccessPath Previous attempt junctioned GITHUB_WORKSPACE to a folder on a single Dev Drive, but Remove-Item on the workspace dir fails because the runner agent holds an open handle on it ("being used by another process"). Set-Location alone wasn't enough to release it. Switch to two ReFS Dev Drive volumes mounted via Add-PartitionAccessPath: - 15 GB volume mounted directly at $GITHUB_WORKSPACE (the empty folder becomes a reparse point overlay; open handles don't matter for that) - 10 GB volume mounted at C:\dd for TEMP, GOCACHE, GOMODCACHE, GOTMPDIR Co-authored-by: Isaac --- .github/workflows/push.yml | 284 ++++++++++++++++++++++--------------- 1 file changed, 172 insertions(+), 112 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 01855fa757..31c14c2df1 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -130,39 +130,65 @@ jobs: steps: # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's - # files land on ReFS directly. Routes TEMP and Go's caches there too. The - # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Mounted at C:\dd (no drive letter) so every path the test process sees - # starts with C:\ — older databricks-bundles tests under bundle/python/* - # call os.path.commonpath across TEMP and the workspace, which raises - # `ValueError: Paths don't have the same drive` for cross-drive paths. + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh run: | $ErrorActionPreference = 'Stop' - $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" - New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null - $disk = Mount-VHD -Path $vhdPath -Passthru | + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | Initialize-Disk -PartitionStyle GPT -Passthru - $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z try { Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } catch { Format-Volume -DriveLetter Z -FileSystem ReFS ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } $mount = 'C:\dd' New-Item -ItemType Directory -Force -Path $mount | Out-Null - Add-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath $mount - Remove-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' - New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV @@ -170,17 +196,6 @@ jobs: "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - $ws = $env:GITHUB_WORKSPACE - if (Test-Path $ws) { - if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { - throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." - } - # The shell starts with CWD inside $ws; cd out before removing it. - Set-Location C:\ - Remove-Item -Force $ws - } - New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null - - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -261,39 +276,65 @@ jobs: steps: # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's - # files land on ReFS directly. Routes TEMP and Go's caches there too. The - # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Mounted at C:\dd (no drive letter) so every path the test process sees - # starts with C:\ — older databricks-bundles tests under bundle/python/* - # call os.path.commonpath across TEMP and the workspace, which raises - # `ValueError: Paths don't have the same drive` for cross-drive paths. + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh run: | $ErrorActionPreference = 'Stop' - $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" - New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null - $disk = Mount-VHD -Path $vhdPath -Passthru | + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | Initialize-Disk -PartitionStyle GPT -Passthru - $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z try { Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } catch { Format-Volume -DriveLetter Z -FileSystem ReFS ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } $mount = 'C:\dd' New-Item -ItemType Directory -Force -Path $mount | Out-Null - Add-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath $mount - Remove-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' - New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV @@ -301,17 +342,6 @@ jobs: "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - $ws = $env:GITHUB_WORKSPACE - if (Test-Path $ws) { - if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { - throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." - } - # The shell starts with CWD inside $ws; cd out before removing it. - Set-Location C:\ - Remove-Item -Force $ws - } - New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null - - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -365,39 +395,65 @@ jobs: steps: # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's - # files land on ReFS directly. Routes TEMP and Go's caches there too. The - # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Mounted at C:\dd (no drive letter) so every path the test process sees - # starts with C:\ — older databricks-bundles tests under bundle/python/* - # call os.path.commonpath across TEMP and the workspace, which raises - # `ValueError: Paths don't have the same drive` for cross-drive paths. + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh run: | $ErrorActionPreference = 'Stop' - $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" - New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null - $disk = Mount-VHD -Path $vhdPath -Passthru | + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | Initialize-Disk -PartitionStyle GPT -Passthru - $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z try { Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } catch { Format-Volume -DriveLetter Z -FileSystem ReFS ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } $mount = 'C:\dd' New-Item -ItemType Directory -Force -Path $mount | Out-Null - Add-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath $mount - Remove-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' - New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV @@ -405,17 +461,6 @@ jobs: "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - $ws = $env:GITHUB_WORKSPACE - if (Test-Path $ws) { - if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { - throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." - } - # The shell starts with CWD inside $ws; cd out before removing it. - Set-Location C:\ - Remove-Item -Force $ws - } - New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null - - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -469,39 +514,65 @@ jobs: steps: # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's - # files land on ReFS directly. Routes TEMP and Go's caches there too. The - # default C: drive on GH runners is ~4.3k IOPS; a Dev Drive is ~127k. - # Mounted at C:\dd (no drive letter) so every path the test process sees - # starts with C:\ — older databricks-bundles tests under bundle/python/* - # call os.path.commonpath across TEMP and the workspace, which raises - # `ValueError: Paths don't have the same drive` for cross-drive paths. + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. - name: Mount ReFS Dev Drive (Windows) if: runner.os == 'Windows' shell: pwsh run: | $ErrorActionPreference = 'Stop' - $vhdPath = "$env:RUNNER_TEMP\devdrive.vhdx" - New-VHD -Path $vhdPath -SizeBytes 20GB -Dynamic | Out-Null - $disk = Mount-VHD -Path $vhdPath -Passthru | + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | Initialize-Disk -PartitionStyle GPT -Passthru - $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -DriveLetter Z + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z try { Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } catch { Format-Volume -DriveLetter Z -FileSystem ReFS ` - -NewFileSystemLabel DevDrive -Confirm:$false -Force | Out-Null + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null } $mount = 'C:\dd' New-Item -ItemType Directory -Force -Path $mount | Out-Null - Add-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath $mount - Remove-PartitionAccessPath -DiskNumber $disk.Number ` - -PartitionNumber $part.PartitionNumber -AccessPath 'Z:\' + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' - New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp,C:\dd\ws | Out-Null + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV @@ -509,17 +580,6 @@ jobs: "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV - $ws = $env:GITHUB_WORKSPACE - if (Test-Path $ws) { - if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { - throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." - } - # The shell starts with CWD inside $ws; cd out before removing it. - Set-Location C:\ - Remove-Item -Force $ws - } - New-Item -ItemType Junction -Path $ws -Target C:\dd\ws | Out-Null - - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 07408b27918618048f6f7611c5f83826a2aa408b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 7 May 2026 16:00:51 +0200 Subject: [PATCH 14/15] ci: skip checkout clean on Dev Drive to avoid System Volume Information ReFS volumes auto-create a "System Volume Information" folder at their root that scandir can't enumerate (EPERM), which trips actions/checkout's pre-clone cleanup. The workspace is freshly mounted and otherwise empty, so disabling the clean step is safe. Co-authored-by: Isaac --- .github/workflows/push.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 31c14c2df1..25b5e9e38d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -198,6 +198,11 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Skip the pre-checkout clean — the workspace is a freshly-mounted + # ReFS Dev Drive whose only contents are the system-protected + # "System Volume Information" folder, which scandir can't read. + clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment @@ -344,6 +349,11 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Skip the pre-checkout clean — the workspace is a freshly-mounted + # ReFS Dev Drive whose only contents are the system-protected + # "System Volume Information" folder, which scandir can't read. + clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment @@ -463,6 +473,11 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Skip the pre-checkout clean — the workspace is a freshly-mounted + # ReFS Dev Drive whose only contents are the system-protected + # "System Volume Information" folder, which scandir can't read. + clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment @@ -582,6 +597,11 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Skip the pre-checkout clean — the workspace is a freshly-mounted + # ReFS Dev Drive whose only contents are the system-protected + # "System Volume Information" folder, which scandir can't read. + clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment From d74e5892d7f0807178fbf5ed4158005c7505b842 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 7 May 2026 16:38:04 +0200 Subject: [PATCH 15/15] ci: strip System Volume Information from freshly-mounted ReFS workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReFS auto-creates a System Volume Information folder at the volume root with ACLs that block scandir for non-SYSTEM users, which trips actions/checkout (EPERM during readdir). Take ownership and remove it right after mount; the workspace volume is fresh and we don't need it. Also drop `clean: false` — it didn't avoid the scandir, since checkout walks the workspace before the clean step. Co-authored-by: Isaac --- .github/workflows/push.yml | 60 +++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 25b5e9e38d..077adc5dd6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -167,6 +167,16 @@ jobs: Remove-PartitionAccessPath -DiskNumber $diskA.Number ` -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + # --- Volume B: TEMP + Go caches --- $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null @@ -198,11 +208,6 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - # Skip the pre-checkout clean — the workspace is a freshly-mounted - # ReFS Dev Drive whose only contents are the system-protected - # "System Volume Information" folder, which scandir can't read. - clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment @@ -318,6 +323,16 @@ jobs: Remove-PartitionAccessPath -DiskNumber $diskA.Number ` -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + # --- Volume B: TEMP + Go caches --- $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null @@ -349,11 +364,6 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - # Skip the pre-checkout clean — the workspace is a freshly-mounted - # ReFS Dev Drive whose only contents are the system-protected - # "System Volume Information" folder, which scandir can't read. - clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment @@ -442,6 +452,16 @@ jobs: Remove-PartitionAccessPath -DiskNumber $diskA.Number ` -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + # --- Volume B: TEMP + Go caches --- $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null @@ -473,11 +493,6 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - # Skip the pre-checkout clean — the workspace is a freshly-mounted - # ReFS Dev Drive whose only contents are the system-protected - # "System Volume Information" folder, which scandir can't read. - clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment @@ -566,6 +581,16 @@ jobs: Remove-PartitionAccessPath -DiskNumber $diskA.Number ` -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + # --- Volume B: TEMP + Go caches --- $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null @@ -597,11 +622,6 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - # Skip the pre-checkout clean — the workspace is a freshly-mounted - # ReFS Dev Drive whose only contents are the system-protected - # "System Volume Information" folder, which scandir can't read. - clean: false - name: Setup build environment uses: ./.github/actions/setup-build-environment