diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2be5811c93..077adc5dd6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -129,6 +129,83 @@ jobs: name: macos steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -208,6 +285,83 @@ jobs: # labels: windows-server-latest-large steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -260,6 +414,83 @@ jobs: labels: macos-latest steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -312,6 +543,83 @@ jobs: labels: macos-latest steps: + # Mount a ReFS Dev Drive on Windows BEFORE the checkout so the checkout's + # files land on ReFS directly. The default C: drive on GH runners is + # ~4.3k IOPS; a Dev Drive is ~127k. Two access paths on the same volume: + # - $GITHUB_WORKSPACE for the checkout (mounted as folder, not a + # junction, so the runner's open handle on that dir is irrelevant) + # - C:\dd for TEMP and Go caches (separate namespace so it doesn't + # pollute the workspace tree) + # All paths start with C:\, so older databricks-bundles tests under + # bundle/python/* don't trip os.path.commonpath across drives. + - name: Mount ReFS Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + # --- Volume A: workspace --- + $vhdA = "$env:RUNNER_TEMP\devdrive-ws.vhdx" + New-VHD -Path $vhdA -SizeBytes 15GB -Dynamic | Out-Null + $diskA = Mount-VHD -Path $vhdA -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partA = New-Partition -DiskNumber $diskA.Number -UseMaximumSize -DriveLetter Y + try { + Format-Volume -DriveLetter Y -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Y -FileSystem ReFS ` + -NewFileSystemLabel DevDriveWS -Confirm:$false -Force | Out-Null + } + + $ws = $env:GITHUB_WORKSPACE + if ((Get-ChildItem -Force $ws | Measure-Object).Count -gt 0) { + throw "GITHUB_WORKSPACE ($ws) is not empty; this step must run before actions/checkout." + } + Add-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath $ws + Remove-PartitionAccessPath -DiskNumber $diskA.Number ` + -PartitionNumber $partA.PartitionNumber -AccessPath 'Y:\' + + # ReFS auto-creates "System Volume Information" at the volume root; + # it has restrictive ACLs that make scandir fail with EPERM, which + # trips actions/checkout. Take ownership and remove it. + $svi = Join-Path $ws 'System Volume Information' + if (Test-Path $svi) { + cmd /c "takeown /F `"$svi`" /A /R" | Out-Null + cmd /c "icacls `"$svi`" /grant Administrators:F /T /C" | Out-Null + Remove-Item -Path $svi -Recurse -Force -ErrorAction SilentlyContinue + } + + # --- Volume B: TEMP + Go caches --- + $vhdB = "$env:RUNNER_TEMP\devdrive-cache.vhdx" + New-VHD -Path $vhdB -SizeBytes 10GB -Dynamic | Out-Null + $diskB = Mount-VHD -Path $vhdB -Passthru | + Initialize-Disk -PartitionStyle GPT -Passthru + $partB = New-Partition -DiskNumber $diskB.Number -UseMaximumSize -DriveLetter Z + try { + Format-Volume -DriveLetter Z -FileSystem ReFS -DevDrive ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } catch { + Format-Volume -DriveLetter Z -FileSystem ReFS ` + -NewFileSystemLabel DevDriveCache -Confirm:$false -Force | Out-Null + } + + $mount = 'C:\dd' + New-Item -ItemType Directory -Force -Path $mount | Out-Null + Add-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath $mount + Remove-PartitionAccessPath -DiskNumber $diskB.Number ` + -PartitionNumber $partB.PartitionNumber -AccessPath 'Z:\' + + New-Item -ItemType Directory -Path C:\dd\tmp,C:\dd\go-build,C:\dd\go-mod,C:\dd\go-tmp | Out-Null + + "TMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "TEMP=C:\dd\tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOCACHE=C:\dd\go-build" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOMODCACHE=C:\dd\go-mod" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + "GOTMPDIR=C:\dd\go-tmp" | Out-File -Append -Encoding utf8 $env:GITHUB_ENV + - name: Checkout repository and submodules uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2