diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 961fd314..49def4d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ on: env: PYTHON_STANDALONE_VERSIONS: | [ - "20260510" + "20260602" ] PYTHON_VERSIONS: | [ @@ -40,6 +40,8 @@ jobs: fail-fast: false runs-on: ubuntu-24.04 steps: + - name: Setup QEMU + uses: docker/setup-qemu-action@v4 - name: clone run: | # can't use actions/checkout here as transferring the shallow clone fails when using upload-/download-artifact @@ -59,7 +61,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.26 + go-version-file: go.mod - name: build-tag run: | git config --global user.email "no@mail.exists" @@ -109,7 +111,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.26 + go-version-file: go.mod - name: run tests shell: bash run: | diff --git a/go.mod b/go.mod index 8c221fad..5ce8528d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/klauspost/compress v1.18.6 github.com/sirupsen/logrus v1.9.4 github.com/stretchr/testify v1.11.1 - golang.org/x/sync v0.20.0 + golang.org/x/sync v0.21.0 ) require ( @@ -16,6 +16,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect - golang.org/x/sys v0.44.0 // indirect + golang.org/x/sys v0.46.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4fffc98b..ab5a2550 100644 --- a/go.sum +++ b/go.sum @@ -34,11 +34,15 @@ golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/strip.go b/internal/strip.go new file mode 100644 index 00000000..f0ea7697 --- /dev/null +++ b/internal/strip.go @@ -0,0 +1,55 @@ +package internal + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + +func StripBinaries(path string, arch string) error { + var binFiles []string + + err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error { + relPath, err := filepath.Rel(path, p) + if err != nil { + return err + } + if info.Mode().IsRegular() && info.Mode()&0111 != 0 { + b, err := os.ReadFile(p) + if err != nil { + return err + } + if len(b) >= 4 && b[0] == 0x7F && b[1] == 0x45 && b[2] == 0x4c && b[3] == 0x46 { + binFiles = append(binFiles, relPath) + } + } + return nil + }) + if err != nil { + return err + } + + log.Infof("stripping %d files in %s", len(binFiles), path) + + script := `set -e +export DEBIAN_FRONTEND=noninteractive +apt update +apt install -y llvm +cd /host +` + for _, file := range binFiles { + script += fmt.Sprintf("llvm-strip %s\n", file) + } + + cmd := exec.Command("docker", "run", "-i", "-v", fmt.Sprintf("%s:/host", path), "-w", "/host", "debian:trixie", "sh", "-c", script) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + return err + } + return nil +} diff --git a/pip/internal/requirements.txt b/pip/internal/requirements.txt index 07427648..657e2159 100644 --- a/pip/internal/requirements.txt +++ b/pip/internal/requirements.txt @@ -1 +1 @@ -pip==26.1.1 +pip==26.1.2 diff --git a/python/generate/main.go b/python/generate/main.go index 50ba39ef..a32f9bab 100644 --- a/python/generate/main.go +++ b/python/generate/main.go @@ -1,6 +1,7 @@ package main import ( + "compress/gzip" "flag" "fmt" "io" @@ -78,25 +79,27 @@ func main() { os string arch string dist string + ext string keepPatterns []glob.Glob + strip bool } jobs := []job{ - {"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns}, - {"linux", "arm64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns}, - {"darwin", "amd64", "apple-darwin-pgo+lto-full", keepNixPatterns}, - {"darwin", "arm64", "apple-darwin-pgo+lto-full", keepNixPatterns}, - {"windows", "amd64", "pc-windows-msvc-pgo-full", keepWinPatterns}, + {"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", "tar.zst", keepNixPatterns, true}, + {"linux", "arm64", "unknown-linux-gnu-pgo+lto-full", "tar.zst", keepNixPatterns, true}, + {"darwin", "amd64", "apple-darwin-pgo+lto-full", "tar.zst", keepNixPatterns, false}, + {"darwin", "arm64", "apple-darwin-pgo+lto-full", "tar.zst", keepNixPatterns, false}, + {"windows", "amd64", "pc-windows-msvc-pgo-full", "tar.zst", keepWinPatterns, false}, } for _, j := range jobs { j := j wg.Add(1) go func() { if *runPrepare { - downloadAndPrepare(j.os, j.arch, j.dist, j.keepPatterns) + downloadAndPrepare(j.os, j.arch, j.dist, j.ext, j.keepPatterns, j.strip) } if *runPack { - packPrepared(j.os, j.arch, j.dist, targetPath) + packPrepared(j.os, j.arch, j.dist, j.ext, targetPath) } wg.Done() }() @@ -104,8 +107,16 @@ func main() { wg.Wait() } -func downloadAndPrepare(osName string, arch string, dist string, keepPatterns []glob.Glob) { - downloadPath := download(osName, arch, dist) +func calcInstallPath(extractPath string, dist string) string { + installPath := filepath.Join(extractPath, "python") + if !strings.Contains(dist, "install_only") { + installPath = filepath.Join(installPath, "install") + } + return installPath +} + +func downloadAndPrepare(osName string, arch string, dist string, ext string, keepPatterns []glob.Glob, strip bool) { + downloadPath := download(osName, arch, dist, ext) extractPath := downloadPath + ".extracted" err := os.RemoveAll(extractPath) @@ -113,9 +124,13 @@ func downloadAndPrepare(osName string, arch string, dist string, keepPatterns [] log.Panic(err) } - extract(downloadPath, extractPath) + _, err = extract(downloadPath, extractPath, ext) + if err != nil { + log.Errorf("extract failed: %v", err) + os.Exit(1) + } - installPath := filepath.Join(extractPath, "python", "install") + installPath := calcInstallPath(extractPath, dist) var libPath string if osName == "windows" { @@ -132,11 +147,18 @@ func downloadAndPrepare(osName string, arch string, dist string, keepPatterns [] if err != nil { panic(err) } + + if strip { + err = internal.StripBinaries(installPath, arch) + if err != nil { + panic(err) + } + } } -func packPrepared(osName string, arch string, dist string, targetPath string) { - extractPath := generateDownloadPath(arch, dist) + ".extracted" - installPath := filepath.Join(extractPath, "python", "install") +func packPrepared(osName string, arch string, dist string, ext string, targetPath string) { + extractPath := generateDownloadPath(arch, dist, ext) + ".extracted" + installPath := calcInstallPath(extractPath, dist) err := embed_util.CopyForEmbed(filepath.Join(targetPath, fmt.Sprintf("%s-%s", osName, arch)), installPath) if err != nil { panic(err) @@ -158,21 +180,21 @@ func packPrepared(osName string, arch string, dist string, targetPath string) { } } -func generateDownloadPath(arch string, dist string) string { +func generateDownloadPath(arch string, dist string, ext string) string { pythonArch, ok := archMapping[arch] if !ok { log.Errorf("arch %s not supported", arch) os.Exit(1) } - fname := fmt.Sprintf("cpython-%s+%s-%s-%s.tar.zst", *pythonVersion, *pythonStandaloneVersion, pythonArch, dist) + fname := fmt.Sprintf("cpython-%s+%s-%s-%s.%s", *pythonVersion, *pythonStandaloneVersion, pythonArch, dist, ext) return filepath.Join(*preparePath, fname) } -func download(osName string, arch string, dist string) string { +func download(osName string, arch string, dist string, ext string) string { downloadLock.Lock() defer downloadLock.Unlock() - downloadPath := generateDownloadPath(arch, dist) + downloadPath := generateDownloadPath(arch, dist, ext) fname := filepath.Base(downloadPath) downloadUrl := fmt.Sprintf("https://github.com/astral-sh/python-build-standalone/releases/download/%s/%s", *pythonStandaloneVersion, fname) @@ -211,27 +233,38 @@ func download(osName string, arch string, dist string) string { return downloadPath } -func extract(archivePath string, targetPath string) string { +func extract(archivePath string, targetPath string, ext string) (string, error) { f, err := os.Open(archivePath) if err != nil { - log.Errorf("opening file failed: %v", err) - os.Exit(1) + return "", err } defer f.Close() - z, err := zstd.NewReader(f) - if err != nil { - log.Errorf("decompression failed: %v", err) - os.Exit(1) + var dc io.Reader + if ext == "tar.gz" { + gz, err := gzip.NewReader(f) + if err != nil { + return "", err + } + defer gz.Close() + dc = gz + } else if ext == "tar.zst" { + z, err := zstd.NewReader(f) + if err != nil { + return "", err + } + defer z.Close() + dc = z + } else { + return "", fmt.Errorf("unknown extension: %s", ext) } - defer z.Close() log.Infof("decompressing %s", archivePath) - err = internal.ExtractTarStream(z, targetPath) + err = internal.ExtractTarStream(dc, targetPath) if err != nil { log.Errorf("decompression failed: %v", err) os.Exit(1) } - return targetPath + return targetPath, nil } diff --git a/python/internal/data/dummy.go b/python/internal/data/dummy.go index 9566a9f1..07bb62f2 100644 --- a/python/internal/data/dummy.go +++ b/python/internal/data/dummy.go @@ -1,6 +1,6 @@ package data // PLEASE READ THIS!!!! -// This file is really just a dummy. The release process will remove this file and generate some read embedded files +// This file is really just a dummy. The release process will remove this file and generate some real embedded files // and commit these into a temporary branch and then tag it. This is to avoid clogging up the main branch with too many // binary files, which would be a very bad experience when pulling in go-embed-python as a dependency.