feat: Strip debug info from binaries to save space

.github/workflows/release.yml

@@ -11,7 +11,7 @@ on:
 env:
   PYTHON_STANDALONE_VERSIONS: |
     [
-      "20260510"
+      "20260602"
     ]
   PYTHON_VERSIONS: |
     [
@@ -40,6 +40,8 @@ jobs:
       fail-fast: false
     runs-on: ubuntu-24.04
     steps:
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v4
       - name: clone
         run: |
           # can't use actions/checkout here as transferring the shallow clone fails when using upload-/download-artifact
@@ -59,7 +61,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.26
+          go-version-file: go.mod
       - name: build-tag
         run: |
           git config --global user.email "no@mail.exists"
@@ -109,7 +111,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.26
+          go-version-file: go.mod
       - name: run tests
         shell: bash
         run: |

go.mod

@@ -8,14 +8,14 @@ require (
 	github.com/klauspost/compress v1.18.6
 	github.com/sirupsen/logrus v1.9.4
 	github.com/stretchr/testify v1.11.1
-	golang.org/x/sync v0.20.0
+	golang.org/x/sync v0.21.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect
-	golang.org/x/sys v0.44.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -34,11 +34,15 @@ golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
 golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

internal/strip.go

@@ -0,0 +1,55 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func StripBinaries(path string, arch string) error {
+	var binFiles []string
+
+	err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
+		relPath, err := filepath.Rel(path, p)
+		if err != nil {
+			return err
+		}
+		if info.Mode().IsRegular() && info.Mode()&0111 != 0 {
+			b, err := os.ReadFile(p)
+			if err != nil {
+				return err
+			}
+			if len(b) >= 4 && b[0] == 0x7F && b[1] == 0x45 && b[2] == 0x4c && b[3] == 0x46 {
+				binFiles = append(binFiles, relPath)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	log.Infof("stripping %d files in %s", len(binFiles), path)
+
+	script := `set -e
+export DEBIAN_FRONTEND=noninteractive
+apt update
+apt install -y llvm
+cd /host
+`
+	for _, file := range binFiles {
+		script += fmt.Sprintf("llvm-strip %s\n", file)
+	}
+
+	cmd := exec.Command("docker", "run", "-i", "-v", fmt.Sprintf("%s:/host", path), "-w", "/host", "debian:trixie", "sh", "-c", script)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}

pip/internal/requirements.txt

@@ -1 +1 @@
-pip==26.1.1
+pip==26.1.2

python/generate/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"compress/gzip"
 	"flag"
 	"fmt"
 	"io"
@@ -78,44 +79,58 @@ func main() {
 		os           string
 		arch         string
 		dist         string
+		ext          string
 		keepPatterns []glob.Glob
+		strip        bool
 	}
 
 	jobs := []job{
-		{"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns},
-		{"linux", "arm64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns},
-		{"darwin", "amd64", "apple-darwin-pgo+lto-full", keepNixPatterns},
-		{"darwin", "arm64", "apple-darwin-pgo+lto-full", keepNixPatterns},
-		{"windows", "amd64", "pc-windows-msvc-pgo-full", keepWinPatterns},
+		{"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", "tar.zst", keepNixPatterns, true},
+		{"linux", "arm64", "unknown-linux-gnu-pgo+lto-full", "tar.zst", keepNixPatterns, true},
+		{"darwin", "amd64", "apple-darwin-pgo+lto-full", "tar.zst", keepNixPatterns, false},
+		{"darwin", "arm64", "apple-darwin-pgo+lto-full", "tar.zst", keepNixPatterns, false},
+		{"windows", "amd64", "pc-windows-msvc-pgo-full", "tar.zst", keepWinPatterns, false},
 	}
 	for _, j := range jobs {
 		j := j
 		wg.Add(1)
 		go func() {
 			if *runPrepare {
-				downloadAndPrepare(j.os, j.arch, j.dist, j.keepPatterns)
+				downloadAndPrepare(j.os, j.arch, j.dist, j.ext, j.keepPatterns, j.strip)
 			}
 			if *runPack {
-				packPrepared(j.os, j.arch, j.dist, targetPath)
+				packPrepared(j.os, j.arch, j.dist, j.ext, targetPath)
 			}
 			wg.Done()
 		}()
 	}
 	wg.Wait()
 }
 
-func downloadAndPrepare(osName string, arch string, dist string, keepPatterns []glob.Glob) {
-	downloadPath := download(osName, arch, dist)
+func calcInstallPath(extractPath string, dist string) string {
+	installPath := filepath.Join(extractPath, "python")
+	if !strings.Contains(dist, "install_only") {
+		installPath = filepath.Join(installPath, "install")
+	}
+	return installPath
+}
+
+func downloadAndPrepare(osName string, arch string, dist string, ext string, keepPatterns []glob.Glob, strip bool) {
+	downloadPath := download(osName, arch, dist, ext)
 
 	extractPath := downloadPath + ".extracted"
 	err := os.RemoveAll(extractPath)
 	if err != nil {
 		log.Panic(err)
 	}
 
-	extract(downloadPath, extractPath)
+	_, err = extract(downloadPath, extractPath, ext)
+	if err != nil {
+		log.Errorf("extract failed: %v", err)
+		os.Exit(1)
+	}
 
-	installPath := filepath.Join(extractPath, "python", "install")
+	installPath := calcInstallPath(extractPath, dist)
 
 	var libPath string
 	if osName == "windows" {
@@ -132,11 +147,18 @@ func downloadAndPrepare(osName string, arch string, dist string, keepPatterns []
 	if err != nil {
 		panic(err)
 	}
+
+	if strip {
+		err = internal.StripBinaries(installPath, arch)
+		if err != nil {
+			panic(err)
+		}
+	}
 }
 
-func packPrepared(osName string, arch string, dist string, targetPath string) {
-	extractPath := generateDownloadPath(arch, dist) + ".extracted"
-	installPath := filepath.Join(extractPath, "python", "install")
+func packPrepared(osName string, arch string, dist string, ext string, targetPath string) {
+	extractPath := generateDownloadPath(arch, dist, ext) + ".extracted"
+	installPath := calcInstallPath(extractPath, dist)
 	err := embed_util.CopyForEmbed(filepath.Join(targetPath, fmt.Sprintf("%s-%s", osName, arch)), installPath)
 	if err != nil {
 		panic(err)
@@ -158,21 +180,21 @@ func packPrepared(osName string, arch string, dist string, targetPath string) {
 	}
 }
 
-func generateDownloadPath(arch string, dist string) string {
+func generateDownloadPath(arch string, dist string, ext string) string {
 	pythonArch, ok := archMapping[arch]
 	if !ok {
 		log.Errorf("arch %s not supported", arch)
 		os.Exit(1)
 	}
-	fname := fmt.Sprintf("cpython-%s+%s-%s-%s.tar.zst", *pythonVersion, *pythonStandaloneVersion, pythonArch, dist)
+	fname := fmt.Sprintf("cpython-%s+%s-%s-%s.%s", *pythonVersion, *pythonStandaloneVersion, pythonArch, dist, ext)
 	return filepath.Join(*preparePath, fname)
 }
 
-func download(osName string, arch string, dist string) string {
+func download(osName string, arch string, dist string, ext string) string {
 	downloadLock.Lock()
 	defer downloadLock.Unlock()
 
-	downloadPath := generateDownloadPath(arch, dist)
+	downloadPath := generateDownloadPath(arch, dist, ext)
 	fname := filepath.Base(downloadPath)
 	downloadUrl := fmt.Sprintf("https://github.com/astral-sh/python-build-standalone/releases/download/%s/%s", *pythonStandaloneVersion, fname)
 
@@ -211,27 +233,38 @@ func download(osName string, arch string, dist string) string {
 	return downloadPath
 }
 
-func extract(archivePath string, targetPath string) string {
+func extract(archivePath string, targetPath string, ext string) (string, error) {
 	f, err := os.Open(archivePath)
 	if err != nil {
-		log.Errorf("opening file failed: %v", err)
-		os.Exit(1)
+		return "", err
 	}
 	defer f.Close()
 
-	z, err := zstd.NewReader(f)
-	if err != nil {
-		log.Errorf("decompression failed: %v", err)
-		os.Exit(1)
+	var dc io.Reader
+	if ext == "tar.gz" {
+		gz, err := gzip.NewReader(f)
+		if err != nil {
+			return "", err
+		}
+		defer gz.Close()
+		dc = gz
+	} else if ext == "tar.zst" {
+		z, err := zstd.NewReader(f)
+		if err != nil {
+			return "", err
+		}
+		defer z.Close()
+		dc = z
+	} else {
+		return "", fmt.Errorf("unknown extension: %s", ext)
 	}
-	defer z.Close()
 
 	log.Infof("decompressing %s", archivePath)
-	err = internal.ExtractTarStream(z, targetPath)
+	err = internal.ExtractTarStream(dc, targetPath)
 	if err != nil {
 		log.Errorf("decompression failed: %v", err)
 		os.Exit(1)
 	}
 
-	return targetPath
+	return targetPath, nil
 }

python/internal/data/dummy.go

@@ -1,6 +1,6 @@
 package data
 
 // PLEASE READ THIS!!!!
-// This file is really just a dummy. The release process will remove this file and generate some read embedded files
+// This file is really just a dummy. The release process will remove this file and generate some real embedded files
 // and commit these into a temporary branch and then tag it. This is to avoid clogging up the main branch with too many
 // binary files, which would be a very bad experience when pulling in go-embed-python as a dependency.