feat: add perf support

.github/workflows/ci.yml

@@ -47,6 +47,7 @@ jobs:
       - name: Run the benchmarks
         uses: CodSpeedHQ/action@main
         with:
+          mode: walltime
           working-directory: example
           run: cargo r --manifest-path ../go-runner/Cargo.toml -- test -bench=.
 

Cargo.toml

@@ -1,2 +1,3 @@
 [workspace]
+resolver = "3"
 members = ["go-runner"]

README.md

@@ -38,7 +38,7 @@ $ go-runner test -bench=.
 [INFO  go_runner] Found BenchmarkFibonacci20_Loop      in "fib_test.go"
 [INFO  go_runner] Generating custom runner for package: example
 [INFO  go_runner] Running benchmarks for package: example
-Running with CodSpeed instrumentation
+Running with CodSpeed (mode: walltime)
 goos: linux
 goarch: amd64
 cpu: 12th Gen Intel(R) Core(TM) i7-1260P @ 1672.130MHz

example-codspeed/fib_codspeed.go

@@ -5,31 +5,31 @@ import (
 )
 
 func BenchmarkFibonacci10(b *testing.B) {
-	// b.Run("fibonacci(40)", func(b *testing.B) {
-	// 	for i := 0; i < b.N; i++ {
-	// 		fibonacci(10)
-	// 	}
-	// })
-	// b.Run("fibonacci(20)", func(b *testing.B) {
-	// 	for i := 0; i < b.N; i++ {
-	// 		fibonacci(20)
-	// 	}
-	// })
-	b.RunParallel(func(b *testing.PB) {
-		for b.Next() {
-			fibonacci(30)
+	b.Run("fibonacci(10)", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			fibonacci(10)
+		}
+	})
+	b.Run("fibonacci(20)", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			fibonacci(20)
 		}
 	})
+	// b.RunParallel(func(b *testing.PB) {
+	// 	for b.Next() {
+	// 		fibonacci(30)
+	// 	}
+	// })
 }
 
 func BenchmarkFibonacci20(b *testing.B) {
 	for b.Loop() {
-		fibonacci(30)
+		fibonacci(20)
 	}
 }
 
-// func BenchmarkFibonacci30(b *testing.B) {
-// 	for i := 0; i < b.N; i++ {
-// 		fibonacci(30)
-// 	}
-// }
+func BenchmarkFibonacci30(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		fibonacci(30)
+	}
+}

example-codspeed/go-runner.metadata

@@ -0,0 +1,4 @@
+{
+    "profile_folder": "/tmp",
+    "relative_package_path": "codspeed-go"
+}

example/sleep_test.go

@@ -18,38 +18,80 @@ func BenchmarkSleep100ns(b *testing.B) {
 	}
 }
 
+func BenchmarkSleep100ns_Loop(b *testing.B) {
+	for b.Loop() {
+		busyWait(100 * time.Nanosecond)
+	}
+}
+
 func BenchmarkSleep1us(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		busyWait(1 * time.Microsecond)
 	}
 }
 
+func BenchmarkSleep1us_Loop(b *testing.B) {
+	for b.Loop() {
+		busyWait(1 * time.Microsecond)
+	}
+}
+
 func BenchmarkSleep10us(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		busyWait(10 * time.Microsecond)
 	}
 }
 
+func BenchmarkSleep10us_Loop(b *testing.B) {
+	for b.Loop() {
+		busyWait(10 * time.Microsecond)
+	}
+}
+
 func BenchmarkSleep100us(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		busyWait(100 * time.Microsecond)
 	}
 }
 
+func BenchmarkSleep100us_Loop(b *testing.B) {
+	for b.Loop() {
+		busyWait(100 * time.Microsecond)
+	}
+}
+
 func BenchmarkSleep1ms(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		busyWait(1 * time.Millisecond)
 	}
 }
 
+func BenchmarkSleep1ms_Loop(b *testing.B) {
+	for b.Loop() {
+		busyWait(1 * time.Millisecond)
+	}
+}
+
 func BenchmarkSleep10ms(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		busyWait(10 * time.Millisecond)
 	}
 }
 
+func BenchmarkSleep10ms_Loop(b *testing.B) {
+	for b.Loop() {
+		busyWait(10 * time.Millisecond)
+	}
+}
+
 func BenchmarkSleep50ms(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		busyWait(50 * time.Millisecond)
 	}
 }
+
+func BenchmarkSleep50ms_Loop(b *testing.B) {
+	for b.Loop() {
+		busyWait(50 * time.Millisecond)
+	}
+}

example/very/nested/module/example_test.go

@@ -0,0 +1,9 @@
+package example
+
+import "testing"
+
+func BenchmarkExample(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = 42
+	}
+}

go-runner/src/builder/discovery.rs

@@ -233,8 +233,11 @@ impl BenchmarkPackage {
         }
     }
 
-    pub fn from_project(go_project_path: &Path) -> anyhow::Result<Vec<BenchmarkPackage>> {
-        let raw_packages = Self::run_go_list(go_project_path)?;
+    pub fn from_project(
+        go_project_path: &Path,
+        packages: &[String],
+    ) -> anyhow::Result<Vec<BenchmarkPackage>> {
+        let raw_packages = Self::run_go_list(go_project_path, packages)?;
         let has_test_files =
             |files: &Vec<String>| files.iter().any(|name| name.ends_with("_test.go"));
         let has_test_imports = |imports: &Vec<String>| {
@@ -297,10 +300,13 @@ impl BenchmarkPackage {
         Ok(packages)
     }
 
-    fn run_go_list(go_project_path: &Path) -> anyhow::Result<Vec<GoPackage>> {
-        // Execute 'go list -test -compiled -json ./...' to get package information
+    fn run_go_list(go_project_path: &Path, packages: &[String]) -> anyhow::Result<Vec<GoPackage>> {
+        // Execute 'go list -test -compiled -json <packages>' to get package information
+        let mut args = vec!["list", "-test", "-compiled", "-json"];
+        args.extend(packages.iter().map(|s| s.as_str()));
+
         let output = Command::new("go")
-            .args(["list", "-test", "-compiled", "-json", "./..."])
+            .args(args)
             .current_dir(go_project_path)
             .output()?;
 
@@ -333,9 +339,11 @@ mod tests {
 
     #[test]
     fn test_discover_benchmarks() {
-        let packages =
-            BenchmarkPackage::from_project(Path::new("testdata/projects/golang-benchmarks"))
-                .unwrap();
+        let packages = BenchmarkPackage::from_project(
+            Path::new("testdata/projects/golang-benchmarks"),
+            &["./...".to_string()],
+        )
+        .unwrap();
 
         insta::assert_json_snapshot!(packages, {
             ".**[\"Dir\"]" => "[package_dir]",

go-runner/src/builder/mod.rs

@@ -4,3 +4,60 @@ pub mod templater;
 pub mod verifier;
 
 pub use discovery::*;
+
+use crate::prelude::*;
+use std::{path::Path, process::Command};
+
+/// Builds a Go runner file into an executable binary
+pub fn build_binary<P: AsRef<Path>>(runner_go_path: P) -> anyhow::Result<std::path::PathBuf> {
+    let runner_go_path = runner_go_path.as_ref();
+    let file_dir = runner_go_path.parent().unwrap();
+    let module_root = file_dir.parent().unwrap();
+    let relative_path = runner_go_path.strip_prefix(module_root).unwrap();
+
+    debug!(
+        "Building codspeed runner binary: {:?} (root = {:?})",
+        module_root.join(relative_path),
+        module_root
+    );
+
+    let binary_path = runner_go_path.with_extension("bin");
+
+    // Set the integration version in our testing library and include debug symbols
+    let ldflags = format!(
+        "-X github.com/CodSpeedHQ/codspeed-go/testing/capi.integrationVersion={} -s=false -w=false",
+        env!("CARGO_PKG_VERSION")
+    );
+
+    let args = vec![
+        "build",
+        "-tags=codspeed",
+        "-ldflags",
+        &ldflags,
+        "-o",
+        binary_path.to_str().unwrap(),
+        relative_path.to_str().unwrap(),
+    ];
+
+    let output = Command::new("go")
+        .args(&args)
+        .current_dir(module_root)
+        .output()
+        .context("Failed to execute go build command")?;
+
+    if !output.status.success() {
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let stderr = String::from_utf8_lossy(&output.stderr);
+
+        warn!("Build command output: {stdout}");
+        warn!("Build command error output: {stderr}");
+
+        bail!(
+            "Failed to build benchmark binary. Exit status: {}",
+            output.status
+        );
+    }
+
+    debug!("Successfully built binary: {binary_path:?}");
+    Ok(binary_path)
+}

go-runner/src/builder/patcher.rs

@@ -5,6 +5,30 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
+pub fn replace_pkg<P: AsRef<Path>>(folder: P) -> anyhow::Result<()> {
+    let codspeed_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
+    let replace_arg = format!(
+        "github.com/CodSpeedHQ/codspeed-go={}",
+        codspeed_root.display()
+    );
+    debug!("Replacing codspeed-go with {}", codspeed_root.display());
+
+    let output = Command::new("go")
+        .args(["mod", "edit", "-replace", &replace_arg])
+        .current_dir(folder.as_ref())
+        .output()
+        .context("Failed to execute 'go mod edit' command")?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        bail!("Failed to add replace directive: {}", stderr);
+    }
+
+    debug!("Added local replace directive to go.mod");
+
+    Ok(())
+}
+
 pub fn patch_imports<P: AsRef<Path>>(
     folder: P,
     files_to_patch: Vec<PathBuf>,
@@ -31,8 +55,8 @@ pub fn patch_imports<P: AsRef<Path>>(
     debug!("Patched {patched_files} files");
 
     // 2. Update the go module to use the codspeed package
-    let version = env!("CARGO_PKG_VERSION");
-    let pkg = format!("github.com/CodSpeedHQ/codspeed-go@v{version}");
+    let version = format!("v{}", env!("CARGO_PKG_VERSION"));
+    let pkg = format!("github.com/CodSpeedHQ/codspeed-go@{version}");
     debug!("Installing {pkg}");
 
     let mut cmd: Command = Command::new("go");
@@ -51,6 +75,12 @@ pub fn patch_imports<P: AsRef<Path>>(
 
     debug!("Successfully installed codspeed-go dependency");
 
+    // Ensure we have the latest codspeed-go package installed. Just
+    // use the local one which might contain uncommitted changes.
+    if std::env::var("GITHUB_ACTIONS").is_ok() || cfg!(test) {
+        replace_pkg(folder)?;
+    }
+
     Ok(())
 }
 

go-runner/src/builder/templater.rs

@@ -9,18 +9,50 @@ use crate::builder::{BenchmarkPackage, GoBenchmark};
 use crate::utils;
 use crate::{builder::patcher, prelude::*};
 
+#[derive(Debug, Serialize)]
+struct GoRunnerMetadata {
+    profile_folder: String,
+    relative_package_path: String,
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 struct TemplateData {
     benchmarks: Vec<GoBenchmark>,
     module_name: String,
 }
 
-pub fn run(package: &BenchmarkPackage) -> anyhow::Result<(TempDir, PathBuf)> {
+pub fn run<P: AsRef<Path>>(
+    package: &BenchmarkPackage,
+    profile_dir: P,
+) -> anyhow::Result<(TempDir, PathBuf)> {
     // 1. Copy the whole module to a build directory
     let target_dir = TempDir::new()?;
     std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?;
     utils::copy_dir_recursively(&package.module.dir, &target_dir)?;
 
+    // Create a new go-runner.metadata file in the root of the project
+    //
+    // The package path will be prepended to the URI. The benchmark will
+    // find the path relative to the root of the `target_dir`.
+    //
+    // This is needed because we could execute a Go project that is a sub-folder
+    // within a Git repository, then we won't copy the .git folder. Therefore, we
+    // have to resolve the .git relative path in go-runner and then combine it.
+    let relative_package_path = utils::get_git_relative_path(&package.dir)
+        .to_string_lossy()
+        .into();
+    debug!("Relative package path: {relative_package_path}");
+
+    let metadata = GoRunnerMetadata {
+        profile_folder: profile_dir.as_ref().to_string_lossy().into(),
+        relative_package_path,
+    };
+    fs::write(
+        target_dir.path().join("go-runner.metadata"),
+        serde_json::to_string_pretty(&metadata)?,
+    )
+    .context("Failed to write go-runner.metadata file")?;
+
     // Get files that need to be renamed first
     let files = package
         .test_go_files