feat: dx improvements for codspeed go

README.md

@@ -38,7 +38,7 @@ $ go-runner test -bench=.
 [INFO  go_runner] Found BenchmarkFibonacci20_Loop      in "fib_test.go"
 [INFO  go_runner] Generating custom runner for package: example
 [INFO  go_runner] Running benchmarks for package: example
-Running with CodSpeed instrumentation
+Running with CodSpeed (mode: walltime)
 goos: linux
 goarch: amd64
 cpu: 12th Gen Intel(R) Core(TM) i7-1260P @ 1672.130MHz

example-codspeed/go-runner.metadata

@@ -0,0 +1,4 @@
+{
+    "profile_folder": "/tmp",
+    "relative_package_path": "codspeed-go"
+}

example/very/nested/module/example_test.go

@@ -0,0 +1,9 @@
+package example
+
+import "testing"
+
+func BenchmarkExample(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = 42
+	}
+}

go-runner/src/builder/discovery.rs

@@ -233,8 +233,11 @@ impl BenchmarkPackage {
         }
     }
 
-    pub fn from_project(go_project_path: &Path) -> anyhow::Result<Vec<BenchmarkPackage>> {
-        let raw_packages = Self::run_go_list(go_project_path)?;
+    pub fn from_project(
+        go_project_path: &Path,
+        packages: &[String],
+    ) -> anyhow::Result<Vec<BenchmarkPackage>> {
+        let raw_packages = Self::run_go_list(go_project_path, packages)?;
         let has_test_files =
             |files: &Vec<String>| files.iter().any(|name| name.ends_with("_test.go"));
         let has_test_imports = |imports: &Vec<String>| {
@@ -297,10 +300,13 @@ impl BenchmarkPackage {
         Ok(packages)
     }
 
-    fn run_go_list(go_project_path: &Path) -> anyhow::Result<Vec<GoPackage>> {
-        // Execute 'go list -test -compiled -json ./...' to get package information
+    fn run_go_list(go_project_path: &Path, packages: &[String]) -> anyhow::Result<Vec<GoPackage>> {
+        // Execute 'go list -test -compiled -json <packages>' to get package information
+        let mut args = vec!["list", "-test", "-compiled", "-json"];
+        args.extend(packages.iter().map(|s| s.as_str()));
+
         let output = Command::new("go")
-            .args(["list", "-test", "-compiled", "-json", "./..."])
+            .args(args)
             .current_dir(go_project_path)
             .output()?;
 
@@ -333,9 +339,11 @@ mod tests {
 
     #[test]
     fn test_discover_benchmarks() {
-        let packages =
-            BenchmarkPackage::from_project(Path::new("testdata/projects/golang-benchmarks"))
-                .unwrap();
+        let packages = BenchmarkPackage::from_project(
+            Path::new("testdata/projects/golang-benchmarks"),
+            &["./...".to_string()],
+        )
+        .unwrap();
 
         insta::assert_json_snapshot!(packages, {
             ".**[\"Dir\"]" => "[package_dir]",

go-runner/src/builder/templater.rs

@@ -9,18 +9,50 @@ use crate::builder::{BenchmarkPackage, GoBenchmark};
 use crate::utils;
 use crate::{builder::patcher, prelude::*};
 
+#[derive(Debug, Serialize)]
+struct GoRunnerMetadata {
+    profile_folder: String,
+    relative_package_path: String,
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 struct TemplateData {
     benchmarks: Vec<GoBenchmark>,
     module_name: String,
 }
 
-pub fn run(package: &BenchmarkPackage) -> anyhow::Result<(TempDir, PathBuf)> {
+pub fn run<P: AsRef<Path>>(
+    package: &BenchmarkPackage,
+    profile_dir: P,
+) -> anyhow::Result<(TempDir, PathBuf)> {
     // 1. Copy the whole module to a build directory
     let target_dir = TempDir::new()?;
     std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?;
     utils::copy_dir_recursively(&package.module.dir, &target_dir)?;
 
+    // Create a new go-runner.metadata file in the root of the project
+    //
+    // The package path will be prepended to the URI. The benchmark will
+    // find the path relative to the root of the `target_dir`.
+    //
+    // This is needed because we could execute a Go project that is a sub-folder
+    // within a Git repository, then we won't copy the .git folder. Therefore, we
+    // have to resolve the .git relative path in go-runner and then combine it.
+    let relative_package_path = utils::get_git_relative_path(&package.dir)
+        .to_string_lossy()
+        .into();
+    debug!("Relative package path: {relative_package_path}");
+
+    let metadata = GoRunnerMetadata {
+        profile_folder: profile_dir.as_ref().to_string_lossy().into(),
+        relative_package_path,
+    };
+    fs::write(
+        target_dir.path().join("go-runner.metadata"),
+        serde_json::to_string_pretty(&metadata)?,
+    )
+    .context("Failed to write go-runner.metadata file")?;
+
     // Get files that need to be renamed first
     let files = package
         .test_go_files

go-runner/src/cli.rs

@@ -10,6 +10,22 @@ pub enum CliExit {
 pub struct Cli {
     /// Run only benchmarks matching regexp
     pub bench: String,
+
+    /// Run each benchmark for duration d (e.g., '3s')
+    pub benchtime: String,
+
+    /// Package patterns to run benchmarks for
+    pub packages: Vec<String>,
+}
+
+impl Default for Cli {
+    fn default() -> Self {
+        Self {
+            bench: ".".into(),
+            benchtime: "3s".into(),
+            packages: vec!["./...".into()],
+        }
+    }
 }
 
 impl Cli {
@@ -24,7 +40,7 @@ impl Cli {
     }
 
     fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Self, CliExit> {
-        let mut bench = ".".to_string();
+        let mut instance = Self::default();
 
         // We currently only support the `test` subcommand.
         let cmd = args.next();
@@ -41,12 +57,19 @@ impl Cli {
 The Codspeed Go Benchmark Runner
 
 USAGE:
-    go-runner test [OPTIONS]
+    go-runner test [OPTIONS] [PACKAGES...]
 
 OPTIONS:
     -bench <pattern>     Run only benchmarks matching regexp (defaults to '.')
+    -benchtime <duration> Run each benchmark for duration d (defaults to '3s')
     -h, --help           Print help information
-    -V, --version        Print version information"
+    -V, --version        Print version information
+
+SUPPORTED FLAGS:
+    -bench, -benchtime
+
+UNSUPPORTED FLAGS (will be warned about):
+    -benchmem, -count, -cpu, -cpuprofile, -memprofile, -trace, etc."
                     );
                     return Err(CliExit::Help);
                 }
@@ -55,35 +78,40 @@ OPTIONS:
                     return Err(CliExit::Version);
                 }
                 "-bench" => {
-                    bench = args.next().ok_or_else(|| {
+                    instance.bench = args.next().ok_or_else(|| {
                         eprintln!("error: `-bench` requires a pattern");
                         CliExit::MissingArgument
                     })?;
                 }
                 s if s.starts_with("-bench=") => {
-                    bench = s.split_once('=').unwrap().1.to_string();
+                    instance.bench = s.split_once('=').unwrap().1.to_string();
                 }
-
-                s if s.starts_with('-') => {
-                    eprintln!("Unknown flag: {s}");
-                    return Err(CliExit::UnknownFlag);
+                "-benchtime" => {
+                    instance.benchtime = args.next().ok_or_else(|| {
+                        eprintln!("error: `-benchtime` requires a duration");
+                        CliExit::MissingArgument
+                    })?;
                 }
-
-                _ => {
+                s if s.starts_with("-benchtime=") => {
+                    instance.benchtime = s.split_once('=').unwrap().1.to_string();
+                }
+                s if s.starts_with('-') => {
                     eprintln!(
-                        "warning: package arguments are not currently supported, ignoring '{arg}'"
+                        "warning: flag '{s}' is not supported by CodSpeed Go runner, ignoring"
                     );
-                    // Consume and ignore all remaining arguments
-                    for remaining_arg in args {
-                        eprintln!(
-                            "warning: package arguments are not currently supported, ignoring '{remaining_arg}'"
-                        );
-                    }
+                }
+                _ => {
+                    // Collect package arguments for filtering
+                    instance.packages = {
+                        let mut packages = vec![arg];
+                        packages.extend(args);
+                        packages
+                    };
                     break;
                 }
             }
         }
-        Ok(Self { bench })
+        Ok(instance)
     }
 }
 
@@ -107,6 +135,8 @@ mod tests {
     fn test_cli_parse_defaults() {
         let cli = str_to_iter("go-runner test").unwrap();
         assert_eq!(cli.bench, ".");
+        assert_eq!(cli.benchtime, Cli::default().benchtime);
+        assert_eq!(cli.packages, Cli::default().packages);
     }
 
     #[test]
@@ -119,9 +149,30 @@ mod tests {
     }
 
     #[test]
-    fn test_cli_parse_ignores_packages() {
+    fn test_cli_parse_with_benchtime_flag() {
+        let cli = str_to_iter("go-runner test -benchtime 3s").unwrap();
+        assert_eq!(cli.benchtime, "3s".to_string());
+
+        let cli = str_to_iter("go-runner test -benchtime=10x").unwrap();
+        assert_eq!(cli.benchtime, "10x".to_string());
+    }
+
+    #[test]
+    fn test_cli_parse_with_packages() {
         let cli = str_to_iter("go-runner test package1 package2").unwrap();
         assert_eq!(cli.bench, ".");
+        assert_eq!(
+            cli.packages,
+            vec!["package1".to_string(), "package2".to_string()]
+        );
+    }
+
+    #[test]
+    fn test_cli_parse_combined_flags() {
+        let cli = str_to_iter("go-runner test -bench=BenchmarkFoo -benchtime 5s ./pkg").unwrap();
+        assert_eq!(cli.bench, "BenchmarkFoo");
+        assert_eq!(cli.benchtime, "5s".to_string());
+        assert_eq!(cli.packages, vec!["./pkg".to_string()]);
     }
 
     #[test]
@@ -147,7 +198,11 @@ mod tests {
         let result = str_to_iter("go-runner test -bench");
         assert!(matches!(result, Err(CliExit::MissingArgument)));
 
+        let result = str_to_iter("go-runner test -benchtime");
+        assert!(matches!(result, Err(CliExit::MissingArgument)));
+
+        // Unknown flags now generate warnings but don't cause errors
         let result = str_to_iter("go-runner test -unknown");
-        assert!(matches!(result, Err(CliExit::UnknownFlag)));
+        assert!(result.is_ok());
     }
 }

go-runner/src/integration_tests.rs

@@ -1,23 +1,10 @@
 use itertools::Itertools;
 use rstest::rstest;
 use std::path::{Path, PathBuf};
-use std::sync::Mutex;
 use tempfile::TempDir;
 
 use crate::results::walltime_results::WalltimeResults;
 
-fn setup_test_project(project_name: &str) -> anyhow::Result<TempDir> {
-    let project_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("testdata/projects")
-        .join(project_name);
-    println!("Project path: {project_path:?}");
-
-    let temp_dir = TempDir::new()?;
-    crate::utils::copy_dir_recursively(&project_path, &temp_dir)?;
-
-    Ok(temp_dir)
-}
-
 fn assert_results_snapshots(profile_dir: &Path, project_name: &str) {
     let glob_pattern = profile_dir.join("results");
     if !glob_pattern.exists() {
@@ -72,16 +59,19 @@ fn assert_results_snapshots(profile_dir: &Path, project_name: &str) {
 // Currently not producing results:
 #[case::fuego("fuego")]
 #[case::cli_runtime("cli-runtime")]
+#[case::example("example")]
 fn test_build_and_run(#[case] project_name: &str) {
-    let temp_dir = setup_test_project(project_name).unwrap();
-
-    // Mutex to prevent concurrent tests from interfering with CODSPEED_PROFILE_FOLDER env var
-    static ENV_MUTEX: Mutex<()> = Mutex::new(());
-    let _env_guard = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
+    let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("testdata/projects")
+        .join(project_name);
 
+    let temp_dir = TempDir::new().unwrap();
     let profile_dir = temp_dir.path().join("profile");
-    unsafe { std::env::set_var("CODSPEED_PROFILE_FOLDER", &profile_dir) };
-    if let Err(error) = crate::run_benchmarks(temp_dir.path(), ".") {
+    let cli = crate::cli::Cli {
+        benchtime: "1x".into(),
+        ..Default::default()
+    };
+    if let Err(error) = crate::run_benchmarks(&profile_dir, project_dir.as_path(), &cli) {
         panic!("Benchmarks couldn't run: {error}");
     }