From 1cc4e1be9559e8f0fbd2be2a01812cdec172a851 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 1 Sep 2025 20:33:15 +0200 Subject: [PATCH 1/3] feat: add nested example --- example/very/nested/module/example_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 example/very/nested/module/example_test.go diff --git a/example/very/nested/module/example_test.go b/example/very/nested/module/example_test.go new file mode 100644 index 00000000..1bee739a --- /dev/null +++ b/example/very/nested/module/example_test.go @@ -0,0 +1,9 @@ +package example + +import "testing" + +func BenchmarkExample(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = 42 + } +} From ddd6040a7825ec7e07878d6a81746204f6a2a2ca Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 1 Sep 2025 20:40:02 +0200 Subject: [PATCH 2/3] feat: resolve git-relative path inside benchmark --- example-codspeed/go-runner.metadata | 4 + go-runner/src/builder/templater.rs | 29 +++ go-runner/src/lib.rs | 30 +-- go-runner/src/results/raw_result.rs | 41 ++-- ...s__assert_results_snapshots@example_0.snap | 180 +--------------- ...s__assert_results_snapshots@example_1.snap | 203 ++++++++++++++++++ ...s__assert_results_snapshots@zerolog_1.snap | 14 +- go-runner/src/utils.rs | 2 - testing/testing/benchmark.go | 47 +++- testing/testing/codspeed.go | 68 ++++++ 10 files changed, 371 insertions(+), 247 deletions(-) create mode 100644 example-codspeed/go-runner.metadata create mode 100644 go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_1.snap create mode 100644 testing/testing/codspeed.go diff --git a/example-codspeed/go-runner.metadata b/example-codspeed/go-runner.metadata new file mode 100644 index 00000000..a192f5fa --- /dev/null +++ b/example-codspeed/go-runner.metadata @@ -0,0 +1,4 @@ +{ + "profile_folder": "/tmp", + "relative_package_path": "codspeed-go" +} diff --git a/go-runner/src/builder/templater.rs b/go-runner/src/builder/templater.rs index 0fb96219..8c29ef2a 100644 --- a/go-runner/src/builder/templater.rs +++ b/go-runner/src/builder/templater.rs @@ -9,6 +9,12 @@ use crate::builder::{BenchmarkPackage, GoBenchmark}; use crate::utils; use crate::{builder::patcher, prelude::*}; +#[derive(Debug, Serialize)] +struct GoRunnerMetadata { + profile_folder: String, + relative_package_path: String, +} + #[derive(Debug, Serialize, Deserialize)] struct TemplateData { benchmarks: Vec, @@ -21,6 +27,29 @@ pub fn run(package: &BenchmarkPackage) -> anyhow::Result<(TempDir, PathBuf)> { std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?; utils::copy_dir_recursively(&package.module.dir, &target_dir)?; + // Create a new go-runner.metadata file in the root of the project + // + // The package path will be prepended to the URI. The benchmark will + // find the path relative to the root of the `target_dir`. + // + // This is needed because we could execute a Go project that is a sub-folder + // within a Git repository, then we won't copy the .git folder. Therefore, we + // have to resolve the .git relative path in go-runner and then combine it. + let relative_package_path = utils::get_git_relative_path(&package.dir) + .to_string_lossy() + .into(); + debug!("Relative package path: {relative_package_path}"); + + let metadata = GoRunnerMetadata { + profile_folder: std::env::var("CODSPEED_PROFILE_FOLDER").unwrap_or("/tmp".into()), + relative_package_path, + }; + fs::write( + target_dir.path().join("go-runner.metadata"), + serde_json::to_string_pretty(&metadata)?, + ) + .context("Failed to write go-runner.metadata file")?; + // Get files that need to be renamed first let files = package .test_go_files diff --git a/go-runner/src/lib.rs b/go-runner/src/lib.rs index 60ba8161..5a10e158 100644 --- a/go-runner/src/lib.rs +++ b/go-runner/src/lib.rs @@ -24,22 +24,15 @@ pub fn run_benchmarks(project_dir: &Path, cli: &crate::cli::Cli) -> anyhow::Resu let packages = BenchmarkPackage::from_project(project_dir, &cli.packages)?; info!("Discovered {} packages", packages.len()); - let mut bench_name_to_path = HashMap::new(); + let total_benchmarks: usize = packages.iter().map(|p| p.benchmarks.len()).sum(); + info!("Total benchmarks discovered: {total_benchmarks}"); + for package in &packages { for benchmark in &package.benchmarks { - // Create absolute path and immediately convert to git-relative path - let abs_path = package.module.dir.join(&benchmark.file_path); - let git_relative_path = crate::utils::get_git_relative_path(&abs_path); - bench_name_to_path.insert(benchmark.name.clone(), git_relative_path); + info!("Found {:30} in {:?}", benchmark.name, benchmark.file_path); } } - let total_benchmarks: usize = packages.iter().map(|p| p.benchmarks.len()).sum(); - info!("Total benchmarks discovered: {total_benchmarks}"); - for (name, path) in &bench_name_to_path { - info!("Found {name:30} in {path:?}"); - } - // 2. Generate codspeed runners, build binaries, and execute them for package in &packages { info!("Generating custom runner for package: {}", package.name); @@ -55,13 +48,13 @@ pub fn run_benchmarks(project_dir: &Path, cli: &crate::cli::Cli) -> anyhow::Resu } // 3. Collect the results - collect_walltime_results(bench_name_to_path)?; + collect_walltime_results()?; Ok(()) } // TODO: This should be merged with codspeed-rust/codspeed/walltime_results.rs -fn collect_walltime_results(bench_name_to_path: HashMap) -> anyhow::Result<()> { +fn collect_walltime_results() -> anyhow::Result<()> { let profile_dir = std::env::var("CODSPEED_PROFILE_FOLDER") .context("CODSPEED_PROFILE_FOLDER env var not set")?; let profile_dir = PathBuf::from(&profile_dir); @@ -71,19 +64,10 @@ fn collect_walltime_results(bench_name_to_path: HashMap) -> any let mut benchmarks_by_pid: HashMap> = HashMap::new(); for raw in raw_results { - // We only parse the `func Benchmark*` name which is the first part of the URI - let func_name = raw - .benchmark_name - .split("::") - .next() - .unwrap_or(&raw.benchmark_name); - let file_path = bench_name_to_path - .get(func_name) - .map(|p| p.to_string_lossy().to_string()); benchmarks_by_pid .entry(raw.pid) .or_default() - .push(raw.into_walltime_benchmark(file_path)); + .push(raw.into_walltime_benchmark()); } for (pid, walltime_benchmarks) in benchmarks_by_pid { diff --git a/go-runner/src/results/raw_result.rs b/go-runner/src/results/raw_result.rs index de940382..731c7246 100644 --- a/go-runner/src/results/raw_result.rs +++ b/go-runner/src/results/raw_result.rs @@ -7,7 +7,8 @@ use crate::results::walltime_results::WalltimeBenchmark; // WARN: Keep in sync with Golang "testing" fork (benchmark.go) #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RawResult { - pub benchmark_name: String, + pub name: String, + pub uri: String, pub pid: u32, pub codspeed_time_per_round_ns: Vec, @@ -32,12 +33,7 @@ impl RawResult { .collect()) } - pub fn into_walltime_benchmark(self, file_path: Option) -> WalltimeBenchmark { - let name = self.benchmark_name; - - let file = file_path.as_deref().unwrap_or("unknown"); - let uri = format!("{file}::{name}"); - + pub fn into_walltime_benchmark(self) -> WalltimeBenchmark { let times_per_round_ns = self .codspeed_time_per_round_ns .iter() @@ -52,7 +48,13 @@ impl RawResult { .collect() }; - WalltimeBenchmark::from_runtime_data(name, uri, iters_per_round, times_per_round_ns, None) + WalltimeBenchmark::from_runtime_data( + self.name, + self.uri, + iters_per_round, + times_per_round_ns, + None, + ) } } @@ -63,33 +65,16 @@ mod tests { #[test] fn test_raw_result_deserialization() { let json_data = r#"{ - "benchmark_name": "BenchmarkFibonacci20-16", + "name": "BenchmarkFibonacci20-16", + "uri": "pkg/foo/fib_test.go::BenchmarkFibonacci20-16", "pid": 777767, "codspeed_time_per_round_ns": [1000, 2000, 3000] }"#; let result: RawResult = serde_json::from_str(json_data).unwrap(); - assert_eq!(result.benchmark_name, "BenchmarkFibonacci20-16"); + assert_eq!(result.name, "BenchmarkFibonacci20-16"); assert_eq!(result.pid, 777767); assert_eq!(result.codspeed_time_per_round_ns.len(), 3); assert_eq!(result.codspeed_iters_per_round.len(), 0); // Default: 1 per round } - - #[test] - fn test_into_walltime_benchmark_with_file_path() { - let raw_result = RawResult { - benchmark_name: "BenchmarkFibonacci20-16".to_string(), - pid: 777767, - codspeed_time_per_round_ns: vec![1000, 2000, 3000], - codspeed_iters_per_round: vec![], - }; - - // Test with file path - should not panic and create successfully - let _walltime_bench = raw_result - .clone() - .into_walltime_benchmark(Some("pkg/foo/fib_test.go".to_string())); - - // Test without file path (should default to TODO) - should not panic and create successfully - let _walltime_bench_no_path = raw_result.into_walltime_benchmark(None); - } } diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_0.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_0.snap index fb6cacd1..8fb1cf8f 100644 --- a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_0.snap +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_0.snap @@ -13,184 +13,8 @@ expression: content }, "benchmarks": [ { - "name": "BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", - "uri": "example/fib_test.go::BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkFibonacci20_Loop", - "uri": "example/fib_test.go::BenchmarkFibonacci20_Loop", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkFibonacci20_bN", - "uri": "example/fib_test.go::BenchmarkFibonacci20_bN", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep100ns", - "uri": "example/sleep_test.go::BenchmarkSleep100ns", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep100ns_Loop", - "uri": "example/sleep_test.go::BenchmarkSleep100ns_Loop", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep100us", - "uri": "example/sleep_test.go::BenchmarkSleep100us", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep100us_Loop", - "uri": "example/sleep_test.go::BenchmarkSleep100us_Loop", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep10ms", - "uri": "example/sleep_test.go::BenchmarkSleep10ms", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep10ms_Loop", - "uri": "example/sleep_test.go::BenchmarkSleep10ms_Loop", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep10us", - "uri": "example/sleep_test.go::BenchmarkSleep10us", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep10us_Loop", - "uri": "example/sleep_test.go::BenchmarkSleep10us_Loop", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep1ms", - "uri": "example/sleep_test.go::BenchmarkSleep1ms", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep1ms_Loop", - "uri": "example/sleep_test.go::BenchmarkSleep1ms_Loop", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep1us", - "uri": "example/sleep_test.go::BenchmarkSleep1us", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep1us_Loop", - "uri": "example/sleep_test.go::BenchmarkSleep1us_Loop", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep50ms", - "uri": "example/sleep_test.go::BenchmarkSleep50ms", - "config": { - "warmup_time_ns": null, - "min_round_time_ns": null, - "max_time_ns": null, - "max_rounds": null - }, - "stats": "[stats]" - }, - { - "name": "BenchmarkSleep50ms_Loop", - "uri": "example/sleep_test.go::BenchmarkSleep50ms_Loop", + "name": "BenchmarkExample", + "uri": "example/very/nested/module/example_test.go::BenchmarkExample", "config": { "warmup_time_ns": null, "min_round_time_ns": null, diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_1.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_1.snap new file mode 100644 index 00000000..fb6cacd1 --- /dev/null +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_1.snap @@ -0,0 +1,203 @@ +--- +source: go-runner/src/integration_tests.rs +expression: content +--- +{ + "creator": { + "name": "codspeed-go", + "version": "0.1.0", + "pid": "[pid]" + }, + "instrument": { + "type": "walltime" + }, + "benchmarks": [ + { + "name": "BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", + "uri": "example/fib_test.go::BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkFibonacci20_Loop", + "uri": "example/fib_test.go::BenchmarkFibonacci20_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkFibonacci20_bN", + "uri": "example/fib_test.go::BenchmarkFibonacci20_bN", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100ns", + "uri": "example/sleep_test.go::BenchmarkSleep100ns", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100ns_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep100ns_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100us", + "uri": "example/sleep_test.go::BenchmarkSleep100us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep100us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10ms", + "uri": "example/sleep_test.go::BenchmarkSleep10ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep10ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10us", + "uri": "example/sleep_test.go::BenchmarkSleep10us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep10us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1ms", + "uri": "example/sleep_test.go::BenchmarkSleep1ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep1ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1us", + "uri": "example/sleep_test.go::BenchmarkSleep1us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep1us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep50ms", + "uri": "example/sleep_test.go::BenchmarkSleep50ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep50ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep50ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + } + ] +} diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap index 1d486fc1..aeb6aa35 100644 --- a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap @@ -146,7 +146,7 @@ expression: content }, { "name": "BenchmarkAppendString::EncodingFirst", - "uri": "internal/json/string_test.go::BenchmarkAppendString::EncodingFirst", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::EncodingFirst", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -157,7 +157,7 @@ expression: content }, { "name": "BenchmarkAppendString::EncodingLast", - "uri": "internal/json/string_test.go::BenchmarkAppendString::EncodingLast", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::EncodingLast", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -168,7 +168,7 @@ expression: content }, { "name": "BenchmarkAppendString::EncodingMiddle", - "uri": "internal/json/string_test.go::BenchmarkAppendString::EncodingMiddle", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::EncodingMiddle", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -179,7 +179,7 @@ expression: content }, { "name": "BenchmarkAppendString::MultiBytesFirst", - "uri": "internal/json/string_test.go::BenchmarkAppendString::MultiBytesFirst", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::MultiBytesFirst", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -190,7 +190,7 @@ expression: content }, { "name": "BenchmarkAppendString::MultiBytesLast", - "uri": "internal/json/string_test.go::BenchmarkAppendString::MultiBytesLast", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::MultiBytesLast", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -201,7 +201,7 @@ expression: content }, { "name": "BenchmarkAppendString::MultiBytesMiddle", - "uri": "internal/json/string_test.go::BenchmarkAppendString::MultiBytesMiddle", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::MultiBytesMiddle", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -212,7 +212,7 @@ expression: content }, { "name": "BenchmarkAppendString::NoEncoding", - "uri": "internal/json/string_test.go::BenchmarkAppendString::NoEncoding", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::NoEncoding", "config": { "warmup_time_ns": null, "min_round_time_ns": null, diff --git a/go-runner/src/utils.rs b/go-runner/src/utils.rs index 2ef3a05d..2a5be482 100644 --- a/go-runner/src/utils.rs +++ b/go-runner/src/utils.rs @@ -19,8 +19,6 @@ pub fn copy_dir_recursively(src: impl AsRef, dst: impl AsRef) -> io: Ok(()) } -// WARN: Git-related helper functions have been taken from codspeed-rust. Keep in sync! - fn get_parent_git_repo_path(abs_path: &Path) -> io::Result { if abs_path.join(".git").exists() { Ok(abs_path.to_path_buf()) diff --git a/testing/testing/benchmark.go b/testing/testing/benchmark.go index 2abb06d8..be424462 100644 --- a/testing/testing/benchmark.go +++ b/testing/testing/benchmark.go @@ -15,6 +15,7 @@ import ( "math" "os" "path/filepath" + "reflect" "runtime" "slices" "strconv" @@ -834,12 +835,37 @@ func (s *benchState) processBench(b *B) { // ############################################################################################ // START CODSPEED type RawResults struct { - BenchmarkName string `json:"benchmark_name"` + Name string `json:"name"` + Uri string `json:"uri"` Pid int `json:"pid"` CodspeedTimePerRoundNs []time.Duration `json:"codspeed_time_per_round_ns"` CodspeedItersPerRound []int64 `json:"codspeed_iters_per_round"` } + // Find the filename of the benchmark file + var benchFile string + if b.benchFunc != nil { + pc := reflect.ValueOf(b.benchFunc).Pointer() + fn := runtime.FuncForPC(pc) + if fn == nil { + continue + } + + file, _ := fn.FileLine(pc) + if strings.HasSuffix(file, "_codspeed.go") { + benchFile = file + } + } + + if benchFile == "" { + panic("Could not determine benchmark file name") + } + + relativeBenchFile := getGitRelativePath(benchFile) + if strings.HasSuffix(relativeBenchFile, "_codspeed.go") { + relativeBenchFile = strings.TrimSuffix(relativeBenchFile, "_codspeed.go") + "_test.go" + } + // Build custom bench name with :: separator var nameParts []string current := &b.common @@ -858,20 +884,23 @@ func (s *benchState) processBench(b *B) { } current = current.parent } - customBenchName := strings.Join(nameParts, "::") + benchName = strings.Join(nameParts, "::") + benchUri := fmt.Sprintf("%s::%s", relativeBenchFile, benchName) rawResults := RawResults{ - BenchmarkName: customBenchName, + Name: benchName, + Uri: benchUri, Pid: os.Getpid(), CodspeedTimePerRoundNs: r.CodspeedTimePerRoundNs, CodspeedItersPerRound: r.CodspeedItersPerRound, } - codspeedProfileFolder := os.Getenv("CODSPEED_PROFILE_FOLDER") - if codspeedProfileFolder == "" { - panic("CODSPEED_PROFILE_FOLDER environment variable is not set") + goRunnerMetadata, err := findGoRunnerMetadata() + if err != nil { + panic(fmt.Sprintf("failed to get go runner metadata: %v", err)) } - if err := os.MkdirAll(filepath.Join(codspeedProfileFolder, "raw_results"), 0755); err != nil { + + if err := os.MkdirAll(filepath.Join(goRunnerMetadata.ProfileFolder, "raw_results"), 0755); err != nil { fmt.Fprintf(os.Stderr, "failed to create raw results directory: %v\n", err) continue } @@ -881,7 +910,7 @@ func (s *benchState) processBench(b *B) { fmt.Fprintf(os.Stderr, "failed to generate random filename: %v\n", err) continue } - rawResultsFile := filepath.Join(codspeedProfileFolder, "raw_results", fmt.Sprintf("%s.json", hex.EncodeToString(randomBytes))) + rawResultsFile := filepath.Join(goRunnerMetadata.ProfileFolder, "raw_results", fmt.Sprintf("%s.json", hex.EncodeToString(randomBytes))) file, err := os.Create(rawResultsFile) if err != nil { fmt.Fprintf(os.Stderr, "failed to create raw results file: %v\n", err) @@ -902,7 +931,7 @@ func (s *benchState) processBench(b *B) { defer file.Close() // Send pid and executed benchmark to the runner - b.codspeed.instrument_hooks.SetExecutedBenchmark(uint32(os.Getpid()), customBenchName) + b.codspeed.instrument_hooks.SetExecutedBenchmark(uint32(os.Getpid()), benchUri) // END CODSPEED // ############################################################################################ diff --git a/testing/testing/codspeed.go b/testing/testing/codspeed.go new file mode 100644 index 00000000..40b4162a --- /dev/null +++ b/testing/testing/codspeed.go @@ -0,0 +1,68 @@ +package testing + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +type GoRunnerMetadata struct { + ProfileFolder string `json:"profile_folder"` + RelativePackagePath string `json:"relative_package_path"` +} + +func findGoRunnerMetadata() (*GoRunnerMetadata, error) { + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + + // Search up the directory tree for go-runner.metadata + currentDir := cwd + for { + metadataPath := filepath.Join(currentDir, "go-runner.metadata") + data, err := os.ReadFile(metadataPath) + if err == nil { + var metadata GoRunnerMetadata + err = json.Unmarshal(data, &metadata) + if err != nil { + return nil, err + } + return &metadata, nil + } + + parentDir := filepath.Dir(currentDir) + if parentDir == currentDir { + // Reached the root directory + break + } + currentDir = parentDir + } + + return nil, os.ErrNotExist +} + +func getGitRelativePath(absPath string) string { + canonicalizedAbsPath, err := filepath.EvalSymlinks(absPath) + if err != nil { + panic(fmt.Sprintf("failed to evaluate symlinks for path %s: %v", absPath, err)) + } + + cwd, err := os.Getwd() + if err != nil { + panic(fmt.Sprintf("failed to get current working directory: %v", err)) + } + + cwdRelativePath, err := filepath.Rel(cwd, canonicalizedAbsPath) + if err != nil { + panic(fmt.Sprintf("failed to compute relative path from %s to %s: %v", cwd, canonicalizedAbsPath, err)) + } + + goRunnerMetadata, err := findGoRunnerMetadata() + if err != nil { + panic(fmt.Sprintf("failed to find go-runner metadata: %v", err)) + } + + return filepath.Join(goRunnerMetadata.RelativePackagePath, cwdRelativePath) +} From b20d4396aadaa06fa016f1292bed111a32961c06 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 1 Sep 2025 20:52:51 +0200 Subject: [PATCH 3/3] fix: remove profile env var mutex in test --- go-runner/src/builder/templater.rs | 7 +++++-- go-runner/src/integration_tests.rs | 8 +------- go-runner/src/lib.rs | 24 ++++++++++-------------- go-runner/src/main.rs | 3 ++- go-runner/tests/utils.rs | 13 ++----------- 5 files changed, 20 insertions(+), 35 deletions(-) diff --git a/go-runner/src/builder/templater.rs b/go-runner/src/builder/templater.rs index 8c29ef2a..9122f120 100644 --- a/go-runner/src/builder/templater.rs +++ b/go-runner/src/builder/templater.rs @@ -21,7 +21,10 @@ struct TemplateData { module_name: String, } -pub fn run(package: &BenchmarkPackage) -> anyhow::Result<(TempDir, PathBuf)> { +pub fn run>( + package: &BenchmarkPackage, + profile_dir: P, +) -> anyhow::Result<(TempDir, PathBuf)> { // 1. Copy the whole module to a build directory let target_dir = TempDir::new()?; std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?; @@ -41,7 +44,7 @@ pub fn run(package: &BenchmarkPackage) -> anyhow::Result<(TempDir, PathBuf)> { debug!("Relative package path: {relative_package_path}"); let metadata = GoRunnerMetadata { - profile_folder: std::env::var("CODSPEED_PROFILE_FOLDER").unwrap_or("/tmp".into()), + profile_folder: profile_dir.as_ref().to_string_lossy().into(), relative_package_path, }; fs::write( diff --git a/go-runner/src/integration_tests.rs b/go-runner/src/integration_tests.rs index 5f0f5f5b..30254935 100644 --- a/go-runner/src/integration_tests.rs +++ b/go-runner/src/integration_tests.rs @@ -1,7 +1,6 @@ use itertools::Itertools; use rstest::rstest; use std::path::{Path, PathBuf}; -use std::sync::Mutex; use tempfile::TempDir; use crate::results::walltime_results::WalltimeResults; @@ -66,18 +65,13 @@ fn test_build_and_run(#[case] project_name: &str) { .join("testdata/projects") .join(project_name); - // Mutex to prevent concurrent tests from interfering with CODSPEED_PROFILE_FOLDER env var - static ENV_MUTEX: Mutex<()> = Mutex::new(()); - let _env_guard = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner()); - let temp_dir = TempDir::new().unwrap(); let profile_dir = temp_dir.path().join("profile"); - unsafe { std::env::set_var("CODSPEED_PROFILE_FOLDER", &profile_dir) }; let cli = crate::cli::Cli { benchtime: "1x".into(), ..Default::default() }; - if let Err(error) = crate::run_benchmarks(project_dir.as_path(), &cli) { + if let Err(error) = crate::run_benchmarks(&profile_dir, project_dir.as_path(), &cli) { panic!("Benchmarks couldn't run: {error}"); } diff --git a/go-runner/src/lib.rs b/go-runner/src/lib.rs index 5a10e158..7d4cb0bc 100644 --- a/go-runner/src/lib.rs +++ b/go-runner/src/lib.rs @@ -1,8 +1,5 @@ use crate::{builder::BenchmarkPackage, prelude::*}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; +use std::{collections::HashMap, path::Path}; pub mod builder; pub mod cli; @@ -15,9 +12,11 @@ pub(crate) mod utils; mod integration_tests; /// Builds and runs the specified Go project benchmarks, writing results to the .codspeed folder. -pub fn run_benchmarks(project_dir: &Path, cli: &crate::cli::Cli) -> anyhow::Result<()> { - let profile_dir = std::env::var("CODSPEED_PROFILE_FOLDER") - .context("CODSPEED_PROFILE_FOLDER env var not set")?; +pub fn run_benchmarks>( + profile_dir: P, + project_dir: &Path, + cli: &crate::cli::Cli, +) -> anyhow::Result<()> { std::fs::remove_dir_all(&profile_dir).ok(); // 1. Build phase - Benchmark and package discovery @@ -36,7 +35,7 @@ pub fn run_benchmarks(project_dir: &Path, cli: &crate::cli::Cli) -> anyhow::Resu // 2. Generate codspeed runners, build binaries, and execute them for package in &packages { info!("Generating custom runner for package: {}", package.name); - let (_target_dir, runner_path) = builder::templater::run(package)?; + let (_target_dir, runner_path) = builder::templater::run(package, &profile_dir)?; info!("Building binary for package: {}", package.name); let binary_path = builder::build_binary(&runner_path)?; @@ -48,17 +47,14 @@ pub fn run_benchmarks(project_dir: &Path, cli: &crate::cli::Cli) -> anyhow::Resu } // 3. Collect the results - collect_walltime_results()?; + collect_walltime_results(profile_dir.as_ref())?; Ok(()) } // TODO: This should be merged with codspeed-rust/codspeed/walltime_results.rs -fn collect_walltime_results() -> anyhow::Result<()> { - let profile_dir = std::env::var("CODSPEED_PROFILE_FOLDER") - .context("CODSPEED_PROFILE_FOLDER env var not set")?; - let profile_dir = PathBuf::from(&profile_dir); - let raw_results = results::raw_result::RawResult::parse_folder(&profile_dir)?; +fn collect_walltime_results(profile_dir: &Path) -> anyhow::Result<()> { + let raw_results = results::raw_result::RawResult::parse_folder(profile_dir)?; info!("Parsed {} raw results", raw_results.len()); let mut benchmarks_by_pid: HashMap> = diff --git a/go-runner/src/main.rs b/go-runner/src/main.rs index bd78b8d5..9a0b08cd 100644 --- a/go-runner/src/main.rs +++ b/go-runner/src/main.rs @@ -9,7 +9,8 @@ fn main() -> anyhow::Result<()> { .init(); let cli = Cli::parse(); - codspeed_go_runner::run_benchmarks(Path::new("."), &cli)?; + let profile_dir = std::env::var("CODSPEED_PROFILE_FOLDER").unwrap_or("/tmp".into()); + codspeed_go_runner::run_benchmarks(profile_dir, Path::new("."), &cli)?; Ok(()) } diff --git a/go-runner/tests/utils.rs b/go-runner/tests/utils.rs index 8ab713d1..4794b96b 100644 --- a/go-runner/tests/utils.rs +++ b/go-runner/tests/utils.rs @@ -1,19 +1,10 @@ use codspeed_go_runner::{builder, builder::BenchmarkPackage, cli::Cli, runner}; use std::path::Path; -use std::sync::Mutex; -use tempfile::TempDir; /// Helper function to run a single package with arguments pub fn run_package_with_args(package: &BenchmarkPackage, args: &[&str]) -> anyhow::Result { - // Mutex to prevent concurrent tests from interfering with CODSPEED_PROFILE_FOLDER env var - static ENV_MUTEX: Mutex<()> = Mutex::new(()); - let _env_guard = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner()); - - let temp_dir = TempDir::new()?; - let profile_dir = temp_dir.path().join("profile"); - unsafe { std::env::set_var("CODSPEED_PROFILE_FOLDER", &profile_dir) }; - - let (_dir, runner_path) = builder::templater::run(package)?; + let profile_dir = tempfile::TempDir::new()?; + let (_dir, runner_path) = builder::templater::run(package, profile_dir.as_ref())?; let binary_path = builder::build_binary(&runner_path)?; runner::run_with_stdout(&binary_path, args) }