diff --git a/README.md b/README.md index 93c1cb56..215fb002 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ $ go-runner test -bench=. [INFO go_runner] Found BenchmarkFibonacci20_Loop in "fib_test.go" [INFO go_runner] Generating custom runner for package: example [INFO go_runner] Running benchmarks for package: example -Running with CodSpeed instrumentation +Running with CodSpeed (mode: walltime) goos: linux goarch: amd64 cpu: 12th Gen Intel(R) Core(TM) i7-1260P @ 1672.130MHz diff --git a/example-codspeed/go-runner.metadata b/example-codspeed/go-runner.metadata new file mode 100644 index 00000000..a192f5fa --- /dev/null +++ b/example-codspeed/go-runner.metadata @@ -0,0 +1,4 @@ +{ + "profile_folder": "/tmp", + "relative_package_path": "codspeed-go" +} diff --git a/example/very/nested/module/example_test.go b/example/very/nested/module/example_test.go new file mode 100644 index 00000000..1bee739a --- /dev/null +++ b/example/very/nested/module/example_test.go @@ -0,0 +1,9 @@ +package example + +import "testing" + +func BenchmarkExample(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = 42 + } +} diff --git a/go-runner/src/builder/discovery.rs b/go-runner/src/builder/discovery.rs index 29323408..c22fb5fe 100644 --- a/go-runner/src/builder/discovery.rs +++ b/go-runner/src/builder/discovery.rs @@ -233,8 +233,11 @@ impl BenchmarkPackage { } } - pub fn from_project(go_project_path: &Path) -> anyhow::Result> { - let raw_packages = Self::run_go_list(go_project_path)?; + pub fn from_project( + go_project_path: &Path, + packages: &[String], + ) -> anyhow::Result> { + let raw_packages = Self::run_go_list(go_project_path, packages)?; let has_test_files = |files: &Vec| files.iter().any(|name| name.ends_with("_test.go")); let has_test_imports = |imports: &Vec| { @@ -297,10 +300,13 @@ impl BenchmarkPackage { Ok(packages) } - fn run_go_list(go_project_path: &Path) -> anyhow::Result> { - // Execute 'go list -test -compiled -json ./...' to get package information + fn run_go_list(go_project_path: &Path, packages: &[String]) -> anyhow::Result> { + // Execute 'go list -test -compiled -json ' to get package information + let mut args = vec!["list", "-test", "-compiled", "-json"]; + args.extend(packages.iter().map(|s| s.as_str())); + let output = Command::new("go") - .args(["list", "-test", "-compiled", "-json", "./..."]) + .args(args) .current_dir(go_project_path) .output()?; @@ -333,9 +339,11 @@ mod tests { #[test] fn test_discover_benchmarks() { - let packages = - BenchmarkPackage::from_project(Path::new("testdata/projects/golang-benchmarks")) - .unwrap(); + let packages = BenchmarkPackage::from_project( + Path::new("testdata/projects/golang-benchmarks"), + &["./...".to_string()], + ) + .unwrap(); insta::assert_json_snapshot!(packages, { ".**[\"Dir\"]" => "[package_dir]", diff --git a/go-runner/src/builder/templater.rs b/go-runner/src/builder/templater.rs index 0fb96219..9122f120 100644 --- a/go-runner/src/builder/templater.rs +++ b/go-runner/src/builder/templater.rs @@ -9,18 +9,50 @@ use crate::builder::{BenchmarkPackage, GoBenchmark}; use crate::utils; use crate::{builder::patcher, prelude::*}; +#[derive(Debug, Serialize)] +struct GoRunnerMetadata { + profile_folder: String, + relative_package_path: String, +} + #[derive(Debug, Serialize, Deserialize)] struct TemplateData { benchmarks: Vec, module_name: String, } -pub fn run(package: &BenchmarkPackage) -> anyhow::Result<(TempDir, PathBuf)> { +pub fn run>( + package: &BenchmarkPackage, + profile_dir: P, +) -> anyhow::Result<(TempDir, PathBuf)> { // 1. Copy the whole module to a build directory let target_dir = TempDir::new()?; std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?; utils::copy_dir_recursively(&package.module.dir, &target_dir)?; + // Create a new go-runner.metadata file in the root of the project + // + // The package path will be prepended to the URI. The benchmark will + // find the path relative to the root of the `target_dir`. + // + // This is needed because we could execute a Go project that is a sub-folder + // within a Git repository, then we won't copy the .git folder. Therefore, we + // have to resolve the .git relative path in go-runner and then combine it. + let relative_package_path = utils::get_git_relative_path(&package.dir) + .to_string_lossy() + .into(); + debug!("Relative package path: {relative_package_path}"); + + let metadata = GoRunnerMetadata { + profile_folder: profile_dir.as_ref().to_string_lossy().into(), + relative_package_path, + }; + fs::write( + target_dir.path().join("go-runner.metadata"), + serde_json::to_string_pretty(&metadata)?, + ) + .context("Failed to write go-runner.metadata file")?; + // Get files that need to be renamed first let files = package .test_go_files diff --git a/go-runner/src/cli.rs b/go-runner/src/cli.rs index 8b6d93d3..8bc2e2de 100644 --- a/go-runner/src/cli.rs +++ b/go-runner/src/cli.rs @@ -10,6 +10,22 @@ pub enum CliExit { pub struct Cli { /// Run only benchmarks matching regexp pub bench: String, + + /// Run each benchmark for duration d (e.g., '3s') + pub benchtime: String, + + /// Package patterns to run benchmarks for + pub packages: Vec, +} + +impl Default for Cli { + fn default() -> Self { + Self { + bench: ".".into(), + benchtime: "3s".into(), + packages: vec!["./...".into()], + } + } } impl Cli { @@ -24,7 +40,7 @@ impl Cli { } fn parse_args(mut args: impl Iterator) -> Result { - let mut bench = ".".to_string(); + let mut instance = Self::default(); // We currently only support the `test` subcommand. let cmd = args.next(); @@ -41,12 +57,19 @@ impl Cli { The Codspeed Go Benchmark Runner USAGE: - go-runner test [OPTIONS] + go-runner test [OPTIONS] [PACKAGES...] OPTIONS: -bench Run only benchmarks matching regexp (defaults to '.') + -benchtime Run each benchmark for duration d (defaults to '3s') -h, --help Print help information - -V, --version Print version information" + -V, --version Print version information + +SUPPORTED FLAGS: + -bench, -benchtime + +UNSUPPORTED FLAGS (will be warned about): + -benchmem, -count, -cpu, -cpuprofile, -memprofile, -trace, etc." ); return Err(CliExit::Help); } @@ -55,35 +78,40 @@ OPTIONS: return Err(CliExit::Version); } "-bench" => { - bench = args.next().ok_or_else(|| { + instance.bench = args.next().ok_or_else(|| { eprintln!("error: `-bench` requires a pattern"); CliExit::MissingArgument })?; } s if s.starts_with("-bench=") => { - bench = s.split_once('=').unwrap().1.to_string(); + instance.bench = s.split_once('=').unwrap().1.to_string(); } - - s if s.starts_with('-') => { - eprintln!("Unknown flag: {s}"); - return Err(CliExit::UnknownFlag); + "-benchtime" => { + instance.benchtime = args.next().ok_or_else(|| { + eprintln!("error: `-benchtime` requires a duration"); + CliExit::MissingArgument + })?; } - - _ => { + s if s.starts_with("-benchtime=") => { + instance.benchtime = s.split_once('=').unwrap().1.to_string(); + } + s if s.starts_with('-') => { eprintln!( - "warning: package arguments are not currently supported, ignoring '{arg}'" + "warning: flag '{s}' is not supported by CodSpeed Go runner, ignoring" ); - // Consume and ignore all remaining arguments - for remaining_arg in args { - eprintln!( - "warning: package arguments are not currently supported, ignoring '{remaining_arg}'" - ); - } + } + _ => { + // Collect package arguments for filtering + instance.packages = { + let mut packages = vec![arg]; + packages.extend(args); + packages + }; break; } } } - Ok(Self { bench }) + Ok(instance) } } @@ -107,6 +135,8 @@ mod tests { fn test_cli_parse_defaults() { let cli = str_to_iter("go-runner test").unwrap(); assert_eq!(cli.bench, "."); + assert_eq!(cli.benchtime, Cli::default().benchtime); + assert_eq!(cli.packages, Cli::default().packages); } #[test] @@ -119,9 +149,30 @@ mod tests { } #[test] - fn test_cli_parse_ignores_packages() { + fn test_cli_parse_with_benchtime_flag() { + let cli = str_to_iter("go-runner test -benchtime 3s").unwrap(); + assert_eq!(cli.benchtime, "3s".to_string()); + + let cli = str_to_iter("go-runner test -benchtime=10x").unwrap(); + assert_eq!(cli.benchtime, "10x".to_string()); + } + + #[test] + fn test_cli_parse_with_packages() { let cli = str_to_iter("go-runner test package1 package2").unwrap(); assert_eq!(cli.bench, "."); + assert_eq!( + cli.packages, + vec!["package1".to_string(), "package2".to_string()] + ); + } + + #[test] + fn test_cli_parse_combined_flags() { + let cli = str_to_iter("go-runner test -bench=BenchmarkFoo -benchtime 5s ./pkg").unwrap(); + assert_eq!(cli.bench, "BenchmarkFoo"); + assert_eq!(cli.benchtime, "5s".to_string()); + assert_eq!(cli.packages, vec!["./pkg".to_string()]); } #[test] @@ -147,7 +198,11 @@ mod tests { let result = str_to_iter("go-runner test -bench"); assert!(matches!(result, Err(CliExit::MissingArgument))); + let result = str_to_iter("go-runner test -benchtime"); + assert!(matches!(result, Err(CliExit::MissingArgument))); + + // Unknown flags now generate warnings but don't cause errors let result = str_to_iter("go-runner test -unknown"); - assert!(matches!(result, Err(CliExit::UnknownFlag))); + assert!(result.is_ok()); } } diff --git a/go-runner/src/integration_tests.rs b/go-runner/src/integration_tests.rs index 6f6f75e4..30254935 100644 --- a/go-runner/src/integration_tests.rs +++ b/go-runner/src/integration_tests.rs @@ -1,23 +1,10 @@ use itertools::Itertools; use rstest::rstest; use std::path::{Path, PathBuf}; -use std::sync::Mutex; use tempfile::TempDir; use crate::results::walltime_results::WalltimeResults; -fn setup_test_project(project_name: &str) -> anyhow::Result { - let project_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("testdata/projects") - .join(project_name); - println!("Project path: {project_path:?}"); - - let temp_dir = TempDir::new()?; - crate::utils::copy_dir_recursively(&project_path, &temp_dir)?; - - Ok(temp_dir) -} - fn assert_results_snapshots(profile_dir: &Path, project_name: &str) { let glob_pattern = profile_dir.join("results"); if !glob_pattern.exists() { @@ -72,16 +59,19 @@ fn assert_results_snapshots(profile_dir: &Path, project_name: &str) { // Currently not producing results: #[case::fuego("fuego")] #[case::cli_runtime("cli-runtime")] +#[case::example("example")] fn test_build_and_run(#[case] project_name: &str) { - let temp_dir = setup_test_project(project_name).unwrap(); - - // Mutex to prevent concurrent tests from interfering with CODSPEED_PROFILE_FOLDER env var - static ENV_MUTEX: Mutex<()> = Mutex::new(()); - let _env_guard = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner()); + let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("testdata/projects") + .join(project_name); + let temp_dir = TempDir::new().unwrap(); let profile_dir = temp_dir.path().join("profile"); - unsafe { std::env::set_var("CODSPEED_PROFILE_FOLDER", &profile_dir) }; - if let Err(error) = crate::run_benchmarks(temp_dir.path(), ".") { + let cli = crate::cli::Cli { + benchtime: "1x".into(), + ..Default::default() + }; + if let Err(error) = crate::run_benchmarks(&profile_dir, project_dir.as_path(), &cli) { panic!("Benchmarks couldn't run: {error}"); } diff --git a/go-runner/src/lib.rs b/go-runner/src/lib.rs index 718c173a..7d4cb0bc 100644 --- a/go-runner/src/lib.rs +++ b/go-runner/src/lib.rs @@ -1,96 +1,69 @@ use crate::{builder::BenchmarkPackage, prelude::*}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; +use std::{collections::HashMap, path::Path}; -mod builder; +pub mod builder; pub mod cli; pub mod prelude; mod results; -mod runner; +pub mod runner; pub(crate) mod utils; #[cfg(test)] mod integration_tests; /// Builds and runs the specified Go project benchmarks, writing results to the .codspeed folder. -pub fn run_benchmarks(project_dir: &Path, bench: &str) -> anyhow::Result<()> { - let profile_dir = std::env::var("CODSPEED_PROFILE_FOLDER") - .context("CODSPEED_PROFILE_FOLDER env var not set")?; +pub fn run_benchmarks>( + profile_dir: P, + project_dir: &Path, + cli: &crate::cli::Cli, +) -> anyhow::Result<()> { std::fs::remove_dir_all(&profile_dir).ok(); // 1. Build phase - Benchmark and package discovery - let packages = BenchmarkPackage::from_project(project_dir)?; + let packages = BenchmarkPackage::from_project(project_dir, &cli.packages)?; info!("Discovered {} packages", packages.len()); - let mut bench_name_to_path = HashMap::new(); + let total_benchmarks: usize = packages.iter().map(|p| p.benchmarks.len()).sum(); + info!("Total benchmarks discovered: {total_benchmarks}"); + for package in &packages { for benchmark in &package.benchmarks { - bench_name_to_path.insert(benchmark.name.clone(), benchmark.file_path.clone()); + info!("Found {:30} in {:?}", benchmark.name, benchmark.file_path); } } - let total_benchmarks: usize = packages.iter().map(|p| p.benchmarks.len()).sum(); - info!("Total benchmarks discovered: {total_benchmarks}"); - for (name, path) in &bench_name_to_path { - info!("Found {name:30} in {path:?}"); - } - // 2. Generate codspeed runners, build binaries, and execute them for package in &packages { info!("Generating custom runner for package: {}", package.name); - let (_target_dir, runner_path) = builder::templater::run(package)?; + let (_target_dir, runner_path) = builder::templater::run(package, &profile_dir)?; info!("Building binary for package: {}", package.name); let binary_path = builder::build_binary(&runner_path)?; - let args = [ - "-test.bench", - bench, - // Use a single iteration in tests to speed up execution, otherwise use 5 seconds - "-test.benchtime", - if cfg!(test) || std::env::var("CODSPEED_ENV").is_err() { - "1x" - } else { - "5s" - }, - ]; - - info!("Running benchmarks for package: {}", package.name); - runner::run(&binary_path, &args)?; + runner::run( + &binary_path, + &["-test.bench", &cli.bench, "-test.benchtime", &cli.benchtime], + )?; } // 3. Collect the results - collect_walltime_results(bench_name_to_path)?; + collect_walltime_results(profile_dir.as_ref())?; Ok(()) } // TODO: This should be merged with codspeed-rust/codspeed/walltime_results.rs -fn collect_walltime_results(bench_name_to_path: HashMap) -> anyhow::Result<()> { - let profile_dir = std::env::var("CODSPEED_PROFILE_FOLDER") - .context("CODSPEED_PROFILE_FOLDER env var not set")?; - let profile_dir = PathBuf::from(&profile_dir); - let raw_results = results::raw_result::RawResult::parse_folder(&profile_dir)?; +fn collect_walltime_results(profile_dir: &Path) -> anyhow::Result<()> { + let raw_results = results::raw_result::RawResult::parse_folder(profile_dir)?; info!("Parsed {} raw results", raw_results.len()); let mut benchmarks_by_pid: HashMap> = HashMap::new(); for raw in raw_results { - // We only parse the `func Benchmark*` name which is the first part of the URI - let func_name = raw - .benchmark_name - .split("::") - .next() - .unwrap_or(&raw.benchmark_name); - let file_path = bench_name_to_path - .get(func_name) - .map(|p| p.to_string_lossy().to_string()); benchmarks_by_pid .entry(raw.pid) .or_default() - .push(raw.into_walltime_benchmark(file_path)); + .push(raw.into_walltime_benchmark()); } for (pid, walltime_benchmarks) in benchmarks_by_pid { diff --git a/go-runner/src/main.rs b/go-runner/src/main.rs index 11ad4dff..9a0b08cd 100644 --- a/go-runner/src/main.rs +++ b/go-runner/src/main.rs @@ -9,7 +9,8 @@ fn main() -> anyhow::Result<()> { .init(); let cli = Cli::parse(); - codspeed_go_runner::run_benchmarks(Path::new("."), &cli.bench)?; + let profile_dir = std::env::var("CODSPEED_PROFILE_FOLDER").unwrap_or("/tmp".into()); + codspeed_go_runner::run_benchmarks(profile_dir, Path::new("."), &cli)?; Ok(()) } diff --git a/go-runner/src/results/raw_result.rs b/go-runner/src/results/raw_result.rs index de940382..731c7246 100644 --- a/go-runner/src/results/raw_result.rs +++ b/go-runner/src/results/raw_result.rs @@ -7,7 +7,8 @@ use crate::results::walltime_results::WalltimeBenchmark; // WARN: Keep in sync with Golang "testing" fork (benchmark.go) #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RawResult { - pub benchmark_name: String, + pub name: String, + pub uri: String, pub pid: u32, pub codspeed_time_per_round_ns: Vec, @@ -32,12 +33,7 @@ impl RawResult { .collect()) } - pub fn into_walltime_benchmark(self, file_path: Option) -> WalltimeBenchmark { - let name = self.benchmark_name; - - let file = file_path.as_deref().unwrap_or("unknown"); - let uri = format!("{file}::{name}"); - + pub fn into_walltime_benchmark(self) -> WalltimeBenchmark { let times_per_round_ns = self .codspeed_time_per_round_ns .iter() @@ -52,7 +48,13 @@ impl RawResult { .collect() }; - WalltimeBenchmark::from_runtime_data(name, uri, iters_per_round, times_per_round_ns, None) + WalltimeBenchmark::from_runtime_data( + self.name, + self.uri, + iters_per_round, + times_per_round_ns, + None, + ) } } @@ -63,33 +65,16 @@ mod tests { #[test] fn test_raw_result_deserialization() { let json_data = r#"{ - "benchmark_name": "BenchmarkFibonacci20-16", + "name": "BenchmarkFibonacci20-16", + "uri": "pkg/foo/fib_test.go::BenchmarkFibonacci20-16", "pid": 777767, "codspeed_time_per_round_ns": [1000, 2000, 3000] }"#; let result: RawResult = serde_json::from_str(json_data).unwrap(); - assert_eq!(result.benchmark_name, "BenchmarkFibonacci20-16"); + assert_eq!(result.name, "BenchmarkFibonacci20-16"); assert_eq!(result.pid, 777767); assert_eq!(result.codspeed_time_per_round_ns.len(), 3); assert_eq!(result.codspeed_iters_per_round.len(), 0); // Default: 1 per round } - - #[test] - fn test_into_walltime_benchmark_with_file_path() { - let raw_result = RawResult { - benchmark_name: "BenchmarkFibonacci20-16".to_string(), - pid: 777767, - codspeed_time_per_round_ns: vec![1000, 2000, 3000], - codspeed_iters_per_round: vec![], - }; - - // Test with file path - should not panic and create successfully - let _walltime_bench = raw_result - .clone() - .into_walltime_benchmark(Some("pkg/foo/fib_test.go".to_string())); - - // Test without file path (should default to TODO) - should not panic and create successfully - let _walltime_bench_no_path = raw_result.into_walltime_benchmark(None); - } } diff --git a/go-runner/src/runner.rs b/go-runner/src/runner.rs index f14862de..37c142b3 100644 --- a/go-runner/src/runner.rs +++ b/go-runner/src/runner.rs @@ -1,7 +1,7 @@ use crate::prelude::*; use std::{path::Path, process::Command}; -pub fn run>(binary_path: P, run_args: &[&str]) -> anyhow::Result<()> { +fn run_cmd>(binary_path: P, run_args: &[&str]) -> anyhow::Result { let binary_path = binary_path.as_ref(); debug!("Running codspeed benchmark binary: {binary_path:?}"); @@ -15,15 +15,12 @@ pub fn run>(binary_path: P, run_args: &[&str]) -> anyhow::Result< // fib_test.go // ``` let module_root = binary_path.parent().unwrap().parent().unwrap(); + let mut cmd = Command::new(binary_path); + cmd.args(run_args).current_dir(module_root); + Ok(cmd) +} - let output = Command::new(binary_path) - .args(run_args) - .current_dir(module_root) - .stdout(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .output() - .context("Failed to execute benchmark binary")?; - +fn check_success(output: &std::process::Output) -> anyhow::Result { if !output.status.success() { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); @@ -33,6 +30,27 @@ pub fn run>(binary_path: P, run_args: &[&str]) -> anyhow::Result< bail!("Failed to run benchmark. Exit status: {}", output.status); } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +/// Runs the cmd and returns the output. +pub fn run_with_stdout>( + binary_path: P, + run_args: &[&str], +) -> anyhow::Result { + let mut cmd = run_cmd(binary_path, run_args)?; + let output = cmd.output().context("Failed to execute go build command")?; + check_success(&output) +} + +/// Runs the cmd and forwards the output to stdout/stderr. +pub fn run>(binary_path: P, run_args: &[&str]) -> anyhow::Result<()> { + let mut cmd = run_cmd(binary_path, run_args)?; + let output = cmd + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .output() + .context("Failed to execute go build command")?; - Ok(()) + check_success(&output).map(|_| ()) } diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_0.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_0.snap new file mode 100644 index 00000000..8fb1cf8f --- /dev/null +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_0.snap @@ -0,0 +1,27 @@ +--- +source: go-runner/src/integration_tests.rs +expression: content +--- +{ + "creator": { + "name": "codspeed-go", + "version": "0.1.0", + "pid": "[pid]" + }, + "instrument": { + "type": "walltime" + }, + "benchmarks": [ + { + "name": "BenchmarkExample", + "uri": "example/very/nested/module/example_test.go::BenchmarkExample", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + } + ] +} diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_1.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_1.snap new file mode 100644 index 00000000..fb6cacd1 --- /dev/null +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example_1.snap @@ -0,0 +1,203 @@ +--- +source: go-runner/src/integration_tests.rs +expression: content +--- +{ + "creator": { + "name": "codspeed-go", + "version": "0.1.0", + "pid": "[pid]" + }, + "instrument": { + "type": "walltime" + }, + "benchmarks": [ + { + "name": "BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", + "uri": "example/fib_test.go::BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkFibonacci20_Loop", + "uri": "example/fib_test.go::BenchmarkFibonacci20_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkFibonacci20_bN", + "uri": "example/fib_test.go::BenchmarkFibonacci20_bN", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100ns", + "uri": "example/sleep_test.go::BenchmarkSleep100ns", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100ns_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep100ns_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100us", + "uri": "example/sleep_test.go::BenchmarkSleep100us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep100us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10ms", + "uri": "example/sleep_test.go::BenchmarkSleep10ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep10ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10us", + "uri": "example/sleep_test.go::BenchmarkSleep10us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep10us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1ms", + "uri": "example/sleep_test.go::BenchmarkSleep1ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep1ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1us", + "uri": "example/sleep_test.go::BenchmarkSleep1us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep1us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep50ms", + "uri": "example/sleep_test.go::BenchmarkSleep50ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep50ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep50ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + } + ] +} diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap index 1d486fc1..aeb6aa35 100644 --- a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@zerolog_1.snap @@ -146,7 +146,7 @@ expression: content }, { "name": "BenchmarkAppendString::EncodingFirst", - "uri": "internal/json/string_test.go::BenchmarkAppendString::EncodingFirst", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::EncodingFirst", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -157,7 +157,7 @@ expression: content }, { "name": "BenchmarkAppendString::EncodingLast", - "uri": "internal/json/string_test.go::BenchmarkAppendString::EncodingLast", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::EncodingLast", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -168,7 +168,7 @@ expression: content }, { "name": "BenchmarkAppendString::EncodingMiddle", - "uri": "internal/json/string_test.go::BenchmarkAppendString::EncodingMiddle", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::EncodingMiddle", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -179,7 +179,7 @@ expression: content }, { "name": "BenchmarkAppendString::MultiBytesFirst", - "uri": "internal/json/string_test.go::BenchmarkAppendString::MultiBytesFirst", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::MultiBytesFirst", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -190,7 +190,7 @@ expression: content }, { "name": "BenchmarkAppendString::MultiBytesLast", - "uri": "internal/json/string_test.go::BenchmarkAppendString::MultiBytesLast", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::MultiBytesLast", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -201,7 +201,7 @@ expression: content }, { "name": "BenchmarkAppendString::MultiBytesMiddle", - "uri": "internal/json/string_test.go::BenchmarkAppendString::MultiBytesMiddle", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::MultiBytesMiddle", "config": { "warmup_time_ns": null, "min_round_time_ns": null, @@ -212,7 +212,7 @@ expression: content }, { "name": "BenchmarkAppendString::NoEncoding", - "uri": "internal/json/string_test.go::BenchmarkAppendString::NoEncoding", + "uri": "internal/cbor/string_test.go::BenchmarkAppendString::NoEncoding", "config": { "warmup_time_ns": null, "min_round_time_ns": null, diff --git a/go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@example_0.snap b/go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@example_0.snap new file mode 100644 index 00000000..fb6cacd1 --- /dev/null +++ b/go-runner/src/snapshots/go_runner__integration_tests__assert_results_snapshots@example_0.snap @@ -0,0 +1,203 @@ +--- +source: go-runner/src/integration_tests.rs +expression: content +--- +{ + "creator": { + "name": "codspeed-go", + "version": "0.1.0", + "pid": "[pid]" + }, + "instrument": { + "type": "walltime" + }, + "benchmarks": [ + { + "name": "BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", + "uri": "example/fib_test.go::BenchmarkFibonacci10::fibonacci(10)::fibonacci(10)", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkFibonacci20_Loop", + "uri": "example/fib_test.go::BenchmarkFibonacci20_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkFibonacci20_bN", + "uri": "example/fib_test.go::BenchmarkFibonacci20_bN", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100ns", + "uri": "example/sleep_test.go::BenchmarkSleep100ns", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100ns_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep100ns_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100us", + "uri": "example/sleep_test.go::BenchmarkSleep100us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep100us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep100us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10ms", + "uri": "example/sleep_test.go::BenchmarkSleep10ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep10ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10us", + "uri": "example/sleep_test.go::BenchmarkSleep10us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep10us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep10us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1ms", + "uri": "example/sleep_test.go::BenchmarkSleep1ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep1ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1us", + "uri": "example/sleep_test.go::BenchmarkSleep1us", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep1us_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep1us_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep50ms", + "uri": "example/sleep_test.go::BenchmarkSleep50ms", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + }, + { + "name": "BenchmarkSleep50ms_Loop", + "uri": "example/sleep_test.go::BenchmarkSleep50ms_Loop", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + } + ] +} diff --git a/go-runner/src/utils.rs b/go-runner/src/utils.rs index 4ee817c5..2a5be482 100644 --- a/go-runner/src/utils.rs +++ b/go-runner/src/utils.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use std::{fs, io}; pub fn copy_dir_recursively(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { @@ -18,3 +18,34 @@ pub fn copy_dir_recursively(src: impl AsRef, dst: impl AsRef) -> io: } Ok(()) } + +fn get_parent_git_repo_path(abs_path: &Path) -> io::Result { + if abs_path.join(".git").exists() { + Ok(abs_path.to_path_buf()) + } else { + get_parent_git_repo_path( + abs_path + .parent() + .ok_or(io::Error::from(io::ErrorKind::NotFound))?, + ) + } +} + +pub fn get_git_relative_path

(abs_path: P) -> PathBuf +where + P: AsRef, +{ + if let Ok(canonicalized_abs_path) = abs_path.as_ref().canonicalize() { + // `repo_path` is still canonicalized as it is a subpath of `canonicalized_abs_path` + if let Ok(repo_path) = get_parent_git_repo_path(&canonicalized_abs_path) { + canonicalized_abs_path + .strip_prefix(repo_path) + .expect("Repository path is malformed.") + .to_path_buf() + } else { + canonicalized_abs_path + } + } else { + abs_path.as_ref().to_path_buf() + } +} diff --git a/go-runner/testdata/projects/example b/go-runner/testdata/projects/example new file mode 120000 index 00000000..9f480167 --- /dev/null +++ b/go-runner/testdata/projects/example @@ -0,0 +1 @@ +../../../example \ No newline at end of file diff --git a/go-runner/tests/error_file.in/error_test.go b/go-runner/tests/error_file.in/error_test.go new file mode 100644 index 00000000..273de8f2 --- /dev/null +++ b/go-runner/tests/error_file.in/error_test.go @@ -0,0 +1,7 @@ +package error_file + +import "testing" + +func BenchmarkErrorFile(b *testing.B) { + b.Error("this_should_be_in_stdout") +} diff --git a/go-runner/tests/error_file.in/go.mod b/go-runner/tests/error_file.in/go.mod new file mode 100644 index 00000000..995513a5 --- /dev/null +++ b/go-runner/tests/error_file.in/go.mod @@ -0,0 +1,3 @@ +module example + +go 1.24.5 diff --git a/go-runner/tests/error_file.rs b/go-runner/tests/error_file.rs new file mode 100644 index 00000000..364788e6 --- /dev/null +++ b/go-runner/tests/error_file.rs @@ -0,0 +1,17 @@ +pub mod utils; + +use utils::run_with_args; + +#[test] +pub fn test_error_has_test_filename() { + let stdout = run_with_args( + "tests/error_file.in", + &["-test.bench", "BenchmarkErrorFile", "-test.benchtime", "1x"], + ) + .unwrap(); + + eprintln!("Error output: {stdout}"); + assert!(stdout.contains("this_should_be_in_stdout")); + assert!(stdout.contains("error_test.go")); + assert!(!stdout.contains("error_codspeed.go")); +} diff --git a/go-runner/tests/pkg_arg.in/bar/bar_test.go b/go-runner/tests/pkg_arg.in/bar/bar_test.go new file mode 100644 index 00000000..baece0df --- /dev/null +++ b/go-runner/tests/pkg_arg.in/bar/bar_test.go @@ -0,0 +1,19 @@ +package bar + +import "testing" + +func BenchmarkBar1(b *testing.B) { + b.Log("bar_bench_1") + for i := 0; i < b.N; i++ { + // Some work + _ = i * 4 + } +} + +func BenchmarkBar2(b *testing.B) { + b.Log("bar_bench_2") + for i := 0; i < b.N; i++ { + // Some work + _ = i * 5 + } +} diff --git a/go-runner/tests/pkg_arg.in/foo/foo_test.go b/go-runner/tests/pkg_arg.in/foo/foo_test.go new file mode 100644 index 00000000..542c76a2 --- /dev/null +++ b/go-runner/tests/pkg_arg.in/foo/foo_test.go @@ -0,0 +1,19 @@ +package foo + +import "testing" + +func BenchmarkFoo1(b *testing.B) { + b.Log("foo_bench_1") + for i := 0; i < b.N; i++ { + // Some work + _ = i * 2 + } +} + +func BenchmarkFoo2(b *testing.B) { + b.Log("foo_bench_2") + for i := 0; i < b.N; i++ { + // Some work + _ = i * 3 + } +} diff --git a/go-runner/tests/pkg_arg.in/go.mod b/go-runner/tests/pkg_arg.in/go.mod new file mode 100644 index 00000000..995513a5 --- /dev/null +++ b/go-runner/tests/pkg_arg.in/go.mod @@ -0,0 +1,3 @@ +module example + +go 1.24.5 diff --git a/go-runner/tests/pkg_arg.rs b/go-runner/tests/pkg_arg.rs new file mode 100644 index 00000000..0f231e59 --- /dev/null +++ b/go-runner/tests/pkg_arg.rs @@ -0,0 +1,54 @@ +use codspeed_go_runner::cli::Cli; +use utils::{run_with_cli, run_with_cli_multi}; + +pub mod utils; + +#[test] +pub fn test_pkg_arg_filters_correctly() { + let cli = Cli { + bench: "BenchmarkBar1".to_string(), + benchtime: "1x".to_string(), + packages: vec!["./bar".to_string()], + }; + let stdout = run_with_cli("tests/pkg_arg.in", &cli).unwrap(); + + // Should contain output from the targeted benchmark + assert!(stdout.contains("bar_bench_1")); + + // Should NOT contain output from other benchmarks + assert!(!stdout.contains("foo_bench_1")); + assert!(!stdout.contains("foo_bench_2")); + assert!(!stdout.contains("bar_bench_2")); +} + +#[test] +pub fn test_pkg_arg_all_packages() { + let cli = Cli { + bench: ".".to_string(), + benchtime: "1x".to_string(), + packages: vec!["./...".to_string()], + }; + let stdout = run_with_cli_multi("tests/pkg_arg.in", &cli).unwrap(); + + // Should contain output from all benchmarks when using ./... + assert!(stdout.contains("foo_bench_1")); + assert!(stdout.contains("foo_bench_2")); + assert!(stdout.contains("bar_bench_1")); + assert!(stdout.contains("bar_bench_2")); +} + +#[test] +pub fn test_pkg_arg_multiple_packages() { + let cli = Cli { + bench: ".".to_string(), + benchtime: "1x".to_string(), + packages: vec!["./foo".to_string(), "./bar".to_string()], + }; + let stdout = run_with_cli_multi("tests/pkg_arg.in", &cli).unwrap(); + + // Should contain output from both foo and bar packages + assert!(stdout.contains("foo_bench_1")); + assert!(stdout.contains("foo_bench_2")); + assert!(stdout.contains("bar_bench_1")); + assert!(stdout.contains("bar_bench_2")); +} diff --git a/go-runner/tests/utils.rs b/go-runner/tests/utils.rs new file mode 100644 index 00000000..4794b96b --- /dev/null +++ b/go-runner/tests/utils.rs @@ -0,0 +1,51 @@ +use codspeed_go_runner::{builder, builder::BenchmarkPackage, cli::Cli, runner}; +use std::path::Path; + +/// Helper function to run a single package with arguments +pub fn run_package_with_args(package: &BenchmarkPackage, args: &[&str]) -> anyhow::Result { + let profile_dir = tempfile::TempDir::new()?; + let (_dir, runner_path) = builder::templater::run(package, profile_dir.as_ref())?; + let binary_path = builder::build_binary(&runner_path)?; + runner::run_with_stdout(&binary_path, args) +} + +/// Helper function to run tests in a directory with specific arguments +pub fn run_with_args>(dir: P, args: &[&str]) -> anyhow::Result { + assert!(dir.as_ref().exists()); + + let packages = BenchmarkPackage::from_project(dir.as_ref(), &["./...".to_string()])?; + assert_eq!(packages.len(), 1); + + run_package_with_args(&packages[0], args) +} + +/// Helper function to run a single package using CLI configuration +pub fn run_with_cli>(dir: P, cli: &Cli) -> anyhow::Result { + assert!(dir.as_ref().exists()); + + let packages = BenchmarkPackage::from_project(dir.as_ref(), &cli.packages)?; + assert_eq!( + packages.len(), + 1, + "Currently only single package is supported" + ); + + let args = ["-test.bench", &cli.bench, "-test.benchtime", &cli.benchtime]; + run_package_with_args(&packages[0], &args) +} + +/// Helper function to run multiple packages using CLI configuration +pub fn run_with_cli_multi>(dir: P, cli: &Cli) -> anyhow::Result { + assert!(dir.as_ref().exists()); + + let packages = BenchmarkPackage::from_project(dir.as_ref(), &cli.packages)?; + + let mut all_stdout = String::new(); + for package in &packages { + let args = ["-test.bench", &cli.bench, "-test.benchtime", &cli.benchtime]; + let stdout = run_package_with_args(package, &args)?; + all_stdout.push_str(&stdout); + } + + Ok(all_stdout) +} diff --git a/testing/testing/benchmark.go b/testing/testing/benchmark.go index 146e8208..be424462 100644 --- a/testing/testing/benchmark.go +++ b/testing/testing/benchmark.go @@ -15,6 +15,7 @@ import ( "math" "os" "path/filepath" + "reflect" "runtime" "slices" "strconv" @@ -296,7 +297,7 @@ var labelsOnce sync.Once // subbenchmarks. b must not have subbenchmarks. func (b *B) run() { labelsOnce.Do(func() { - fmt.Fprintf(b.w, "Running with CodSpeed instrumentation\n") + fmt.Fprintf(b.w, "Running with CodSpeed (mode: walltime)\n") fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS) fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH) @@ -834,12 +835,37 @@ func (s *benchState) processBench(b *B) { // ############################################################################################ // START CODSPEED type RawResults struct { - BenchmarkName string `json:"benchmark_name"` + Name string `json:"name"` + Uri string `json:"uri"` Pid int `json:"pid"` CodspeedTimePerRoundNs []time.Duration `json:"codspeed_time_per_round_ns"` CodspeedItersPerRound []int64 `json:"codspeed_iters_per_round"` } + // Find the filename of the benchmark file + var benchFile string + if b.benchFunc != nil { + pc := reflect.ValueOf(b.benchFunc).Pointer() + fn := runtime.FuncForPC(pc) + if fn == nil { + continue + } + + file, _ := fn.FileLine(pc) + if strings.HasSuffix(file, "_codspeed.go") { + benchFile = file + } + } + + if benchFile == "" { + panic("Could not determine benchmark file name") + } + + relativeBenchFile := getGitRelativePath(benchFile) + if strings.HasSuffix(relativeBenchFile, "_codspeed.go") { + relativeBenchFile = strings.TrimSuffix(relativeBenchFile, "_codspeed.go") + "_test.go" + } + // Build custom bench name with :: separator var nameParts []string current := &b.common @@ -858,20 +884,23 @@ func (s *benchState) processBench(b *B) { } current = current.parent } - customBenchName := strings.Join(nameParts, "::") + benchName = strings.Join(nameParts, "::") + benchUri := fmt.Sprintf("%s::%s", relativeBenchFile, benchName) rawResults := RawResults{ - BenchmarkName: customBenchName, + Name: benchName, + Uri: benchUri, Pid: os.Getpid(), CodspeedTimePerRoundNs: r.CodspeedTimePerRoundNs, CodspeedItersPerRound: r.CodspeedItersPerRound, } - codspeedProfileFolder := os.Getenv("CODSPEED_PROFILE_FOLDER") - if codspeedProfileFolder == "" { - panic("CODSPEED_PROFILE_FOLDER environment variable is not set") + goRunnerMetadata, err := findGoRunnerMetadata() + if err != nil { + panic(fmt.Sprintf("failed to get go runner metadata: %v", err)) } - if err := os.MkdirAll(filepath.Join(codspeedProfileFolder, "raw_results"), 0755); err != nil { + + if err := os.MkdirAll(filepath.Join(goRunnerMetadata.ProfileFolder, "raw_results"), 0755); err != nil { fmt.Fprintf(os.Stderr, "failed to create raw results directory: %v\n", err) continue } @@ -881,7 +910,7 @@ func (s *benchState) processBench(b *B) { fmt.Fprintf(os.Stderr, "failed to generate random filename: %v\n", err) continue } - rawResultsFile := filepath.Join(codspeedProfileFolder, "raw_results", fmt.Sprintf("%s.json", hex.EncodeToString(randomBytes))) + rawResultsFile := filepath.Join(goRunnerMetadata.ProfileFolder, "raw_results", fmt.Sprintf("%s.json", hex.EncodeToString(randomBytes))) file, err := os.Create(rawResultsFile) if err != nil { fmt.Fprintf(os.Stderr, "failed to create raw results file: %v\n", err) @@ -902,7 +931,7 @@ func (s *benchState) processBench(b *B) { defer file.Close() // Send pid and executed benchmark to the runner - b.codspeed.instrument_hooks.SetExecutedBenchmark(uint32(os.Getpid()), customBenchName) + b.codspeed.instrument_hooks.SetExecutedBenchmark(uint32(os.Getpid()), benchUri) // END CODSPEED // ############################################################################################ diff --git a/testing/testing/codspeed.go b/testing/testing/codspeed.go new file mode 100644 index 00000000..40b4162a --- /dev/null +++ b/testing/testing/codspeed.go @@ -0,0 +1,68 @@ +package testing + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +type GoRunnerMetadata struct { + ProfileFolder string `json:"profile_folder"` + RelativePackagePath string `json:"relative_package_path"` +} + +func findGoRunnerMetadata() (*GoRunnerMetadata, error) { + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + + // Search up the directory tree for go-runner.metadata + currentDir := cwd + for { + metadataPath := filepath.Join(currentDir, "go-runner.metadata") + data, err := os.ReadFile(metadataPath) + if err == nil { + var metadata GoRunnerMetadata + err = json.Unmarshal(data, &metadata) + if err != nil { + return nil, err + } + return &metadata, nil + } + + parentDir := filepath.Dir(currentDir) + if parentDir == currentDir { + // Reached the root directory + break + } + currentDir = parentDir + } + + return nil, os.ErrNotExist +} + +func getGitRelativePath(absPath string) string { + canonicalizedAbsPath, err := filepath.EvalSymlinks(absPath) + if err != nil { + panic(fmt.Sprintf("failed to evaluate symlinks for path %s: %v", absPath, err)) + } + + cwd, err := os.Getwd() + if err != nil { + panic(fmt.Sprintf("failed to get current working directory: %v", err)) + } + + cwdRelativePath, err := filepath.Rel(cwd, canonicalizedAbsPath) + if err != nil { + panic(fmt.Sprintf("failed to compute relative path from %s to %s: %v", cwd, canonicalizedAbsPath, err)) + } + + goRunnerMetadata, err := findGoRunnerMetadata() + if err != nil { + panic(fmt.Sprintf("failed to find go-runner metadata: %v", err)) + } + + return filepath.Join(goRunnerMetadata.RelativePackagePath, cwdRelativePath) +} diff --git a/testing/testing/testing.go b/testing/testing/testing.go index 899f9048..4d0a8668 100644 --- a/testing/testing/testing.go +++ b/testing/testing/testing.go @@ -811,6 +811,11 @@ func (c *common) decorate(s string, skip int) string { } else if index := strings.LastIndexAny(file, `/\`); index >= 0 { file = file[index+1:] } + + // Replace _codspeed.go with _test.go for better user experience + if strings.HasSuffix(file, "_codspeed.go") { + file = strings.TrimSuffix(file, "_codspeed.go") + "_test.go" + } } else { file = "???" }