diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..903120cd --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,121 @@ +# Benchmarks + +JMH microbenchmarks for protovalidate-java. +Used locally to quantify performance changes. +Not executed in CI; `./gradlew build` only verifies that benchmark code compiles. + +## Prerequisites + +- JDK 21 +- `buf` CLI (installed automatically by Gradle) +- `jq` and `column` (preinstalled on macOS) + +## Running benchmarks + +Run all benchmarks: + +``` +./gradlew :benchmarks:jmh +``` + +Filter to a subset via `-Pbench` (accepts a regex over method names): + +``` +./gradlew :benchmarks:jmh -Pbench=validateSimple # one method +./gradlew :benchmarks:jmh -Pbench='compile.*' # prefix match +./gradlew :benchmarks:jmh -Pbench='validate.*' # all steady-state +``` + +Results land in `build/results/jmh/results.json`. + +## Comparing before and after a change + +Typical A/B workflow: + +``` +# 1. run baseline on the current tree and save it +./gradlew :benchmarks:jmh -Pbench='compile.*' :benchmarks:jmhSaveBaseline + +# 2. apply your change (edit code, or gh pr checkout ) + +# 3. re-run and diff against the saved baseline +./gradlew :benchmarks:jmh -Pbench='compile.*' :benchmarks:jmhCompare +``` + +Output: + +``` +benchmark metric before after delta +compileValidatorForRepeated time 4696209.43 ns/op 1064942.21 ns/op -77.3% +compileValidatorForRepeated alloc 12950196.95 B/op 3262651.61 B/op -74.8% +``` + +`jmhSaveBaseline` copies the current `results.json` to `results-before.json`. +`jmhCompare` diffs `results-before.json` against `results.json` by default. +Pass explicit paths with `-Pbefore= -Pafter=`. + +## Adding a new benchmark + +Benchmarks live in `src/jmh/java/...` and target proto messages in `src/jmh/proto/...`. + +### 1. Define (or reuse) a proto message + +Edit `src/jmh/proto/bench/v1/bench.proto` to add a message that exercises the code path you want to measure. +`buf generate` runs automatically before `compileJmhJava`, so no separate codegen step is needed. + +### 2. Add a `@Benchmark` method + +Edit `src/jmh/java/build/buf/protovalidate/benchmarks/ValidationBenchmark.java`. +Put one-time state (validator, messages) in `@Setup` and the measured work in the `@Benchmark` method. + +Steady-state (hot-path) pattern: + +```java +@Benchmark +public void validateMyMessage(Blackhole bh) throws ValidationException { + bh.consume(validator.validate(myMessage)); +} +``` + +Cold/compile-path pattern (each iteration builds a fresh validator): + +```java +@Benchmark +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public void compileValidatorForMyMessage(Blackhole bh) throws CompilationException { + Validator v = ValidatorFactory.newBuilder() + .buildWithDescriptors(Collections.singletonList(MyMessage.getDescriptor()), false); + bh.consume(v); +} +``` + +Choose based on what the change you want to measure actually touches. +`EvaluatorBuilder` caches compiled evaluators per descriptor, so after the first `validate()` call, further calls skip compilation. +If your fix is in the compile path (e.g. `RuleCache`, `DescriptorCacheBuilder`), a steady-state benchmark will not show the effect because `@Setup` absorbs it. + +## Configuration + +`build.gradle.kts` holds the JMH plugin config. +Defaults are tuned for fast local iteration (~30s per benchmark): + +- 3 warmup iterations of 2s each +- 5 measurement iterations of 2s each +- 2 forks +- Average-time mode, nanoseconds +- GC profiler on (`gc.alloc.rate.norm` for per-op allocations) + +For higher-confidence numbers (tighter confidence intervals, useful for deltas under ~10%), bump `fork`, `warmup`, and `timeOnIteration` in the `jmh {}` block. +Expect ~5 min per benchmark at `fork=5, warmup=5s, timeOnIteration=5s`. + +## Metrics + +Each benchmark emits: + +- **Primary:** average time per `@Benchmark` invocation (`ns/op` by default). +- **Secondary (GC profiler):** + - `gc.alloc.rate.norm` - bytes allocated per op; deterministic, used by `jmhCompare`. + - `gc.alloc.rate` - allocation rate in MB/sec; varies with CPU. + - `gc.count` / `gc.time` - GC activity during the run. + +For allocation flame graphs, uncomment the `async` profiler line in `build.gradle.kts`. +Requires `async-profiler` installed locally. diff --git a/benchmarks/buf.gen.yaml b/benchmarks/buf.gen.yaml new file mode 100644 index 00000000..4434781f --- /dev/null +++ b/benchmarks/buf.gen.yaml @@ -0,0 +1,6 @@ +version: v2 +plugins: + - remote: buf.build/protocolbuffers/java:$protocJavaPluginVersion + out: build/generated/sources/bufgen +inputs: + - directory: src/jmh/proto diff --git a/benchmarks/buf.lock b/benchmarks/buf.lock new file mode 100644 index 00000000..709ae023 --- /dev/null +++ b/benchmarks/buf.lock @@ -0,0 +1,6 @@ +# Generated by buf. DO NOT EDIT. +version: v2 +deps: + - name: buf.build/bufbuild/protovalidate + commit: 50325440f8f24053b047484a6bf60b76 + digest: b5:74cb6f5c0853c3c10aafc701614194bbd63326bdb8ef4068214454b8894b03ba4113e04b3a33a8321cdf05336e37db4dc14a5e2495db8462566914f36086ba31 diff --git a/benchmarks/buf.yaml b/benchmarks/buf.yaml new file mode 100644 index 00000000..56bbef24 --- /dev/null +++ b/benchmarks/buf.yaml @@ -0,0 +1,5 @@ +version: v2 +modules: + - path: src/jmh/proto +deps: + - buf.build/bufbuild/protovalidate diff --git a/benchmarks/build.gradle.kts b/benchmarks/build.gradle.kts new file mode 100644 index 00000000..e2e35e46 --- /dev/null +++ b/benchmarks/build.gradle.kts @@ -0,0 +1,137 @@ +plugins { + java + alias(libs.plugins.jmh) + alias(libs.plugins.osdetector) +} + +// JMH can use modern bytecode; benchmarks aren't shipped. +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + +val buf: Configuration by configurations.creating + +tasks.register("configureBuf") { + description = "Installs the Buf CLI." + File(buf.asPath).setExecutable(true) +} + +tasks.register("filterBufGenYaml") { + from(files("buf.gen.yaml")) + includeEmptyDirs = false + into(layout.buildDirectory.dir("buf-gen-templates")) + expand("protocJavaPluginVersion" to "v${libs.versions.protobuf.get().substringAfter('.')}") + filteringCharset = "UTF-8" +} + +tasks.register("generateBenchmarkSources") { + dependsOn("configureBuf", "filterBufGenYaml") + description = "Generates Java sources for benchmark protos via buf generate." + val template = layout.buildDirectory.file("buf-gen-templates/buf.gen.yaml") + inputs.files(buf) + inputs.dir("src/jmh/proto") + inputs.file("buf.yaml") + inputs.file(template) + outputs.dir(layout.buildDirectory.dir("generated/sources/bufgen")) + commandLine(buf.asPath, "generate", "--template", template.get().asFile.absolutePath) +} + +sourceSets { + named("jmh") { + java { + srcDir(layout.buildDirectory.dir("generated/sources/bufgen")) + } + } +} + +tasks.matching { it.name == "compileJmhJava" }.configureEach { + dependsOn("generateBenchmarkSources") +} + +// Ensure `./gradlew build` (and `make build`) compiles the JMH sources so CI +// catches breakages in benchmark code. Execution remains gated behind the +// explicit `:benchmarks:jmh` task. +tasks.named("build") { + dependsOn("compileJmhJava") +} + +dependencies { + jmhImplementation(project(":")) + jmhImplementation(libs.protobuf.java) + buf("build.buf:buf:${libs.versions.buf.get()}:${osdetector.classifier}@exe") +} + +// Benchmarks produce fresh timing data each run; disable Gradle's up-to-date +// check so the task always executes (otherwise -Pbench changes are ignored). +tasks.named("jmh") { + outputs.upToDateWhen { false } +} + +jmh { + // Defaults tuned for fast local A/B runs (~90s total). + // For higher-confidence numbers bump iteration time and fork count. + warmupIterations.set(3) + warmup.set("2s") + iterations.set(5) + timeOnIteration.set("2s") + fork.set(2) + timeUnit.set("ns") + benchmarkMode.set(listOf("avgt")) + resultFormat.set("JSON") + // GC profiler reports bytes allocated per op (gc.alloc.rate.norm), which + // jmhCompare can diff alongside timing. ~5-10% overhead on timings. + profilers.set(listOf("gc")) + // For allocation flame graphs (requires async-profiler installed locally): + // profilers.set(listOf("async:event=alloc;output=flamegraph;dir=build/reports/jmh/async")) + + // Filter to a subset of benchmarks via `-Pbench=`. Example: + // ./gradlew :benchmarks:jmh -Pbench=validateSimple + // ./gradlew :benchmarks:jmh -Pbench='compile.*' + project.findProperty("bench")?.toString()?.let { + includes.set(listOf(it)) + } +} + +val jmhResults = layout.buildDirectory.file("results/jmh/results.json") +val jmhBaseline = layout.buildDirectory.file("results/jmh/results-before.json") + +// Saves the latest JMH results.json as the baseline for jmhCompare. +// +// Usage: +// ./gradlew :benchmarks:jmh :benchmarks:jmhSaveBaseline +// # apply change... +// ./gradlew :benchmarks:jmh :benchmarks:jmhCompare +tasks.register("jmhSaveBaseline") { + description = "Copies the latest JMH results.json to results-before.json as the baseline." + from(jmhResults) + into(jmhResults.get().asFile.parentFile) + rename { "results-before.json" } + mustRunAfter("jmh") +} + +// Diffs two JMH results.json files as a concise benchstat-style table. +// Defaults to comparing results-before.json (written by jmhSaveBaseline) +// against the latest results.json. +// +// Override paths: +// ./gradlew :benchmarks:jmhCompare -Pbefore=a.json -Pafter=b.json +tasks.register("jmhCompare") { + description = "Diffs two JMH result JSON files as a concise table." + val before = + project.findProperty("before")?.toString() + ?: jmhBaseline.get().asFile.absolutePath + val after = + project.findProperty("after")?.toString() + ?: jmhResults.get().asFile.absolutePath + val jqScript = file("jmh-compare.jq").absolutePath + commandLine( + "bash", + "-c", + "jq --slurp --raw-output --from-file \"\$1\" \"\$2\" \"\$3\" | column -t -s \$'\\t'", + "jmh-compare", // $0 + jqScript, // $1 + before, // $2 + after, // $3 + ) +} diff --git a/benchmarks/jmh-compare.jq b/benchmarks/jmh-compare.jq new file mode 100644 index 00000000..4d015a36 --- /dev/null +++ b/benchmarks/jmh-compare.jq @@ -0,0 +1,24 @@ +def pct(a; b): + if a == null or b == null or b == 0 then "~" + else (((a - b) / b * 100) * 10 | round / 10) as $d + | if $d > 0 then "+\($d)%" elif $d == 0 then "~" else "\($d)%" end + end; +def num(x): + if x == null then "-" + else (x * 100 | round / 100 | tostring) + end; + +def extract: map({ + key: (.benchmark | split(".") | last), + time: .primaryMetric.score, + time_unit: .primaryMetric.scoreUnit, + alloc: (.secondaryMetrics["·gc.alloc.rate.norm"].score // null) +}); + +(.[0] | extract) as $b +| (.[1] | extract) as $a +| (["benchmark", "metric", "before", "after", "delta"] | @tsv), + ($b[] | . as $bi + | ($a[] | select(.key == $bi.key)) as $ai + | ([$bi.key, "time", "\(num($bi.time)) \($bi.time_unit)", "\(num($ai.time)) \($ai.time_unit)", pct($ai.time; $bi.time)] | @tsv), + ([$bi.key, "alloc", "\(num($bi.alloc)) B/op", "\(num($ai.alloc)) B/op", pct($ai.alloc; $bi.alloc)] | @tsv)) diff --git a/benchmarks/src/jmh/java/build/buf/protovalidate/benchmarks/ValidationBenchmark.java b/benchmarks/src/jmh/java/build/buf/protovalidate/benchmarks/ValidationBenchmark.java new file mode 100644 index 00000000..93f0d314 --- /dev/null +++ b/benchmarks/src/jmh/java/build/buf/protovalidate/benchmarks/ValidationBenchmark.java @@ -0,0 +1,101 @@ +// Copyright 2023-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build.buf.protovalidate.benchmarks; + +import build.buf.protovalidate.Validator; +import build.buf.protovalidate.ValidatorFactory; +import build.buf.protovalidate.benchmarks.gen.ManyUnruledFieldsMessage; +import build.buf.protovalidate.benchmarks.gen.RepeatedRuleMessage; +import build.buf.protovalidate.benchmarks.gen.SimpleStringMessage; +import build.buf.protovalidate.exceptions.CompilationException; +import build.buf.protovalidate.exceptions.ValidationException; +import com.google.protobuf.Descriptors.Descriptor; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +public class ValidationBenchmark { + + private Validator validator; + private SimpleStringMessage simple; + private ManyUnruledFieldsMessage manyUnruled; + + // Descriptor captured once; cheap to reference during benchmark. + private static final Descriptor REPEATED_RULE_DESC = RepeatedRuleMessage.getDescriptor(); + + @Setup + public void setup() throws ValidationException { + validator = ValidatorFactory.newBuilder().build(); + + simple = SimpleStringMessage.newBuilder().setEmail("alice@example.com").build(); + + manyUnruled = + ManyUnruledFieldsMessage.newBuilder() + .setNonEmpty("x") + .setF1("v1") + .setF2("v2") + .setF3("v3") + .setF4("v4") + .setF5("v5") + .setF6("v6") + .setF7("v7") + .setF8("v8") + .setF9("v9") + .build(); + + // Warm evaluator cache for steady-state benchmarks. + validator.validate(simple); + validator.validate(manyUnruled); + } + + // Steady-state validate() benchmarks. These exercise the hot path after the + // evaluator cache is warm. PR #451 does not affect this path. + + @Benchmark + public void validateSimple(Blackhole bh) throws ValidationException { + bh.consume(validator.validate(simple)); + } + + @Benchmark + public void validateManyUnruled(Blackhole bh) throws ValidationException { + bh.consume(validator.validate(manyUnruled)); + } + + // Compile-path benchmark. Measures building a fresh validator and warming + // its RuleCache for RepeatedRuleMessage (20 string fields, all min_len). + // PR #451 affects exactly this path: without the fix, the AST is rebuilt + // for every field; with the fix, fields 2..20 hit the cache. + // + // Time is dominated by Cel environment construction in newCel(); the #451 + // signal is the delta on top of that baseline. + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void compileValidatorForRepeated(Blackhole bh) throws CompilationException { + Validator v = + ValidatorFactory.newBuilder() + .buildWithDescriptors(Collections.singletonList(REPEATED_RULE_DESC), false); + bh.consume(v); + } +} diff --git a/benchmarks/src/jmh/proto/bench/v1/bench.proto b/benchmarks/src/jmh/proto/bench/v1/bench.proto new file mode 100644 index 00000000..a1e47c2e --- /dev/null +++ b/benchmarks/src/jmh/proto/bench/v1/bench.proto @@ -0,0 +1,71 @@ +// Copyright 2023-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package bench.v1; + +option java_multiple_files = true; +option java_package = "build.buf.protovalidate.benchmarks.gen"; + +import "buf/validate/validate.proto"; + +// Simple single-field message. Baseline for validate() overhead on a small +// message with one cheap rule. +message SimpleStringMessage { + string email = 1 [(buf.validate.field).string.email = true]; +} + +// One ruled field and many fields without rules. Targets the tautology skip +// in PR #454: before the fix, no-rule fields still ran a FieldEvaluator on +// every validate() call. +message ManyUnruledFieldsMessage { + string non_empty = 1 [(buf.validate.field).string.min_len = 1]; + string f1 = 2; + string f2 = 3; + string f3 = 4; + string f4 = 5; + string f5 = 6; + string f6 = 7; + string f7 = 8; + string f8 = 9; + string f9 = 10; +} + +// Twenty fields that share the same rule shape (min_len on string). Targets +// the AST cache fix in PR #451. The cache key lookup mismatch made +// compileRule rebuild the AST for every field during evaluator construction. +// With many same-shape fields, the saved work is N-1 AST builds. +message RepeatedRuleMessage { + string f01 = 1 [(buf.validate.field).string.min_len = 1]; + string f02 = 2 [(buf.validate.field).string.min_len = 1]; + string f03 = 3 [(buf.validate.field).string.min_len = 1]; + string f04 = 4 [(buf.validate.field).string.min_len = 1]; + string f05 = 5 [(buf.validate.field).string.min_len = 1]; + string f06 = 6 [(buf.validate.field).string.min_len = 1]; + string f07 = 7 [(buf.validate.field).string.min_len = 1]; + string f08 = 8 [(buf.validate.field).string.min_len = 1]; + string f09 = 9 [(buf.validate.field).string.min_len = 1]; + string f10 = 10 [(buf.validate.field).string.min_len = 1]; + string f11 = 11 [(buf.validate.field).string.min_len = 1]; + string f12 = 12 [(buf.validate.field).string.min_len = 1]; + string f13 = 13 [(buf.validate.field).string.min_len = 1]; + string f14 = 14 [(buf.validate.field).string.min_len = 1]; + string f15 = 15 [(buf.validate.field).string.min_len = 1]; + string f16 = 16 [(buf.validate.field).string.min_len = 1]; + string f17 = 17 [(buf.validate.field).string.min_len = 1]; + string f18 = 18 [(buf.validate.field).string.min_len = 1]; + string f19 = 19 [(buf.validate.field).string.min_len = 1]; + string f20 = 20 [(buf.validate.field).string.min_len = 1]; +} diff --git a/build.gradle.kts b/build.gradle.kts index 03380f7d..eb07a152 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -240,6 +240,7 @@ tasks.register("generate") { "generateSources", "licenseHeader", ":conformance:generateConformance", + ":benchmarks:generateBenchmarkSources", ) } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 24a77079..ff7c394e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,5 +23,6 @@ spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version = [plugins] errorprone = { id = "net.ltgt.errorprone", version = "5.1.0" } +jmh = { id = "me.champeau.jmh", version = "0.7.2" } maven = { id = "com.vanniktech.maven.publish.base", version.ref = "maven-publish" } osdetector = { id = "com.google.osdetector", version = "1.7.3" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 771d1685..271e6cf6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,2 +1,3 @@ rootProject.name = "protovalidate" include("conformance") +include("benchmarks")