diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java index 91f3c5e8101..7236e8c823f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecorator.java @@ -4,6 +4,7 @@ public interface TestDecorator { String TEST_TYPE = "test"; + String TEST_TYPE_BENCHMARK = "benchmark"; AgentSpan afterStart(final AgentSpan span); diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle new file mode 100644 index 00000000000..a5256396007 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'dd-trace-java.instrumentation.testing-framework-tests' +} + +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + group = 'org.openjdk.jmh' + module = 'jmh-core' + versions = '[1.0,)' + } +} + +dependencies { + compileOnly group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.0' + + testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' + testImplementation project(':dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures') + + // JMH annotation processor generates META-INF/BenchmarkList from @Benchmark annotations + testAnnotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37' +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java new file mode 100644 index 00000000000..9b94cb69d87 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/DDOutputFormat.java @@ -0,0 +1,214 @@ +package datadog.trace.instrumentation.jmh; + +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ERROR; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_FORKS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_ITERATIONS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MAX; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MIN; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_MODE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P50; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P90; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P95; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_P99; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_SAMPLE_COUNT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_THREADS; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_TIME_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_UNIT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_VALUE; +import static datadog.trace.bootstrap.instrumentation.api.Tags.BENCHMARK_WARMUP_ITERATIONS; + +import datadog.trace.api.civisibility.InstrumentationBridge; +import datadog.trace.api.civisibility.config.TestSourceData; +import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.Collections; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.results.BenchmarkResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.format.OutputFormat; +import org.openjdk.jmh.util.Statistics; + +/** + * Wraps a JMH {@link OutputFormat} to emit CI Visibility spans for each benchmark method. + * + *

Hooks only fire once per benchmark method (after all forks and iterations complete), so there + * is zero overhead on the benchmark hot path. + */ +public class DDOutputFormat implements OutputFormat { + + private final OutputFormat delegate; + private final TestEventsHandler handler; + private final String frameworkVersion; + + // Keys used as suite/test descriptors in the handler — just the full benchmark name strings. + // We keep suite and test keys separate so the handler can manage their lifetimes independently. + private volatile String currentSuiteKey; + private volatile String currentTestKey; + + public DDOutputFormat(OutputFormat delegate, String frameworkVersion) { + this.delegate = delegate; + this.frameworkVersion = frameworkVersion; + this.handler = + InstrumentationBridge.createTestEventsHandler( + JmhUtils.FRAMEWORK_NAME, null, null, Collections.emptyList()); + } + + @Override + public void startBenchmark(BenchmarkParams benchParams) { + delegate.startBenchmark(benchParams); + + String fullName = benchParams.getBenchmark(); + String[] parts = JmhUtils.splitBenchmarkName(fullName); + String suiteName = parts[0]; + String testName = parts[1]; + String testParameters = JmhUtils.testParameters(fullName); + + currentSuiteKey = suiteName + "#" + fullName; + currentTestKey = fullName; + + handler.onTestSuiteStart( + currentSuiteKey, + suiteName, + JmhUtils.FRAMEWORK_NAME, + frameworkVersion, + null, + Collections.emptyList(), + false, + TestFrameworkInstrumentation.JMH, + null); + + handler.onTestStart( + currentSuiteKey, + currentTestKey, + testName, + JmhUtils.FRAMEWORK_NAME, + frameworkVersion, + testParameters, + Collections.emptyList(), + TestSourceData.UNKNOWN, + null, + null); + } + + @Override + public void endBenchmark(BenchmarkResult result) { + String suiteKey = currentSuiteKey; + String testKey = currentTestKey; + if (suiteKey == null || testKey == null) { + delegate.endBenchmark(result); + return; + } + + try { + tagBenchmarkMetrics(result); + handler.onTestFinish(testKey, null, null); + handler.onTestSuiteFinish(suiteKey, null); + } finally { + delegate.endBenchmark(result); + } + } + + private void tagBenchmarkMetrics(BenchmarkResult result) { + AgentSpan span = AgentTracer.activeSpan(); + if (span == null) { + return; + } + + BenchmarkParams params = result.getParams(); + span.setTag(BENCHMARK_MODE, params.getMode().shortLabel()); + span.setTag(BENCHMARK_ITERATIONS, params.getMeasurement().getCount()); + span.setTag(BENCHMARK_WARMUP_ITERATIONS, params.getWarmup().getCount()); + span.setTag(BENCHMARK_FORKS, params.getForks()); + span.setTag(BENCHMARK_THREADS, params.getThreads()); + span.setTag(BENCHMARK_TIME_UNIT, params.getTimeUnit().name()); + + Result primary = result.getPrimaryResult(); + span.setMetric(BENCHMARK_VALUE, primary.getScore()); + span.setTag(BENCHMARK_UNIT, primary.getScoreUnit()); + + double error = primary.getScoreError(); + if (!Double.isNaN(error)) { + span.setMetric(BENCHMARK_ERROR, error); + } + + Statistics stats = primary.getStatistics(); + if (stats.getN() > 1) { + span.setMetric(BENCHMARK_P50, stats.getPercentile(50)); + span.setMetric(BENCHMARK_P90, stats.getPercentile(90)); + span.setMetric(BENCHMARK_P95, stats.getPercentile(95)); + span.setMetric(BENCHMARK_P99, stats.getPercentile(99)); + span.setMetric(BENCHMARK_MIN, stats.getMin()); + span.setMetric(BENCHMARK_MAX, stats.getMax()); + span.setMetric(BENCHMARK_SAMPLE_COUNT, stats.getN()); + } + } + + // ---- Delegation-only methods ---- + + @Override + public void iteration( + BenchmarkParams benchParams, org.openjdk.jmh.infra.IterationParams params, int iteration) { + delegate.iteration(benchParams, params, iteration); + } + + @Override + public void iterationResult( + BenchmarkParams benchParams, + org.openjdk.jmh.infra.IterationParams params, + int iteration, + org.openjdk.jmh.results.IterationResult data) { + delegate.iterationResult(benchParams, params, iteration, data); + } + + @Override + public void startRun() { + delegate.startRun(); + } + + @Override + public void endRun(java.util.Collection result) { + try { + handler.close(); + } finally { + delegate.endRun(result); + } + } + + @Override + public void print(String s) { + delegate.print(s); + } + + @Override + public void println(String s) { + delegate.println(s); + } + + @Override + public void verbosePrintln(String s) { + delegate.verbosePrintln(s); + } + + @Override + public void write(int b) { + delegate.write(b); + } + + @Override + public void write(byte[] b) throws java.io.IOException { + delegate.write(b); + } + + @Override + public void flush() { + delegate.flush(); + } + + @Override + public void close() { + delegate.close(); + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java new file mode 100644 index 00000000000..5bfb0013ff1 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhInstrumentation.java @@ -0,0 +1,60 @@ +package datadog.trace.instrumentation.jmh; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import net.bytebuddy.asm.Advice; +import org.openjdk.jmh.runner.format.OutputFormat; + +@AutoService(InstrumenterModule.class) +public class JmhInstrumentation extends InstrumenterModule.CiVisibility + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public JmhInstrumentation() { + super("ci-visibility", "jmh"); + } + + @Override + public String instrumentedType() { + // Instrument BaseRunner (where the 'out' field is declared) so that the final-field write + // in RunnerConstructorAdvice is legal: JDK 17+ rejects writing a final field declared in a + // superclass from advice injected into the subclass (Runner). + return "org.openjdk.jmh.runner.BaseRunner"; + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".JmhUtils", packageName + ".DDOutputFormat", + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isConstructor() + .and(takesArgument(0, named("org.openjdk.jmh.runner.options.Options"))) + .and(takesArgument(1, named("org.openjdk.jmh.runner.format.OutputFormat"))), + JmhInstrumentation.class.getName() + "$RunnerConstructorAdvice"); + } + + public static class RunnerConstructorAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void onExit( + @Advice.FieldValue(value = "out", readOnly = false) OutputFormat out) { + if (out instanceof DDOutputFormat) { + return; + } + String version = null; + try { + version = org.openjdk.jmh.Main.class.getPackage().getImplementationVersion(); + } catch (Throwable ignored) { + } + out = new DDOutputFormat(out, version); + } + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java new file mode 100644 index 00000000000..b5bfea0e2ef --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/main/java/datadog/trace/instrumentation/jmh/JmhUtils.java @@ -0,0 +1,52 @@ +package datadog.trace.instrumentation.jmh; + +import javax.annotation.Nullable; + +public final class JmhUtils { + + static final String FRAMEWORK_NAME = "jmh"; + + /** + * Splits a JMH benchmark name into suite (class) and method parts. + * + *

JMH names have the form {@code "com.example.MyBenchmark.myMethod"} or, when {@code @Param} + * combinations are present, {@code "com.example.MyBenchmark.myMethod:size=1000,threads=4"}. + */ + public static String[] splitBenchmarkName(String fullName) { + // Strip any @Param suffix before splitting on the class/method boundary + int colonIdx = fullName.indexOf(':'); + String baseName = colonIdx >= 0 ? fullName.substring(0, colonIdx) : fullName; + + int lastDot = baseName.lastIndexOf('.'); + if (lastDot < 0) { + return new String[] {"", fullName}; + } + return new String[] {baseName.substring(0, lastDot), baseName.substring(lastDot + 1)}; + } + + /** + * Returns the {@code test.parameters} JSON string for a parameterized benchmark, or {@code null} + * for an unparameterized one. + * + *

Follows the same convention as JUnit 5 parameterized tests: {@code + * {"metadata":{"test_name":""}}}. + */ + @Nullable + public static String testParameters(String fullName) { + int colonIdx = fullName.indexOf(':'); + if (colonIdx < 0) { + return null; + } + // fullName after last dot includes the param suffix, e.g. "myMethod:size=1000" + int lastDot = fullName.lastIndexOf('.', colonIdx); + String displayName = lastDot >= 0 ? fullName.substring(lastDot + 1) : fullName; + return "{\"metadata\":{\"test_name\":\"" + escapeJson(displayName) + "\"}}"; + } + + /** Minimal JSON string escaping for benchmark names (no unicode escaping needed). */ + private static String escapeJson(String s) { + return s.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private JmhUtils() {} +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy new file mode 100644 index 00000000000..4b4e6a7ee5f --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/groovy/JmhInstrumentationTest.groovy @@ -0,0 +1,51 @@ +import datadog.trace.api.DisableTestTrace +import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark +import datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark +import org.openjdk.jmh.runner.Runner +import org.openjdk.jmh.runner.options.OptionsBuilder + +@DisableTestTrace(reason = "avoid self-tracing") +class JmhInstrumentationTest extends CiVisibilityInstrumentationTest { + + // Benchmark numeric metrics vary each run — they are verified structurally in the smoke test + static final List BENCHMARK_METRIC_TAGS = [ + "content.metrics.['benchmark.value']", + "content.metrics.['benchmark.error']", + "content.metrics.['benchmark.p50']", + "content.metrics.['benchmark.p90']", + "content.metrics.['benchmark.p95']", + "content.metrics.['benchmark.p99']", + "content.metrics.['benchmark.min']", + "content.metrics.['benchmark.max']", + "content.metrics.['benchmark.sample_count']", + ] + + def "test #testcaseName"() { + runBenchmark(benchmarkClass) + assertSpansData(testcaseName, [:], BENCHMARK_METRIC_TAGS) + + where: + testcaseName | benchmarkClass + "test-benchmark-simple" | SimpleBenchmark + "test-benchmark-parameterized" | ParameterizedBenchmark + } + + private void runBenchmark(Class benchmarkClass) { + def options = new OptionsBuilder() + .include(benchmarkClass.getName()) + .jvmArgsAppend("-Djmh.ignoreLock=true") + .build() + new Runner(options).run() + } + + @Override + String instrumentedLibraryName() { + "jmh" + } + + @Override + String instrumentedLibraryVersion() { + Runner.class.getPackage().getImplementationVersion() + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java new file mode 100644 index 00000000000..4da75dfc901 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/JmhUtilsTest.java @@ -0,0 +1,52 @@ +package datadog.trace.instrumentation.jmh; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class JmhUtilsTest { + + @Test + void splitBenchmarkName_simple() { + String[] parts = JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod"); + assertArrayEquals(new String[] {"com.example.MyBenchmark", "myMethod"}, parts); + } + + @Test + void splitBenchmarkName_withParams() { + String[] parts = + JmhUtils.splitBenchmarkName("com.example.MyBenchmark.myMethod:size=1000,threads=4"); + assertArrayEquals(new String[] {"com.example.MyBenchmark", "myMethod"}, parts); + } + + @Test + void splitBenchmarkName_noPackage() { + String[] parts = JmhUtils.splitBenchmarkName("MyBenchmark.myMethod"); + assertArrayEquals(new String[] {"MyBenchmark", "myMethod"}, parts); + } + + @Test + void splitBenchmarkName_noDot() { + String[] parts = JmhUtils.splitBenchmarkName("noDot"); + assertArrayEquals(new String[] {"", "noDot"}, parts); + } + + @Test + void testParameters_noParams() { + assertNull(JmhUtils.testParameters("com.example.MyBenchmark.myMethod")); + } + + @Test + void testParameters_withParams() { + String result = JmhUtils.testParameters("com.example.MyBenchmark.myMethod:size=1000,threads=4"); + assertEquals("{\"metadata\":{\"test_name\":\"myMethod:size=1000,threads=4\"}}", result); + } + + @Test + void testParameters_escapesQuotes() { + String result = JmhUtils.testParameters("com.example.MyBenchmark.myMethod:key=\"value\""); + assertEquals("{\"metadata\":{\"test_name\":\"myMethod:key=\\\"value\\\"\"}}", result); + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java new file mode 100644 index 00000000000..2a752e58cab --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/ParameterizedBenchmark.java @@ -0,0 +1,30 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +@State(Scope.Benchmark) +public class ParameterizedBenchmark { + + @Param({"1", "2"}) + int size; + + @Benchmark + public int measure() { + return size * 2; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java new file mode 100644 index 00000000000..2b2b5b6ef8a --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/java/datadog/trace/instrumentation/jmh/benchmarks/SimpleBenchmark.java @@ -0,0 +1,22 @@ +package datadog.trace.instrumentation.jmh.benchmarks; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Fork(0) +public class SimpleBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl new file mode 100644 index 00000000000..39dd9782269 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-parameterized/events.ftl @@ -0,0 +1,225 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.ParameterizedBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id_2}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl new file mode 100644 index 00000000000..677f9ae486d --- /dev/null +++ b/dd-java-agent/instrumentation/jmh/jmh-1.0/src/test/resources/test-benchmark-simple/events.ftl @@ -0,0 +1,145 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "jmh.test_suite", + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "benchmark.run.mode" : "avgt", + "benchmark.run.time_unit" : "NANOSECONDS", + "benchmark.unit" : "ns/op", + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.name" : "measure", + "test.status" : "pass", + "test.suite" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "benchmark.run.forks" : 0, + "benchmark.run.iterations" : 1, + "benchmark.run.threads" : 1, + "benchmark.run.warmup_iterations" : 1, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.trace.instrumentation.jmh.benchmarks.SimpleBenchmark.measure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "jmh-1.0", + "test.framework" : "jmh", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "jmh.test_session", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "jmh", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "jmh", + "test.module" : "jmh-1.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "jmh.test_module", + "resource" : "jmh-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-smoke-tests/jmh/build.gradle b/dd-smoke-tests/jmh/build.gradle new file mode 100644 index 00000000000..2bbe5485741 --- /dev/null +++ b/dd-smoke-tests/jmh/build.gradle @@ -0,0 +1,30 @@ +apply from: "$rootDir/gradle/java.gradle" +description = 'JMH CI Visibility Smoke Tests.' + +dependencies { + implementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' + annotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.37' + + testImplementation project(':dd-smoke-tests:backend-mock') +} + +tasks.withType(Test).configureEach { + jvmArgumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + def jmhJar = configurations.named("runtimeClasspath") + .get() + .find { it.name.contains("jmh-core") } + return ["-Ddatadog.smoketest.jmh.core.jar.path=${jmhJar}"] + } + }) + jvmArgumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + def annprocJar = configurations.named("annotationProcessor") + .get() + .find { it.name.contains("jmh-generator-annprocess") } + return ["-Ddatadog.smoketest.jmh.annproc.jar.path=${annprocJar}"] + } + }) +} diff --git a/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java b/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java new file mode 100644 index 00000000000..d9a22def1ff --- /dev/null +++ b/dd-smoke-tests/jmh/src/main/java/datadog/smoketest/SmokeTestBenchmark.java @@ -0,0 +1,16 @@ +package datadog.smoketest; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class SmokeTestBenchmark { + @Benchmark + public int measure() { + return 42; + } +} diff --git a/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java new file mode 100644 index 00000000000..d7e15cacf0e --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/java/datadog/smoketest/JmhSmokeTest.java @@ -0,0 +1,138 @@ +package datadog.smoketest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.civisibility.CiVisibilitySmokeTest; +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class JmhSmokeTest extends CiVisibilitySmokeTest { + + private static final String TEST_SERVICE_NAME = "test-jmh-service"; + private static final int PROCESS_TIMEOUT_SECS = 120; + + private static final String JMH_CORE_JAR = + System.getProperty("datadog.smoketest.jmh.core.jar.path"); + + static final MockBackend mockBackend = new MockBackend(); + + @BeforeEach + void resetMockBackend() { + mockBackend.reset(); + } + + @AfterAll + static void closeMockBackend() throws Exception { + mockBackend.close(); + } + + @Test + void testBenchmarkSpansAreEmitted() throws Exception { + Map agentArgs = new HashMap<>(); + agentArgs.put(CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED, "false"); + agentArgs.put(GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL, mockBackend.getIntakeUrl()); + + int exitCode = runBenchmark(agentArgs); + assertEquals(0, exitCode, "JMH process should exit cleanly"); + + // 4 events: test_session_end, test_module_end, test_suite_end, test (benchmark method) + List> events = mockBackend.waitForEvents(4); + assertEquals(4, events.size()); + + Map testEvent = findEvent(events, "test"); + assertNotNull(testEvent, "Expected a test span for the benchmark method"); + + @SuppressWarnings("unchecked") + Map meta = + (Map) ((Map) testEvent.get("content")).get("meta"); + @SuppressWarnings("unchecked") + Map metrics = + (Map) ((Map) testEvent.get("content")).get("metrics"); + + assertEquals("jmh", meta.get("test.framework")); + assertEquals("measure", meta.get("test.name")); + assertEquals("datadog.smoketest.SmokeTestBenchmark", meta.get("test.suite")); + assertEquals("pass", meta.get("test.status")); + assertEquals("avgt", meta.get("benchmark.run.mode")); + assertEquals("ns/op", meta.get("benchmark.unit")); + + assertNotNull(metrics.get("benchmark.value"), "benchmark.value should be present"); + assertTrue( + ((Number) metrics.get("benchmark.value")).doubleValue() > 0, + "benchmark.value should be positive"); + } + + private int runBenchmark(Map additionalAgentArgs) throws Exception { + assertTrue(new File(JMH_CORE_JAR).isFile(), "JMH core jar not found: " + JMH_CORE_JAR); + + String classpath = buildClasspath(); + + List command = new ArrayList<>(); + command.add(javaPath()); + command.addAll( + buildJvmArguments(mockBackend.getIntakeUrl(), TEST_SERVICE_NAME, additionalAgentArgs)); + Collections.addAll(command, "-cp", classpath); + command.add("org.openjdk.jmh.Main"); + command.add("datadog.smoketest.SmokeTestBenchmark.*"); + Collections.addAll(command, "-f", "0"); // run in-process (no forking) + Collections.addAll(command, "-wi", "1"); // 1 warmup iteration + Collections.addAll(command, "-i", "1"); // 1 measurement iteration + Collections.addAll(command, "-w", "1ms"); // warmup duration + Collections.addAll(command, "-r", "1ms"); // measurement duration + Collections.addAll(command, "-jvmArgs", "-Djmh.ignoreLock=true"); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.redirectErrorStream(true); + processBuilder.environment().put("DD_API_KEY", "01234567890abcdef123456789ABCDEF"); + Process p = processBuilder.start(); + + // consume output to avoid blocking + final java.io.InputStream stdout = p.getInputStream(); + Thread outputConsumer = + new Thread() { + @Override + public void run() { + try { + byte[] buf = new byte[1024]; + int n; + while ((n = stdout.read(buf)) != -1) { + System.out.write(buf, 0, n); + } + } catch (Exception ignored) { + } + } + }; + outputConsumer.setDaemon(true); + outputConsumer.start(); + + if (!p.waitFor(PROCESS_TIMEOUT_SECS, TimeUnit.SECONDS)) { + p.destroyForcibly(); + throw new TimeoutException("JMH process timed out after " + PROCESS_TIMEOUT_SECS + "s"); + } + return p.exitValue(); + } + + private static String buildClasspath() { + // Use the current test process classpath — it includes jmh-core, SmokeTestBenchmark, and its + // META-INF/BenchmarkList (generated by the annotation processor at compile time) + return System.getProperty("java.class.path"); + } + + @SuppressWarnings("unchecked") + private static Map findEvent(List> events, String type) { + return events.stream().filter(e -> type.equals(e.get("type"))).findFirst().orElse(null); + } +} diff --git a/dd-smoke-tests/jmh/src/test/resources/logback.xml b/dd-smoke-tests/jmh/src/test/resources/logback.xml new file mode 100644 index 00000000000..24d6bcd768a --- /dev/null +++ b/dd-smoke-tests/jmh/src/test/resources/logback.xml @@ -0,0 +1,3 @@ + + + diff --git a/docs/design/jmh-ci-visibility.md b/docs/design/jmh-ci-visibility.md new file mode 100644 index 00000000000..dc62f345021 --- /dev/null +++ b/docs/design/jmh-ci-visibility.md @@ -0,0 +1,235 @@ +# Design: JMH Benchmark CI Visibility Instrumentation (SDTEST-930) + +## Problem + +JMH (Java Microbenchmark Harness, `org.openjdk.jmh`, version 1.37) is the dominant Java +benchmarking framework. Benchmark runs are not currently reported to CI Visibility, so +performance regressions are invisible in the Datadog test explorer. + +## Goals + +- Report each JMH benchmark method as a CI Visibility **test span** (`test.type = "benchmark"`) +- Attach aggregated performance metrics (score, error, unit, percentiles) as span tags +- **Zero overhead on the benchmark hot path** — hook only fires once per benchmark method, not per invocation +- Disabled by default (like Renaissance); opt-in via `DD_TRACE_JMH_ENABLED=true` + +## JMH Lifecycle and Hook Point + +JMH execution flow (per benchmark method): + +``` +Runner.run() + └─ for each benchmark method: + OutputFormat.startBenchmark(BenchmarkParams) ← span start + for each fork: + for each warmup iteration: + OutputFormat.iteration(...) + for each measurement iteration: + OutputFormat.iterationResult(...) + OutputFormat.iteration(...) + OutputFormat.endBenchmark(BenchmarkResult) ← span finish + attach metrics + └─ OutputFormat.endRun(Collection) +``` + +`OutputFormat` is an interface that JMH calls for all lifecycle events. Critically: +- `startBenchmark(BenchmarkParams)` fires **once** per benchmark method before any invocations +- `endBenchmark(BenchmarkResult)` fires **once** per benchmark method after all forks and iterations + +These are the only two hooks needed. No hot-path instrumentation is required. + +### Why not instrument `@Benchmark`-annotated methods directly? + +Those methods are called millions of times during warmup and measurement. Advice on the +hot path would perturb the benchmark results and add massive instrumentation overhead. + +## Instrumentation Strategy + +### Hook: `OutputFormat` injection + +JMH's `Runner` is constructed by user code: + +```java +Runner runner = new Runner(options); +runner.run(); +``` + +The `Runner` constructor accepts an `Options` object, which includes an `OutputFormat`. We +instrument `Runner.` to wrap the user-supplied `OutputFormat` with our own +`DDOutputFormat` decorator before the field is stored. + +**Bytecode advice on `Runner.`:** + +```java +@Advice.OnMethodExit +public static void onExit(@Advice.FieldValue(value = "out", readOnly = false) OutputFormat out) { + out = new DDOutputFormat(out); +} +``` + +`Runner.out` is the `OutputFormat` field. Wrapping it at construction time means our +decorator receives all lifecycle callbacks without any per-invocation cost. + +### Alternative: instrument `Runner.run()` return value + +If wrapping the constructor is fragile due to JMH refactors, a fallback is to instrument +`Runner.run()` / `Runner.runBenchmarks()` exit and iterate the returned +`Collection` to emit spans retroactively. This loses wall-clock timing fidelity +(span duration reflects only the post-run callback time) but is simpler and requires no +field access. + +The constructor approach is preferred; the `run()` return approach is the fallback. + +## Data Model + +### Span structure + +Each JMH benchmark method produces **two spans**: + +| Span | Mapping | Notes | +|------|---------|-------| +| Test suite span | Benchmark class (e.g., `com.example.MyBenchmark`) | One per class | +| Test span | Benchmark method (e.g., `myMethod`) | One per `@Benchmark` method per parameter set | + +`BenchmarkParams.getBenchmark()` returns the fully-qualified name +`"com.example.MyBenchmark.myMethod"` — split on the last `.` to get class and method. + +### Standard CI Visibility tags + +| Tag | Source | Value | +|-----|--------|-------| +| `test.type` | constant | `"benchmark"` | +| `test.framework` | constant | `"jmh"` | +| `test.framework_version` | `Version.getVersion(Runner.class)` | e.g. `"1.37"` | +| `test.name` | `BenchmarkParams.getBenchmark()` last segment | e.g. `"myMethod"` | +| `test.suite` | `BenchmarkParams.getBenchmark()` prefix | e.g. `"com.example.MyBenchmark"` | +| `test.status` | always `"pass"` (JMH throws on error) | `"fail"` if exception from `endBenchmark` | +| `test.parameters` | `BenchmarkParams.getParamsKeys()` + `getParam(key)` | JSON object, omit if empty | +| `test.source.class` | derived from suite name | class name | +| `test.source.method` | derived from test name | method name | + +### Benchmark-specific metric tags + +These are numeric tags added on the test span, not on suite spans: + +| Tag | Source | Notes | +|-----|--------|-------| +| `benchmark.run.iterations` | `BenchmarkParams.getMeasurement().getCount()` | Measurement iteration count | +| `benchmark.run.forks` | `BenchmarkParams.getForks()` | Fork count | +| `benchmark.run.threads` | `BenchmarkParams.getThreads()` | Thread count | +| `benchmark.run.warmup_iterations` | `BenchmarkParams.getWarmup().getCount()` | Warmup iteration count | +| `benchmark.run.time_unit` | `BenchmarkParams.getTimeUnit().name()` | e.g. `"NANOSECONDS"` | +| `benchmark.run.mode` | `BenchmarkParams.getMode().shortLabel()` | e.g. `"thrpt"`, `"avgt"` | +| `benchmark.value` | `Result.getScore()` | Primary metric score | +| `benchmark.error` | `Result.getScoreError()` | 99.9% CI half-width; `NaN` for single-shot | +| `benchmark.unit` | `Result.getScoreUnit()` | e.g. `"ops/ms"`, `"ns/op"` | +| `benchmark.p50` | `Statistics.getPercentile(50)` | Median | +| `benchmark.p90` | `Statistics.getPercentile(90)` | | +| `benchmark.p95` | `Statistics.getPercentile(95)` | | +| `benchmark.p99` | `Statistics.getPercentile(99)` | | +| `benchmark.min` | `Statistics.getMin()` | | +| `benchmark.max` | `Statistics.getMax()` | | +| `benchmark.sample_count` | `Statistics.getN()` | Total sample count | + +`Statistics` is available via `BenchmarkResult.getPrimaryResult().getStatistics()`. + +For `SingleShotTime` mode, only `benchmark.value` and `benchmark.unit` are populated +(no iterations → no distribution). + +### Access path summary (zero hot-path calls) + +``` +endBenchmark(BenchmarkResult result) +├── result.getParams() → BenchmarkParams (all config) +│ ├── .getBenchmark() → "com.example.MyBenchmark.myMethod" +│ ├── .getMode().shortLabel() → "thrpt" +│ ├── .getThreads() → 4 +│ ├── .getForks() → 5 +│ ├── .getMeasurement().getCount() → 5 +│ ├── .getWarmup().getCount() → 5 +│ ├── .getTimeUnit() → NANOSECONDS +│ └── .getParamsKeys() + .getParam(key) → {"size": "1000"} +└── result.getPrimaryResult() → Result + ├── .getScore() → 1234.56 + ├── .getScoreError() → 12.34 + ├── .getScoreUnit() → "ns/op" + └── .getStatistics() → Statistics + ├── .getPercentile(50/90/95/99) → distribution + ├── .getMin() / .getMax() → bounds + └── .getN() → sample count +``` + +## Module Layout + +``` +dd-java-agent/instrumentation/jmh/ +└── jmh-1.0/ # JMH's OutputFormat API is stable since 1.0 + ├── build.gradle + ├── gradle.lockfile + └── src/ + └── main/java/datadog/trace/instrumentation/jmh/ + ├── JmhInstrumentation.java # InstrumenterModule targeting Runner. + ├── DDOutputFormat.java # OutputFormat decorator + └── JmhUtils.java # Parsing helpers (benchmark name split, etc.) +``` + +### `build.gradle` + +```groovy +apply from: "$rootDir/gradle/java.gradle" + +dependencies { + compileOnly group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.0' + testImplementation group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.37' +} +``` + +## Changes Required Outside the New Module + +| File | Change | +|------|--------| +| `internal-api/.../telemetry/tag/TestFrameworkInstrumentation.java` | Add `JMH` enum constant | +| `internal-api/.../bootstrap/instrumentation/api/Tags.java` | Add `benchmark.*` tag constants | +| `internal-api/.../decorator/TestDecorator.java` | Add `TEST_TYPE_BENCHMARK = "benchmark"` constant | +| `dd-java-agent/agent-ci-visibility/.../decorator/TestDecoratorImpl.java` | Handle benchmark type | + +## Resolved Design Decisions + +### Suite span scope +Flat: each benchmark method (including each `@Param` combination) gets its own suite span +and test span pair. No grouping by class. + +### Parameterized benchmarks +Follow the same pattern as JUnit 5 parameterized tests: set `test.parameters` to +`{"metadata":{"test_name":""}}` where the display name is the parameterized +suffix of the benchmark name. + +JMH encodes `@Param` combinations by appending them after a colon: +`"com.example.MyBenchmark.myMethod:size=1000,threads=4"` + +Parsing: +- `test.suite` = everything before the last `.` before the colon: `"com.example.MyBenchmark"` +- `test.name` = method name segment without params: `"myMethod"` +- `test.parameters` = `{"metadata":{"test_name":"myMethod:size=1000,threads=4"}}` (non-null only when a colon is present) + +The parameterized variant is a distinct test identity (unique `test.name` + `test.parameters` +combination), matching how JUnit 5 handles `@ParameterizedTest`. + +## Open Questions + +1. **CI Visibility opt-out for ITR**: JMH benchmarks should likely be excluded from + Intelligent Test Runner (skip logic) since skipping a benchmark run defeats its purpose. + Mark them as `@ITRUnskippable` equivalent or configure the handler to always-run. + +2. **Forked JVM mode**: When `@Fork(1+)` is used, each fork is a separate JVM process. + The tracer in the forked process needs to propagate the session/module/suite IDs from + the parent. This is the same challenge as Gradle worker forks — check if the existing + IPC mechanism in `ProcessHierarchy` covers it. + +## Implementation Order + +1. Add `benchmark.*` tag constants to `Tags.java` +2. Add `JMH` to `TestFrameworkInstrumentation` +3. Add `TEST_TYPE_BENCHMARK` to `TestDecorator` +4. Implement `DDOutputFormat` + `JmhInstrumentation` + `JmhUtils` +5. Wire module into the agent's instrumentation list +6. Add JUnit 5 integration tests (use JMH `Runner` programmatically in test; verify spans) diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java index eedbaed7e80..20604f54739 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/TestFrameworkInstrumentation.java @@ -13,6 +13,7 @@ public enum TestFrameworkInstrumentation implements TagValue { SCALATEST, KARATE, WEAVER, + JMH, OTHER; private final String s; diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index 14496e8b243..66018e35cec 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -86,6 +86,23 @@ public class Tags { public static final String TEST_BROWSER_VERSION = "test.browser.version"; public static final String TEST_CALLBACK = "test.callback"; + public static final String BENCHMARK_VALUE = "benchmark.value"; + public static final String BENCHMARK_ERROR = "benchmark.error"; + public static final String BENCHMARK_UNIT = "benchmark.unit"; + public static final String BENCHMARK_MODE = "benchmark.run.mode"; + public static final String BENCHMARK_ITERATIONS = "benchmark.run.iterations"; + public static final String BENCHMARK_WARMUP_ITERATIONS = "benchmark.run.warmup_iterations"; + public static final String BENCHMARK_FORKS = "benchmark.run.forks"; + public static final String BENCHMARK_THREADS = "benchmark.run.threads"; + public static final String BENCHMARK_TIME_UNIT = "benchmark.run.time_unit"; + public static final String BENCHMARK_P50 = "benchmark.p50"; + public static final String BENCHMARK_P90 = "benchmark.p90"; + public static final String BENCHMARK_P95 = "benchmark.p95"; + public static final String BENCHMARK_P99 = "benchmark.p99"; + public static final String BENCHMARK_MIN = "benchmark.min"; + public static final String BENCHMARK_MAX = "benchmark.max"; + public static final String BENCHMARK_SAMPLE_COUNT = "benchmark.sample_count"; + public static final String TEST_SESSION_ID = "test_session_id"; public static final String TEST_MODULE_ID = "test_module_id"; public static final String TEST_SUITE_ID = "test_suite_id"; diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 32e40412662..6cf109ac1d9 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7281,6 +7281,14 @@ "aliases": ["DD_TRACE_INTEGRATION_JETTY_WEBSOCKET_ENABLED", "DD_INTEGRATION_JETTY_WEBSOCKET_ENABLED"] } ], + "DD_TRACE_JMH_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_JMH_ENABLED", "DD_INTEGRATION_JMH_ENABLED"] + } + ], "DD_TRACE_JMS_1_ENABLED": [ { "version": "A", diff --git a/settings.gradle.kts b/settings.gradle.kts index dda1f432e6f..fe18d3f349c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -201,6 +201,7 @@ include( ":dd-smoke-tests:jersey-2", ":dd-smoke-tests:jersey-3", ":dd-smoke-tests:jboss-modules", + ":dd-smoke-tests:jmh", ":dd-smoke-tests:junit-console", ":dd-smoke-tests:kafka-2", ":dd-smoke-tests:kafka-3", @@ -445,6 +446,7 @@ include( ":dd-java-agent:instrumentation:jetty:jetty-util-9.4.31", ":dd-java-agent:instrumentation:jms:jakarta-jms-3.0", ":dd-java-agent:instrumentation:jms:javax-jms-1.1", + ":dd-java-agent:instrumentation:jmh:jmh-1.0", ":dd-java-agent:instrumentation:jose-jwt-4.0", ":dd-java-agent:instrumentation:jsp-2.3", ":dd-java-agent:instrumentation:junit:junit-4:junit-4-cucumber-5.4",