diff --git a/benchmarks/README.md b/benchmarks/README.md index b4c24e301..44272a887 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -4,63 +4,60 @@ See [RESULTS.md](RESULTS.md) for detailed benchmark results and analysis. ## Running -Run all benchmarks for a given implementation: +All benchmarks use [kotlinx-benchmark](https://github.com/Kotlin/kotlinx-benchmark), +which delegates to JMH on JVM and uses its own runtime on Kotlin/Native. ``` -./gradlew :benchmarks:protokt-benchmarks:run -./gradlew :benchmarks:protobuf-java-benchmarks:run -./gradlew :benchmarks:wire-benchmarks:run +./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark +./gradlew :benchmarks:protokt-benchmarks:macosArm64Benchmark +./gradlew :benchmarks:protobuf-java-benchmarks:benchmark +./gradlew :benchmarks:wire-benchmarks:benchmark ``` -## Flags +All modules share the same default configuration (3 warmup iterations, +5 measurement iterations, 10s each, 2 forks, average time in ms/op). -Flags are passed via `--args`: +## Gradle properties -| Flag | Description | -|------|-------------| -| `-i regex` | Include only benchmarks matching regex | -| `-e regex` | Exclude benchmarks matching regex | -| `-p name=value` | Set a JMH parameter | +Override defaults at invocation time via `-P`: -When no `-i` flag is given, all benchmarks in the class are included. - -## Parameters - -protokt benchmarks accept these JMH parameters (set via `-p`): - -| Parameter | Values | Default | -|-----------|--------|---------| -| `collectionFactory` | `protokt.v1.DefaultCollectionFactory`, `protokt.v1.PersistentCollectionFactory` | Both | -| `codec` | `protokt.v1.ProtobufJavaCodec`, `protokt.v1.KotlinxIoCodec`, `protokt.v1.ProtoktCodec`, `protokt.v1.OptimalKmpCodec`, `protokt.v1.OptimalJvmCodec` | All | +| Property | Description | Default | +|----------|-------------|---------| +| `benchmarkInclude` | Regex to include matching benchmarks | all | +| `benchmarkExclude` | Regex to exclude matching benchmarks | none | +| `benchmarkParam` | Comma-separated `name=value` pairs for `@Param` fields | all values | +| `benchmarkWarmups` | Warmup iterations | 3 | +| `benchmarkIterations` | Measurement iterations | 5 | +| `benchmarkForks` | JVM forks | 2 | ## Examples -Run a single benchmark method: +Run only serialization benchmarks: ``` -./gradlew :benchmarks:protokt-benchmarks:run --args="-i serializeSmall" +./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark -PbenchmarkInclude=.*serialize.* ``` -Exclude copy/append benchmarks: +Run with a specific codec: ``` -./gradlew :benchmarks:protokt-benchmarks:run --args="-e .*copyAppend.*" +./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark -PbenchmarkParam=codec=protokt.v1.ProtoktCodec ``` -Pin a JMH parameter: +Quick smoke test (1 warmup, 1 iteration, 1 fork): ``` -./gradlew :benchmarks:protokt-benchmarks:run --args="-p collectionFactory=protokt.v1.DefaultCollectionFactory" +./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark -PbenchmarkWarmups=1 -PbenchmarkIterations=1 -PbenchmarkForks=1 ``` -Run with a specific codec: +## Parameters -``` -./gradlew :benchmarks:protokt-benchmarks:run --args="-p codec=protokt.v1.ProtoktCodec" -``` +protokt benchmarks accept `@Param`-annotated properties: -Combine flags: +| Parameter | Targets | Values | +|-----------|---------|--------| +| `collectionFactory` | JVM + Native | `protokt.v1.DefaultCollectionFactory`, `protokt.v1.PersistentCollectionFactory` | +| `codec` | JVM only | `protokt.v1.ProtobufJavaCodec`, `protokt.v1.KotlinxIoCodec`, `protokt.v1.ProtoktCodec`, `protokt.v1.OptimalKmpCodec`, `protokt.v1.OptimalJvmCodec` | -``` -./gradlew :benchmarks:protokt-benchmarks:run --args="-i serialize -e .*String.*" -``` +Native benchmarks always use `ProtoktCodec`. Collection factory selection works +on native via direct override (no env var or system property needed). diff --git a/benchmarks/benchmarks-util/build.gradle.kts b/benchmarks/benchmarks-util/build.gradle.kts index 6bbb2c926..834fcd472 100644 --- a/benchmarks/benchmarks-util/build.gradle.kts +++ b/benchmarks/benchmarks-util/build.gradle.kts @@ -14,5 +14,7 @@ */ plugins { - id("protokt.benchmarks-conventions") + id("protokt.multiplatform-conventions") } + +enableNativeTargets() diff --git a/benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/BenchmarkUtils.kt b/benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/BenchmarkUtils.kt new file mode 100644 index 000000000..0ec3e0d02 --- /dev/null +++ b/benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/BenchmarkUtils.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +import kotlin.random.Random + +expect fun readDatasetBytes(name: String): ByteArray + +/** Mix of 1-byte (ASCII), 2-byte (Latin Extended), and 3-byte (CJK) UTF-8 characters. */ +fun randomUtf8String(random: Random, charCount: Int): String { + val sb = StringBuilder(charCount) + repeat(charCount) { + sb.append( + when (random.nextInt(3)) { + 0 -> 'a' + random.nextInt(26) + 1 -> (0x00C0 + random.nextInt(64)).toChar() + else -> (0x4E00 + random.nextInt(0x5000)).toChar() + } + ) + } + return sb.toString() +} diff --git a/benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/ProtobufBenchmarkSet.kt b/benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/ProtobufBenchmarkSet.kt new file mode 100644 index 000000000..6663622d5 --- /dev/null +++ b/benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/ProtobufBenchmarkSet.kt @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +interface ProtobufBenchmarkSet { + fun deserializeLargeFromMemory(bh: T) + fun deserializeMediumFromMemory(bh: T) + fun deserializeSmallFromMemory(bh: T) + fun serializeLargeToMemory(bh: T) + fun serializeMediumToMemory(bh: T) + fun serializeSmallToMemory(bh: T) + fun serializeLargeStreaming(bh: T) + fun serializeMediumStreaming(bh: T) + fun serializeSmallStreaming(bh: T) + fun copyAppendListLarge(bh: T) + fun copyAppendMapLarge(bh: T) + fun copyAppendListMedium(bh: T) + fun copyAppendMapMedium(bh: T) + fun copyAppendListSmall(bh: T) + fun copyAppendMapSmall(bh: T) + fun passThroughLargeFromMemory(bh: T) + fun passThroughMediumFromMemory(bh: T) + fun passThroughSmallFromMemory(bh: T) + fun mutateAndSerializeStringHeavy(bh: T) + fun mutateAndSerializeStringHeavyStreaming(bh: T) + fun passThroughStringHeavy(bh: T) + fun mutateAndSerializeStringOneof(bh: T) + fun passThroughStringOneof(bh: T) + fun mutateAndSerializeStringOneofStreaming(bh: T) + fun mutateAndSerializeStringOneof20k(bh: T) + fun mutateAndSerializeStringOneof20kStreaming(bh: T) + fun mutateAndSerializeStringOneofVeryHeavy(bh: T) + fun mutateAndSerializeStringVeryHeavy(bh: T) + fun passThroughStringRepeated(bh: T) + fun passThroughStringMap(bh: T) + fun deserializeStringRepeated(bh: T) + fun deserializeStringMap(bh: T) + fun copyAppendRepeatedString(bh: T) + fun copyAppendMapStringString(bh: T) +} diff --git a/benchmarks/benchmarks-util/src/jsMain/kotlin/protokt/v1/benchmarks/DatasetReader.kt b/benchmarks/benchmarks-util/src/jsMain/kotlin/protokt/v1/benchmarks/DatasetReader.kt new file mode 100644 index 000000000..37ff5fdc0 --- /dev/null +++ b/benchmarks/benchmarks-util/src/jsMain/kotlin/protokt/v1/benchmarks/DatasetReader.kt @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +actual fun readDatasetBytes(name: String): ByteArray = + error("Benchmarks not supported on JS") diff --git a/benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmBenchmarkUtils.kt b/benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmBenchmarkUtils.kt new file mode 100644 index 000000000..9db849495 --- /dev/null +++ b/benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmBenchmarkUtils.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +import java.io.File + +fun readData(dataset: String) = + File("../build/datasets/dataset-$dataset").inputStream().buffered() + +actual fun readDatasetBytes(name: String): ByteArray = + File("../build/datasets/dataset-$name").readBytes() diff --git a/benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmProtobufBenchmarkSet.kt b/benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmProtobufBenchmarkSet.kt new file mode 100644 index 000000000..182b8380e --- /dev/null +++ b/benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmProtobufBenchmarkSet.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +interface JvmProtobufBenchmarkSet : ProtobufBenchmarkSet { + fun deserializeStringHeavyStreaming(bh: T) + fun deserializeStringOneofStreaming(bh: T) + fun deserializeLargeStreaming(bh: T) + fun deserializeMediumStreaming(bh: T) + fun deserializeSmallStreaming(bh: T) +} diff --git a/benchmarks/benchmarks-util/src/main/kotlin/protokt/v1/benchmarks/BenchmarkUtils.kt b/benchmarks/benchmarks-util/src/main/kotlin/protokt/v1/benchmarks/BenchmarkUtils.kt deleted file mode 100644 index e59487cc1..000000000 --- a/benchmarks/benchmarks-util/src/main/kotlin/protokt/v1/benchmarks/BenchmarkUtils.kt +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2019 Toast, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package protokt.v1.benchmarks - -import org.openjdk.jmh.results.format.ResultFormatType -import org.openjdk.jmh.runner.Runner -import org.openjdk.jmh.runner.options.OptionsBuilder -import java.io.File -import java.util.Random -import kotlin.reflect.KClass - -fun readData(dataset: String) = - File("../build/datasets/dataset-$dataset").inputStream().buffered() - -/** Mix of 1-byte (ASCII), 2-byte (Latin Extended), and 3-byte (CJK) UTF-8 characters. */ -fun randomUtf8String(random: Random, charCount: Int): String { - val sb = StringBuilder(charCount) - repeat(charCount) { - sb.append( - when (random.nextInt(3)) { - 0 -> 'a' + random.nextInt(26) - 1 -> (0x00C0 + random.nextInt(64)).toChar() - else -> (0x4E00 + random.nextInt(0x5000)).toChar() - } - ) - } - return sb.toString() -} - -/** - * Runs JMH benchmarks for the given class. - * - * Supported args (passed via `--args`): - * -i regex Include only benchmarks matching regex (e.g. `-i mutateAndSerialize`) - * -e regex Exclude benchmarks matching regex (e.g. `-e .*copyAppend.*`) - * -p name=value Set a JMH parameter (e.g. `-p codec=protokt.v1.ProtoktCodec`) - * -prof name Add a JMH profiler (e.g. `-prof async`) - * -wi n Warmup iterations (default 3) - * -mi n Measurement iterations (default 5) - * -f n Forks (default 2) - * -jvmArgs args Extra JVM args (e.g. `-jvmArgs -agentpath:...`) - */ -fun run(self: KClass<*>, args: Array = emptyArray()) { - var resultSuffix = "" - val opts = OptionsBuilder() - .warmupIterations(3) - .measurementIterations(5) - .forks(2) - .resultFormat(ResultFormatType.JSON) - - var hasInclude = false - - args.toList() - .windowed(2, 2, partialWindows = false) - .forEach { (flag, spec) -> - when (flag) { - "-o" -> resultSuffix = "-$spec" - - "-i" -> { - hasInclude = true - opts.include(".*" + self.simpleName + "." + spec + ".*") - } - - "-e" -> opts.exclude(spec) - - "-p" -> { - val (name, value) = spec.split("=", limit = 2) - opts.param(name, *value.split(",").toTypedArray()) - } - - "-prof" -> { - val parts = spec.split(":", limit = 2) - if (parts.size == 2) { - opts.addProfiler(parts[0], parts[1]) - } else { - opts.addProfiler(parts[0]) - } - } - - "-wi" -> opts.warmupIterations(spec.toInt()) - - "-mi" -> opts.measurementIterations(spec.toInt()) - - "-f" -> opts.forks(spec.toInt()) - - "-jvmArgs" -> opts.jvmArgs(spec) - } - } - - opts.result("../build/jmh-${self.simpleName}$resultSuffix.json") - - if (!hasInclude) { - opts.include(".*" + self.simpleName + ".*") - } - - Runner(opts.build()).run() -} diff --git a/benchmarks/benchmarks-util/src/main/kotlin/protokt/v1/benchmarks/ProtobufBenchmarkSet.kt b/benchmarks/benchmarks-util/src/main/kotlin/protokt/v1/benchmarks/ProtobufBenchmarkSet.kt deleted file mode 100644 index ac85f32b2..000000000 --- a/benchmarks/benchmarks-util/src/main/kotlin/protokt/v1/benchmarks/ProtobufBenchmarkSet.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2022 Toast, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package protokt.v1.benchmarks - -import org.openjdk.jmh.infra.Blackhole - -interface ProtobufBenchmarkSet { - fun deserializeLargeFromMemory(bh: Blackhole) - fun deserializeMediumFromMemory(bh: Blackhole) - fun deserializeSmallFromMemory(bh: Blackhole) - fun serializeLargeToMemory(bh: Blackhole) - fun serializeMediumToMemory(bh: Blackhole) - fun serializeSmallToMemory(bh: Blackhole) - fun serializeLargeStreaming(bh: Blackhole) - fun serializeMediumStreaming(bh: Blackhole) - fun serializeSmallStreaming(bh: Blackhole) - fun copyAppendListLarge(bh: Blackhole) - fun copyAppendMapLarge(bh: Blackhole) - fun copyAppendListMedium(bh: Blackhole) - fun copyAppendMapMedium(bh: Blackhole) - fun copyAppendListSmall(bh: Blackhole) - fun copyAppendMapSmall(bh: Blackhole) - fun passThroughLargeFromMemory(bh: Blackhole) - fun passThroughMediumFromMemory(bh: Blackhole) - fun passThroughSmallFromMemory(bh: Blackhole) - fun mutateAndSerializeStringHeavy(bh: Blackhole) - fun mutateAndSerializeStringHeavyStreaming(bh: Blackhole) - fun passThroughStringHeavy(bh: Blackhole) - fun deserializeStringHeavyStreaming(bh: Blackhole) - fun deserializeStringOneofStreaming(bh: Blackhole) - fun mutateAndSerializeStringOneof(bh: Blackhole) - fun passThroughStringOneof(bh: Blackhole) - fun mutateAndSerializeStringOneofStreaming(bh: Blackhole) - fun mutateAndSerializeStringOneof20k(bh: Blackhole) - fun mutateAndSerializeStringOneof20kStreaming(bh: Blackhole) - fun mutateAndSerializeStringOneofVeryHeavy(bh: Blackhole) - fun mutateAndSerializeStringVeryHeavy(bh: Blackhole) - fun passThroughStringRepeated(bh: Blackhole) - fun passThroughStringMap(bh: Blackhole) - fun deserializeStringRepeated(bh: Blackhole) - fun deserializeStringMap(bh: Blackhole) - fun copyAppendRepeatedString(bh: Blackhole) - fun copyAppendMapStringString(bh: Blackhole) - fun deserializeLargeStreaming(bh: Blackhole) - fun deserializeMediumStreaming(bh: Blackhole) - fun deserializeSmallStreaming(bh: Blackhole) -} diff --git a/benchmarks/benchmarks-util/src/nativeMain/kotlin/protokt/v1/benchmarks/DatasetReader.kt b/benchmarks/benchmarks-util/src/nativeMain/kotlin/protokt/v1/benchmarks/DatasetReader.kt new file mode 100644 index 000000000..7bbcb826a --- /dev/null +++ b/benchmarks/benchmarks-util/src/nativeMain/kotlin/protokt/v1/benchmarks/DatasetReader.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class) + +package protokt.v1.benchmarks + +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.convert +import kotlinx.cinterop.usePinned +import platform.posix.SEEK_END +import platform.posix.fclose +import platform.posix.fopen +import platform.posix.fread +import platform.posix.fseek +import platform.posix.ftell +import platform.posix.rewind + +actual fun readDatasetBytes(name: String): ByteArray { + val file = fopen("../build/datasets/dataset-$name", "rb") ?: error("Cannot open dataset: $name") + fseek(file, 0, SEEK_END) + @Suppress("REDUNDANT_CALL_OF_CONVERSION_METHOD") + val size = ftell(file).toInt() + rewind(file) + val bytes = ByteArray(size) + bytes.usePinned { pinned -> + fread(pinned.addressOf(0), 1u.convert(), size.convert(), file) + } + fclose(file) + return bytes +} diff --git a/benchmarks/protobuf-java-benchmarks/build.gradle.kts b/benchmarks/protobuf-java-benchmarks/build.gradle.kts index 4c12e847e..ee4ee7dcc 100644 --- a/benchmarks/protobuf-java-benchmarks/build.gradle.kts +++ b/benchmarks/protobuf-java-benchmarks/build.gradle.kts @@ -16,14 +16,14 @@ plugins { id("protokt.benchmarks-conventions") id("com.google.protobuf") - application } defaultProtoc() -configure { - mainClass.set("protokt.v1.benchmarks.ProtobufBenchmarksKt") - executableDir = ".." +benchmark { + targets { + register("main") + } } dependencies { @@ -33,6 +33,6 @@ dependencies { protobuf(project(":benchmarks:schema")) } -tasks.named("run") { +tasks.matching { it.name.contains("benchmark", ignoreCase = true) }.configureEach { dependsOn(":benchmarks:datasets") } diff --git a/benchmarks/protobuf-java-benchmarks/src/main/kotlin/protokt/v1/benchmarks/ProtobufBenchmarks.kt b/benchmarks/protobuf-java-benchmarks/src/main/kotlin/protokt/v1/benchmarks/ProtobufBenchmarks.kt index b94d95bcf..da428c0a8 100644 --- a/benchmarks/protobuf-java-benchmarks/src/main/kotlin/protokt/v1/benchmarks/ProtobufBenchmarks.kt +++ b/benchmarks/protobuf-java-benchmarks/src/main/kotlin/protokt/v1/benchmarks/ProtobufBenchmarks.kt @@ -19,23 +19,23 @@ import com.google.protobuf.ByteString import com.google.protobuf.UnsafeByteOperations import com.google.protobuf.benchmarks.Benchmarks import com.toasttab.protokt.v1.benchmarks.GenericMessage -import org.openjdk.jmh.annotations.Benchmark -import org.openjdk.jmh.annotations.BenchmarkMode -import org.openjdk.jmh.annotations.Mode -import org.openjdk.jmh.annotations.OutputTimeUnit -import org.openjdk.jmh.annotations.Scope -import org.openjdk.jmh.annotations.Setup -import org.openjdk.jmh.annotations.State -import org.openjdk.jmh.infra.Blackhole +import kotlinx.benchmark.Benchmark +import kotlinx.benchmark.BenchmarkMode +import kotlinx.benchmark.BenchmarkTimeUnit +import kotlinx.benchmark.Blackhole +import kotlinx.benchmark.Mode +import kotlinx.benchmark.OutputTimeUnit +import kotlinx.benchmark.Scope +import kotlinx.benchmark.Setup +import kotlinx.benchmark.State import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream -import java.util.Random -import java.util.concurrent.TimeUnit +import kotlin.random.Random @BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MILLISECONDS) +@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @State(Scope.Benchmark) -open class ProtobufBenchmarks : ProtobufBenchmarkSet { +class ProtobufBenchmarks : JvmProtobufBenchmarkSet { private lateinit var largeDataset: Benchmarks.BenchmarkDataset private lateinit var largeParsedDataset: List private lateinit var mediumDataset: Benchmarks.BenchmarkDataset @@ -487,7 +487,3 @@ open class ProtobufBenchmarks : ProtobufBenchmarkSet { } } } - -fun main(args: Array) { - run(ProtobufBenchmarks::class, args) -} diff --git a/benchmarks/protokt-benchmarks/build.gradle.kts b/benchmarks/protokt-benchmarks/build.gradle.kts index ac58227d3..1b756a0cf 100644 --- a/benchmarks/protokt-benchmarks/build.gradle.kts +++ b/benchmarks/protokt-benchmarks/build.gradle.kts @@ -16,26 +16,58 @@ import com.google.protobuf.gradle.protobuf plugins { - id("protokt.benchmarks-conventions") - application + id("protokt.multiplatform-conventions") + id("org.jetbrains.kotlinx.benchmark") + id("org.jetbrains.kotlin.plugin.allopen") +} + +allOpen { + annotation("org.openjdk.jmh.annotations.State") + annotation("kotlinx.benchmark.State") } localProtokt() +friendPaths(":protokt-runtime", ":protokt-runtime-persistent-collections") -configure { - mainClass.set("protokt.v1.benchmarks.ProtoktBenchmarksKt") - executableDir = ".." +enableNativeTargets() + +configure { + compilerOptions { + freeCompilerArgs.add("-opt-in=protokt.v1.OnlyForUseByGeneratedProtoCode") + } +} +configureBenchmarks() + +kotlin { + sourceSets { + val commonMain by getting { + dependencies { + implementation(project(":benchmarks:benchmarks-util")) + implementation(project(":protokt-runtime-kotlinx-io")) + implementation(project(":protokt-runtime-persistent-collections")) + implementation(libs.kotlinx.benchmark.runtime) + } + } + + val jvmMain by getting { + dependencies { + implementation(kotlin("reflect")) + } + } + } } dependencies { protobuf(project(":benchmarks:schema")) +} - implementation(kotlin("reflect")) - implementation(project(":benchmarks:benchmarks-util")) - implementation(project(":protokt-runtime-persistent-collections")) - implementation(project(":protokt-runtime-kotlinx-io")) +benchmark { + targets { + register("jvm") + register("macosArm64") + } } -tasks.named("run") { +tasks.matching { it.name.contains("benchmark", ignoreCase = true) }.configureEach { dependsOn(":benchmarks:datasets") } diff --git a/benchmarks/protokt-benchmarks/src/commonMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt b/benchmarks/protokt-benchmarks/src/commonMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt new file mode 100644 index 000000000..4b54d165f --- /dev/null +++ b/benchmarks/protokt-benchmarks/src/commonMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +expect fun applyBenchmarkConfig(collectionFactory: String, codec: String) diff --git a/benchmarks/protokt-benchmarks/src/main/kotlin/protokt/v1/benchmarks/ProtoktBenchmarks.kt b/benchmarks/protokt-benchmarks/src/commonMain/kotlin/protokt/v1/benchmarks/ProtoktMultiplatformBenchmarks.kt similarity index 81% rename from benchmarks/protokt-benchmarks/src/main/kotlin/protokt/v1/benchmarks/ProtoktBenchmarks.kt rename to benchmarks/protokt-benchmarks/src/commonMain/kotlin/protokt/v1/benchmarks/ProtoktMultiplatformBenchmarks.kt index 63056050a..849a10493 100644 --- a/benchmarks/protokt-benchmarks/src/main/kotlin/protokt/v1/benchmarks/ProtoktBenchmarks.kt +++ b/benchmarks/protokt-benchmarks/src/commonMain/kotlin/protokt/v1/benchmarks/ProtoktMultiplatformBenchmarks.kt @@ -15,31 +15,29 @@ package protokt.v1.benchmarks -import kotlinx.io.asSink -import kotlinx.io.buffered -import org.openjdk.jmh.annotations.Benchmark -import org.openjdk.jmh.annotations.BenchmarkMode -import org.openjdk.jmh.annotations.Mode -import org.openjdk.jmh.annotations.OutputTimeUnit -import org.openjdk.jmh.annotations.Param -import org.openjdk.jmh.annotations.Scope -import org.openjdk.jmh.annotations.Setup -import org.openjdk.jmh.annotations.State -import org.openjdk.jmh.infra.Blackhole +import kotlinx.benchmark.Benchmark +import kotlinx.benchmark.BenchmarkMode +import kotlinx.benchmark.BenchmarkTimeUnit +import kotlinx.benchmark.Blackhole +import kotlinx.benchmark.Mode +import kotlinx.benchmark.OutputTimeUnit +import kotlinx.benchmark.Param +import kotlinx.benchmark.Scope +import kotlinx.benchmark.Setup +import kotlinx.benchmark.State +import kotlinx.io.Buffer import protokt.v1.Bytes import protokt.v1.serialize -import java.io.ByteArrayOutputStream -import java.util.Random -import java.util.concurrent.TimeUnit +import kotlin.random.Random -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) -open class ProtoktBenchmarks : ProtobufBenchmarkSet { +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) +class ProtoktMultiplatformBenchmarks : ProtobufBenchmarkSet { @Param("protokt.v1.DefaultCollectionFactory", "protokt.v1.PersistentCollectionFactory") var collectionFactory: String = "protokt.v1.DefaultCollectionFactory" - @Param("protokt.v1.ProtobufJavaCodec", "protokt.v1.KotlinxIoCodec", "protokt.v1.ProtoktCodec", "protokt.v1.OptimalKmpCodec", "protokt.v1.OptimalJvmCodec") + @Param("protokt.v1.ProtobufJavaCodec", "protokt.v1.KotlinxIoCodec", "protokt.v1.ProtoktCodec") var codec: String = "protokt.v1.ProtobufJavaCodec" private lateinit var largeDataset: BenchmarkDataset @@ -72,11 +70,9 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { @Setup fun setup() { + applyBenchmarkConfig(collectionFactory, codec) byteValues = Array(1000) { i -> Bytes.from(byteArrayOf(i.toByte())) } - System.setProperty("protokt.v1.collection.factory", collectionFactory) - System.setProperty("protokt.v1.codec", codec) - val random = Random(42) stringHeavyPayloads = (0 until 100).map { GenericMessage1 { @@ -122,7 +118,7 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { stringRepeatedPayloads = (0 until 20).map { StringCollectionMessage { - fieldRepeatedString = (0 until 500).map { i -> randomUtf8String(random, 100) } + fieldRepeatedString = (0 until 500).map { randomUtf8String(random, 100) } } }.map { Bytes.from(it.serialize()) } @@ -136,19 +132,13 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { stringMapParsed = stringMapPayloads.map { StringCollectionMessage.deserialize(it) } - readData("large").use { stream -> - largeDataset = BenchmarkDataset.deserialize(stream.readBytes()) - } + largeDataset = BenchmarkDataset.deserialize(readDatasetBytes("large")) largeParsedDataset = largeDataset.payload.map { GenericMessage1.deserialize(it) } - readData("medium").use { stream -> - mediumDataset = BenchmarkDataset.deserialize(stream.readBytes()) - } + mediumDataset = BenchmarkDataset.deserialize(readDatasetBytes("medium")) mediumParsedDataset = mediumDataset.payload.map { GenericMessage1.deserialize(it) } - readData("small").use { stream -> - smallDataset = BenchmarkDataset.deserialize(stream.readBytes()) - } + smallDataset = BenchmarkDataset.deserialize(readDatasetBytes("small")) smallParsedDataset = smallDataset.payload.map { GenericMessage4.deserialize(it) } } @@ -244,37 +234,31 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { @Benchmark override fun serializeLargeStreaming(bh: Blackhole) { - val baos = ByteArrayOutputStream() - val sink = baos.asSink().buffered() + val buffer = Buffer() largeParsedDataset.forEach { msg -> - msg.serialize(sink) - sink.flush() - bh.consume(baos.size()) - baos.reset() + msg.serialize(buffer) + bh.consume(buffer.size) + buffer.clear() } } @Benchmark override fun serializeMediumStreaming(bh: Blackhole) { - val baos = ByteArrayOutputStream() - val sink = baos.asSink().buffered() + val buffer = Buffer() mediumParsedDataset.forEach { msg -> - msg.serialize(sink) - sink.flush() - bh.consume(baos.size()) - baos.reset() + msg.serialize(buffer) + bh.consume(buffer.size) + buffer.clear() } } @Benchmark override fun serializeSmallStreaming(bh: Blackhole) { - val baos = ByteArrayOutputStream() - val sink = baos.asSink().buffered() + val buffer = Buffer() smallParsedDataset.forEach { msg -> - msg.serialize(sink) - sink.flush() - bh.consume(baos.size()) - baos.reset() + msg.serialize(buffer) + bh.consume(buffer.size) + buffer.clear() } } @@ -314,8 +298,7 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { @Benchmark override fun mutateAndSerializeStringHeavyStreaming(bh: Blackhole) { - val baos = ByteArrayOutputStream() - val sink = baos.asSink().buffered() + val buffer = Buffer() stringHeavyPayloads.forEach { bytes -> val msg = GenericMessage1.deserialize(bytes) val mutated = msg.copy { @@ -323,10 +306,9 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { fieldString2 = msg.fieldString2 + "x" fieldString3000 = msg.fieldString3000 + "x" } - mutated.serialize(sink) - sink.flush() - bh.consume(baos.size()) - baos.reset() + mutated.serialize(buffer) + bh.consume(buffer.size) + buffer.clear() } } @@ -337,20 +319,6 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { } } - @Benchmark - override fun deserializeStringHeavyStreaming(bh: Blackhole) { - stringHeavyPayloads.forEach { bytes -> - bh.consume(GenericMessage1.deserialize(bytes.inputStream())) - } - } - - @Benchmark - override fun deserializeStringOneofStreaming(bh: Blackhole) { - stringOneofPayloads.forEach { bytes -> - bh.consume(StringOneofMessage.deserialize(bytes.inputStream())) - } - } - @Benchmark override fun mutateAndSerializeStringOneof(bh: Blackhole) { stringOneofPayloads.forEach { bytes -> @@ -379,8 +347,7 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { @Benchmark override fun mutateAndSerializeStringOneofStreaming(bh: Blackhole) { - val baos = ByteArrayOutputStream() - val sink = baos.asSink().buffered() + val buffer = Buffer() stringOneofPayloads.forEach { bytes -> val msg = StringOneofMessage.deserialize(bytes) val mutated = msg.copy { @@ -394,10 +361,9 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { (msg.content3 as StringOneofMessage.Content3.StringVal3).stringVal3 + "x" ) } - mutated.serialize(sink) - sink.flush() - bh.consume(baos.size()) - baos.reset() + mutated.serialize(buffer) + bh.consume(buffer.size) + buffer.clear() } } @@ -422,8 +388,7 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { @Benchmark override fun mutateAndSerializeStringOneof20kStreaming(bh: Blackhole) { - val baos = ByteArrayOutputStream() - val sink = baos.asSink().buffered() + val buffer = Buffer() stringOneof20kPayloads.forEach { bytes -> val msg = StringOneofMessage.deserialize(bytes) val mutated = msg.copy { @@ -437,10 +402,9 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { (msg.content3 as StringOneofMessage.Content3.StringVal3).stringVal3 + "x" ) } - mutated.serialize(sink) - sink.flush() - bh.consume(baos.size()) - baos.reset() + mutated.serialize(buffer) + bh.consume(buffer.size) + buffer.clear() } } @@ -521,29 +485,4 @@ open class ProtoktBenchmarks : ProtobufBenchmarkSet { } bh.consume(msg) } - - @Benchmark - override fun deserializeLargeStreaming(bh: Blackhole) { - largeDataset.payload.forEach { bytes -> - bh.consume(GenericMessage1.deserialize(bytes.inputStream())) - } - } - - @Benchmark - override fun deserializeMediumStreaming(bh: Blackhole) { - mediumDataset.payload.forEach { bytes -> - bh.consume(GenericMessage1.deserialize(bytes.inputStream())) - } - } - - @Benchmark - override fun deserializeSmallStreaming(bh: Blackhole) { - smallDataset.payload.forEach { bytes -> - bh.consume(GenericMessage4.deserialize(bytes.inputStream())) - } - } -} - -fun main(args: Array) { - run(ProtoktBenchmarks::class, args) } diff --git a/benchmarks/protokt-benchmarks/src/jsMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt b/benchmarks/protokt-benchmarks/src/jsMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt new file mode 100644 index 000000000..2ab9bc2ce --- /dev/null +++ b/benchmarks/protokt-benchmarks/src/jsMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +actual fun applyBenchmarkConfig(collectionFactory: String, codec: String) {} diff --git a/benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt b/benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt new file mode 100644 index 000000000..60d5e37dc --- /dev/null +++ b/benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +actual fun applyBenchmarkConfig(collectionFactory: String, codec: String) { + System.setProperty("protokt.v1.collection.factory", collectionFactory) + System.setProperty("protokt.v1.codec", codec) +} diff --git a/benchmarks/protokt-benchmarks/src/main/kotlin/protokt/v1/benchmarks/FixtureGenerator.kt b/benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/FixtureGenerator.kt similarity index 100% rename from benchmarks/protokt-benchmarks/src/main/kotlin/protokt/v1/benchmarks/FixtureGenerator.kt rename to benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/FixtureGenerator.kt diff --git a/benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/ProtoktJvmBenchmarks.kt b/benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/ProtoktJvmBenchmarks.kt new file mode 100644 index 000000000..be63ae5a9 --- /dev/null +++ b/benchmarks/protokt-benchmarks/src/jvmMain/kotlin/protokt/v1/benchmarks/ProtoktJvmBenchmarks.kt @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protokt.v1.benchmarks + +import kotlinx.benchmark.Benchmark +import kotlinx.benchmark.BenchmarkMode +import kotlinx.benchmark.BenchmarkTimeUnit +import kotlinx.benchmark.Blackhole +import kotlinx.benchmark.Mode +import kotlinx.benchmark.OutputTimeUnit +import kotlinx.benchmark.Param +import kotlinx.benchmark.Scope +import kotlinx.benchmark.Setup +import kotlinx.benchmark.State +import protokt.v1.Bytes +import kotlin.random.Random + +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) +class ProtoktJvmBenchmarks { + @Param("protokt.v1.DefaultCollectionFactory", "protokt.v1.PersistentCollectionFactory") + var collectionFactory: String = "protokt.v1.DefaultCollectionFactory" + + @Param("protokt.v1.ProtobufJavaCodec", "protokt.v1.KotlinxIoCodec", "protokt.v1.ProtoktCodec") + var codec: String = "protokt.v1.ProtobufJavaCodec" + + private lateinit var largeDataset: BenchmarkDataset + private lateinit var mediumDataset: BenchmarkDataset + private lateinit var smallDataset: BenchmarkDataset + private lateinit var stringHeavyPayloads: List + private lateinit var stringOneofPayloads: List + + @Setup + fun setup() { + applyBenchmarkConfig(collectionFactory, codec) + + val random = Random(42) + + stringHeavyPayloads = (0 until 100).map { + GenericMessage1 { + fieldString1 = randomUtf8String(random, 10_000) + fieldString2 = randomUtf8String(random, 10_000) + fieldString3000 = randomUtf8String(random, 10_000) + } + }.map { Bytes.from(it.serialize()) } + + stringOneofPayloads = (0 until 100).map { + StringOneofMessage { + content1 = StringOneofMessage.Content1.StringVal1(randomUtf8String(random, 10_000)) + content2 = StringOneofMessage.Content2.StringVal2(randomUtf8String(random, 10_000)) + content3 = StringOneofMessage.Content3.StringVal3(randomUtf8String(random, 10_000)) + } + }.map { Bytes.from(it.serialize()) } + + largeDataset = BenchmarkDataset.deserialize(readDatasetBytes("large")) + mediumDataset = BenchmarkDataset.deserialize(readDatasetBytes("medium")) + smallDataset = BenchmarkDataset.deserialize(readDatasetBytes("small")) + } + + @Benchmark + fun deserializeLargeStreaming(bh: Blackhole) { + largeDataset.payload.forEach { bytes -> + bh.consume(GenericMessage1.deserialize(bytes.inputStream())) + } + } + + @Benchmark + fun deserializeMediumStreaming(bh: Blackhole) { + mediumDataset.payload.forEach { bytes -> + bh.consume(GenericMessage1.deserialize(bytes.inputStream())) + } + } + + @Benchmark + fun deserializeSmallStreaming(bh: Blackhole) { + smallDataset.payload.forEach { bytes -> + bh.consume(GenericMessage4.deserialize(bytes.inputStream())) + } + } + + @Benchmark + fun deserializeStringHeavyStreaming(bh: Blackhole) { + stringHeavyPayloads.forEach { bytes -> + bh.consume(GenericMessage1.deserialize(bytes.inputStream())) + } + } + + @Benchmark + fun deserializeStringOneofStreaming(bh: Blackhole) { + stringOneofPayloads.forEach { bytes -> + bh.consume(StringOneofMessage.deserialize(bytes.inputStream())) + } + } +} diff --git a/benchmarks/protokt-benchmarks/src/nativeMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt b/benchmarks/protokt-benchmarks/src/nativeMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt new file mode 100644 index 000000000..5691204b6 --- /dev/null +++ b/benchmarks/protokt-benchmarks/src/nativeMain/kotlin/protokt/v1/benchmarks/BenchmarkConfig.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:OptIn(protokt.v1.OnlyForUseByGeneratedProtoCode::class) + +package protokt.v1.benchmarks + +import protokt.v1.PersistentCollectionFactory +import protokt.v1.collectionFactoryOverride + +actual fun applyBenchmarkConfig(collectionFactory: String, codec: String) { + collectionFactoryOverride = + if (collectionFactory == "protokt.v1.PersistentCollectionFactory") { + PersistentCollectionFactory + } else { + null + } +} diff --git a/benchmarks/wire-benchmarks/build.gradle.kts b/benchmarks/wire-benchmarks/build.gradle.kts index 545f88b43..ca1e195aa 100644 --- a/benchmarks/wire-benchmarks/build.gradle.kts +++ b/benchmarks/wire-benchmarks/build.gradle.kts @@ -15,13 +15,13 @@ plugins { id("protokt.benchmarks-conventions") - application alias(libs.plugins.wire) } -application { - mainClass.set("protokt.v1.benchmarks.WireBenchmarksKt") - executableDir = ".." +benchmark { + targets { + register("main") + } } dependencies { @@ -43,6 +43,6 @@ sourceSets { } } -tasks.named("run") { +tasks.matching { it.name.contains("benchmark", ignoreCase = true) }.configureEach { dependsOn(":benchmarks:datasets") } diff --git a/benchmarks/wire-benchmarks/src/main/kotlin/protokt/v1/benchmarks/WireBenchmarks.kt b/benchmarks/wire-benchmarks/src/main/kotlin/protokt/v1/benchmarks/WireBenchmarks.kt index c6a2d5375..4e378a6ac 100644 --- a/benchmarks/wire-benchmarks/src/main/kotlin/protokt/v1/benchmarks/WireBenchmarks.kt +++ b/benchmarks/wire-benchmarks/src/main/kotlin/protokt/v1/benchmarks/WireBenchmarks.kt @@ -20,28 +20,28 @@ import com.toasttab.protokt.v1.benchmarks.GenericMessage1 import com.toasttab.protokt.v1.benchmarks.GenericMessage4 import com.toasttab.protokt.v1.benchmarks.StringCollectionMessage import com.toasttab.protokt.v1.benchmarks.StringOneofMessage +import kotlinx.benchmark.Benchmark +import kotlinx.benchmark.BenchmarkMode +import kotlinx.benchmark.BenchmarkTimeUnit +import kotlinx.benchmark.Blackhole +import kotlinx.benchmark.Mode +import kotlinx.benchmark.OutputTimeUnit +import kotlinx.benchmark.Scope +import kotlinx.benchmark.Setup +import kotlinx.benchmark.State import okio.ByteString import okio.ByteString.Companion.toByteString import okio.buffer import okio.sink import okio.source -import org.openjdk.jmh.annotations.Benchmark -import org.openjdk.jmh.annotations.BenchmarkMode -import org.openjdk.jmh.annotations.Mode -import org.openjdk.jmh.annotations.OutputTimeUnit -import org.openjdk.jmh.annotations.Scope -import org.openjdk.jmh.annotations.Setup -import org.openjdk.jmh.annotations.State -import org.openjdk.jmh.infra.Blackhole import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream -import java.util.Random -import java.util.concurrent.TimeUnit +import kotlin.random.Random @BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MILLISECONDS) +@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) @State(Scope.Benchmark) -open class WireBenchmarks : ProtobufBenchmarkSet { +class WireBenchmarks : JvmProtobufBenchmarkSet { private lateinit var largeDataset: BenchmarkDataset private lateinit var largeParsedDataset: List private lateinit var mediumDataset: BenchmarkDataset @@ -505,7 +505,3 @@ open class WireBenchmarks : ProtobufBenchmarkSet { } } } - -fun main(args: Array) { - run(WireBenchmarks::class, args) -} diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index cbea751ae..53dd24761 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -33,14 +33,22 @@ repositories { dependencies { implementation(libs.androidGradlePlugin) - implementation(libs.expediter) implementation(libs.binaryCompatibilityValidator) + implementation(libs.expediter) implementation(libs.gradleMavenPublishPlugin) implementation(libs.kotlinGradlePlugin) + implementation("org.jetbrains.kotlin:kotlin-allopen:${libs.versions.kotlin.get()}") + implementation(libs.kotlinx.benchmark.plugin) implementation(libs.protobuf.gradlePlugin) implementation(libs.spotlessGradlePlugin) implementation(kotlin("gradle-plugin-api")) implementation(files(libs.javaClass.superclass.protectionDomain.codeSource.location)) + + // kotlinx-benchmark-plugin pulls an older KotlinPoet that's binary-incompatible + // with the version used by protokt-codegen; force alignment + constraints { + implementation(libs.kotlinPoet) + } } sourceSets { diff --git a/buildSrc/src/main/kotlin/BenchmarkConfiguration.kt b/buildSrc/src/main/kotlin/BenchmarkConfiguration.kt new file mode 100644 index 000000000..a87373921 --- /dev/null +++ b/buildSrc/src/main/kotlin/BenchmarkConfiguration.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026 Toast, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import kotlinx.benchmark.gradle.BenchmarksExtension +import org.gradle.api.Project +import org.gradle.kotlin.dsl.configure + +fun Project.configureBenchmarks() { + configure { + configurations.named("main").configure { + val wi = findProperty("benchmarkWarmups")?.toString()?.toInt() ?: 3 + val mi = findProperty("benchmarkIterations")?.toString()?.toInt() ?: 5 + val f = findProperty("benchmarkForks")?.toString()?.toInt() ?: 2 + + warmups = wi + iterations = mi + iterationTime = 10 + iterationTimeUnit = "s" + mode = "avgt" + outputTimeUnit = "ms" + advanced("jvmForks", f) + advanced("jmhIgnoreLock", true) + reportFormat = "json" + + findProperty("benchmarkInclude")?.toString()?.let { include(it) } + findProperty("benchmarkExclude")?.toString()?.let { exclude(it) } + findProperty("benchmarkParam")?.toString()?.split(",")?.forEach { + val (name, value) = it.split("=", limit = 2) + param(name, value) + } + } + } +} diff --git a/buildSrc/src/main/kotlin/RuntimeFriendPaths.kt b/buildSrc/src/main/kotlin/RuntimeFriendPaths.kt index 950f19fbd..06fbcb379 100644 --- a/buildSrc/src/main/kotlin/RuntimeFriendPaths.kt +++ b/buildSrc/src/main/kotlin/RuntimeFriendPaths.kt @@ -16,6 +16,7 @@ import org.gradle.api.Project import org.gradle.kotlin.dsl.configure import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension +import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation import org.jetbrains.kotlin.gradle.tasks.BaseKotlinCompile import org.jetbrains.kotlin.gradle.tasks.KotlinNativeCompile import java.io.File @@ -33,46 +34,46 @@ import java.io.File // 2. -friend-modules only honours the last occurrence when passed multiple times, // so all paths must be joined into a single argument. fun Project.runtimeFriendPaths() { + friendPaths(":protokt-runtime") + configure { compilerOptions { freeCompilerArgs.add("-opt-in=protokt.v1.OnlyForUseByGeneratedProtoCode") } + } +} +fun Project.friendPaths(vararg friendProjectPaths: String) { + configure { targets.all { compilations.all { val compilation = this + val friendCompilations = resolveFriendCompilations(compilation, friendProjectPaths) + if (friendCompilations.isEmpty()) return@all + compileTaskProvider.configure { - val runtimeProject = project(":protokt-runtime") - val runtimeKmp = runtimeProject - .extensions - .getByType(KotlinMultiplatformExtension::class.java) - val runtimeCompilation = runtimeKmp - .targets - .getByName(compilation.target.name) - .compilations - .getByName(compilation.compilationName) when (this) { is BaseKotlinCompile -> { - friendPaths.from( - runtimeCompilation.output.classesDirs, - runtimeCompilation.output.allOutputs - ) - val jarTaskName = "${compilation.target.name}Jar" - if (jarTaskName in runtimeProject.tasks.names) { - friendPaths.from(runtimeProject.tasks.named(jarTaskName)) + for ((friendProject, _, friendCompilation) in friendCompilations) { + friendPaths.from(friendCompilation.output.classesDirs, friendCompilation.output.allOutputs) + val jarTaskName = "${compilation.target.name}Jar" + if (jarTaskName in friendProject.tasks.names) { + friendPaths.from(friendProject.tasks.named(jarTaskName)) + } } } is KotlinNativeCompile -> { - val friendDirs = + val allDirs = friendCompilations.flatMap { (_, friendKmp, friendCompilation) -> if (compilation.target.name == "metadata") { - runtimeKmp.targets.getByName("metadata") + friendKmp.targets.getByName("metadata") .compilations .flatMap { it.output.classesDirs } } else { - runtimeCompilation.output.classesDirs.toList() + friendCompilation.output.classesDirs.toList() } - val joined = friendDirs.joinToString(File.pathSeparator) { it.absolutePath } + } + val joined = allDirs.joinToString(File.pathSeparator) { it.absolutePath } compilerOptions.freeCompilerArgs.add("-friend-modules=$joined") } } @@ -81,3 +82,24 @@ fun Project.runtimeFriendPaths() { } } } + +private fun Project.resolveFriendCompilations( + compilation: KotlinCompilation<*>, + friendProjectPaths: Array +) = + friendProjectPaths.mapNotNull { path -> + val friendProject = project(path) + val friendKmp = friendProject + .extensions + .getByType(KotlinMultiplatformExtension::class.java) + val friendTarget = friendKmp + .targets + .findByName(compilation.target.name) ?: return@mapNotNull null + val name = + if (friendTarget.compilations.findByName(compilation.compilationName) != null) { + compilation.compilationName + } else { + KotlinCompilation.MAIN_COMPILATION_NAME + } + Triple(friendProject, friendKmp, friendTarget.compilations.getByName(name)) + } diff --git a/buildSrc/src/main/kotlin/protokt.benchmarks-conventions.gradle.kts b/buildSrc/src/main/kotlin/protokt.benchmarks-conventions.gradle.kts index 40f0513f2..a46d679e7 100644 --- a/buildSrc/src/main/kotlin/protokt.benchmarks-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/protokt.benchmarks-conventions.gradle.kts @@ -15,10 +15,17 @@ plugins { id("protokt.jvm-conventions") - `kotlin-kapt` + id("org.jetbrains.kotlinx.benchmark") + id("org.jetbrains.kotlin.plugin.allopen") +} + +allOpen { + annotation("org.openjdk.jmh.annotations.State") + annotation("kotlinx.benchmark.State") } dependencies { - implementation(libs.jmh.core) - kapt(libs.jmh.generator) + implementation(libs.kotlinx.benchmark.runtime) } + +configureBenchmarks() diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 70eb1061c..16a695fd1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -45,6 +45,7 @@ spotless = "8.4.0" datasets = "0.1.0" download = "5.7.0" jmh = "1.37" +kotlinx-benchmark = "0.4.16" wire = "6.2.0" # test @@ -63,6 +64,7 @@ protoGoogleCommonProtos = "2.69.0" [plugins] buildConfig = { id = "com.github.gmazzo.buildconfig", version.ref = "buildConfig" } download = { id = "de.undercouch.download", version.ref = "download" } +kotlinx-benchmark = { id = "org.jetbrains.kotlinx.benchmark", version.ref = "kotlinx-benchmark" } pluginPublish = { id = "com.gradle.plugin-publish", version.ref = "pluginPublish" } wire = { id = "com.squareup.wire", version.ref = "wire" } @@ -97,6 +99,8 @@ spotlessGradlePlugin = { module = "com.diffplug.spotless:spotless-plugin-gradle" # benchmarks jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } jmh-generator = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" } +kotlinx-benchmark-plugin = { module = "org.jetbrains.kotlinx:kotlinx-benchmark-plugin", version.ref = "kotlinx-benchmark" } +kotlinx-benchmark-runtime = { module = "org.jetbrains.kotlinx:kotlinx-benchmark-runtime", version.ref = "kotlinx-benchmark" } wireRuntime = { module = "com.squareup.wire:wire-runtime", version.ref = "wire" } # third party