Migrate benchmarks to kotlinx-benchmark for JVM+Native

benchmarks/README.md

@@ -4,63 +4,60 @@ See [RESULTS.md](RESULTS.md) for detailed benchmark results and analysis.
 
 ## Running
 
-Run all benchmarks for a given implementation:
+All benchmarks use [kotlinx-benchmark](https://github.com/Kotlin/kotlinx-benchmark),
+which delegates to JMH on JVM and uses its own runtime on Kotlin/Native.
 
 ```
-./gradlew :benchmarks:protokt-benchmarks:run
-./gradlew :benchmarks:protobuf-java-benchmarks:run
-./gradlew :benchmarks:wire-benchmarks:run
+./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark
+./gradlew :benchmarks:protokt-benchmarks:macosArm64Benchmark
+./gradlew :benchmarks:protobuf-java-benchmarks:benchmark
+./gradlew :benchmarks:wire-benchmarks:benchmark
 ```
 
-## Flags
+All modules share the same default configuration (3 warmup iterations,
+5 measurement iterations, 10s each, 2 forks, average time in ms/op).
 
-Flags are passed via `--args`:
+## Gradle properties
 
-| Flag | Description |
-|------|-------------|
-| `-i regex` | Include only benchmarks matching regex |
-| `-e regex` | Exclude benchmarks matching regex |
-| `-p name=value` | Set a JMH parameter |
+Override defaults at invocation time via `-P`:
 
-When no `-i` flag is given, all benchmarks in the class are included.
-
-## Parameters
-
-protokt benchmarks accept these JMH parameters (set via `-p`):
-
-| Parameter | Values | Default |
-|-----------|--------|---------|
-| `collectionFactory` | `protokt.v1.DefaultCollectionFactory`, `protokt.v1.PersistentCollectionFactory` | Both |
-| `codec` | `protokt.v1.ProtobufJavaCodec`, `protokt.v1.KotlinxIoCodec`, `protokt.v1.ProtoktCodec`, `protokt.v1.OptimalKmpCodec`, `protokt.v1.OptimalJvmCodec` | All |
+| Property | Description | Default |
+|----------|-------------|---------|
+| `benchmarkInclude` | Regex to include matching benchmarks | all |
+| `benchmarkExclude` | Regex to exclude matching benchmarks | none |
+| `benchmarkParam` | Comma-separated `name=value` pairs for `@Param` fields | all values |
+| `benchmarkWarmups` | Warmup iterations | 3 |
+| `benchmarkIterations` | Measurement iterations | 5 |
+| `benchmarkForks` | JVM forks | 2 |
 
 ## Examples
 
-Run a single benchmark method:
+Run only serialization benchmarks:
 
 ```
-./gradlew :benchmarks:protokt-benchmarks:run --args="-i serializeSmall"
+./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark -PbenchmarkInclude=.*serialize.*
 ```
 
-Exclude copy/append benchmarks:
+Run with a specific codec:
 
 ```
-./gradlew :benchmarks:protokt-benchmarks:run --args="-e .*copyAppend.*"
+./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark -PbenchmarkParam=codec=protokt.v1.ProtoktCodec
 ```
 
-Pin a JMH parameter:
+Quick smoke test (1 warmup, 1 iteration, 1 fork):
 
 ```
-./gradlew :benchmarks:protokt-benchmarks:run --args="-p collectionFactory=protokt.v1.DefaultCollectionFactory"
+./gradlew :benchmarks:protokt-benchmarks:jvmBenchmark -PbenchmarkWarmups=1 -PbenchmarkIterations=1 -PbenchmarkForks=1
 ```
 
-Run with a specific codec:
+## Parameters
 
-```
-./gradlew :benchmarks:protokt-benchmarks:run --args="-p codec=protokt.v1.ProtoktCodec"
-```
+protokt benchmarks accept `@Param`-annotated properties:
 
-Combine flags:
+| Parameter | Targets | Values |
+|-----------|---------|--------|
+| `collectionFactory` | JVM + Native | `protokt.v1.DefaultCollectionFactory`, `protokt.v1.PersistentCollectionFactory` |
+| `codec` | JVM only | `protokt.v1.ProtobufJavaCodec`, `protokt.v1.KotlinxIoCodec`, `protokt.v1.ProtoktCodec`, `protokt.v1.OptimalKmpCodec`, `protokt.v1.OptimalJvmCodec` |
 
-```
-./gradlew :benchmarks:protokt-benchmarks:run --args="-i serialize -e .*String.*"
-```
+Native benchmarks always use `ProtoktCodec`. Collection factory selection works
+on native via direct override (no env var or system property needed).

benchmarks/benchmarks-util/build.gradle.kts

@@ -14,5 +14,7 @@
  */
 
 plugins {
-    id("protokt.benchmarks-conventions")
+    id("protokt.multiplatform-conventions")
 }
+
+enableNativeTargets()

benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/BenchmarkUtils.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 Toast, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package protokt.v1.benchmarks
+
+import kotlin.random.Random
+
+expect fun readDatasetBytes(name: String): ByteArray
+
+/** Mix of 1-byte (ASCII), 2-byte (Latin Extended), and 3-byte (CJK) UTF-8 characters. */
+fun randomUtf8String(random: Random, charCount: Int): String {
+    val sb = StringBuilder(charCount)
+    repeat(charCount) {
+        sb.append(
+            when (random.nextInt(3)) {
+                0 -> 'a' + random.nextInt(26)
+                1 -> (0x00C0 + random.nextInt(64)).toChar()
+                else -> (0x4E00 + random.nextInt(0x5000)).toChar()
+            }
+        )
+    }
+    return sb.toString()
+}

benchmarks/benchmarks-util/src/commonMain/kotlin/protokt/v1/benchmarks/ProtobufBenchmarkSet.kt

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2022 Toast, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package protokt.v1.benchmarks
+
+interface ProtobufBenchmarkSet<T> {
+    fun deserializeLargeFromMemory(bh: T)
+    fun deserializeMediumFromMemory(bh: T)
+    fun deserializeSmallFromMemory(bh: T)
+    fun serializeLargeToMemory(bh: T)
+    fun serializeMediumToMemory(bh: T)
+    fun serializeSmallToMemory(bh: T)
+    fun serializeLargeStreaming(bh: T)
+    fun serializeMediumStreaming(bh: T)
+    fun serializeSmallStreaming(bh: T)
+    fun copyAppendListLarge(bh: T)
+    fun copyAppendMapLarge(bh: T)
+    fun copyAppendListMedium(bh: T)
+    fun copyAppendMapMedium(bh: T)
+    fun copyAppendListSmall(bh: T)
+    fun copyAppendMapSmall(bh: T)
+    fun passThroughLargeFromMemory(bh: T)
+    fun passThroughMediumFromMemory(bh: T)
+    fun passThroughSmallFromMemory(bh: T)
+    fun mutateAndSerializeStringHeavy(bh: T)
+    fun mutateAndSerializeStringHeavyStreaming(bh: T)
+    fun passThroughStringHeavy(bh: T)
+    fun mutateAndSerializeStringOneof(bh: T)
+    fun passThroughStringOneof(bh: T)
+    fun mutateAndSerializeStringOneofStreaming(bh: T)
+    fun mutateAndSerializeStringOneof20k(bh: T)
+    fun mutateAndSerializeStringOneof20kStreaming(bh: T)
+    fun mutateAndSerializeStringOneofVeryHeavy(bh: T)
+    fun mutateAndSerializeStringVeryHeavy(bh: T)
+    fun passThroughStringRepeated(bh: T)
+    fun passThroughStringMap(bh: T)
+    fun deserializeStringRepeated(bh: T)
+    fun deserializeStringMap(bh: T)
+    fun copyAppendRepeatedString(bh: T)
+    fun copyAppendMapStringString(bh: T)
+}

benchmarks/benchmarks-util/src/jsMain/kotlin/protokt/v1/benchmarks/DatasetReader.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2026 Toast, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package protokt.v1.benchmarks
+
+actual fun readDatasetBytes(name: String): ByteArray =
+    error("Benchmarks not supported on JS")

benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmBenchmarkUtils.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2019 Toast, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package protokt.v1.benchmarks
+
+import java.io.File
+
+fun readData(dataset: String) =
+    File("../build/datasets/dataset-$dataset").inputStream().buffered()
+
+actual fun readDatasetBytes(name: String): ByteArray =
+    File("../build/datasets/dataset-$name").readBytes()

benchmarks/benchmarks-util/src/jvmMain/kotlin/protokt/v1/benchmarks/JvmProtobufBenchmarkSet.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2026 Toast, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package protokt.v1.benchmarks
+
+interface JvmProtobufBenchmarkSet<T> : ProtobufBenchmarkSet<T> {
+    fun deserializeStringHeavyStreaming(bh: T)
+    fun deserializeStringOneofStreaming(bh: T)
+    fun deserializeLargeStreaming(bh: T)
+    fun deserializeMediumStreaming(bh: T)
+    fun deserializeSmallStreaming(bh: T)
+}