Skip to content

Bench/remove iterations #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Antonov548 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Bench/remove iterations #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
72f1c6e
remove iterations from benchmarks
Antonov548 Feb 19, 2026
5a2bacf
fix formatting
Antonov548 Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 13 additions & 40 deletions packages/treecrdt-benchmark/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import type { TreecrdtAdapter, SerializeNodeId, SerializeReplica, Operation } from "@treecrdt/interface";
import { nodeIdToBytes16, replicaIdToBytes } from "@treecrdt/interface/ids";
import { envInt, quantile } from "./stats.js";
import type { WorkloadName } from "./workloads.js";

export type BenchmarkResult = {
Expand All @@ -14,8 +13,6 @@ export type BenchmarkResult = {
export type BenchmarkWorkload = {
name: string;
totalOps?: number;
iterations?: number;
warmupIterations?: number;
prepare?: () => Promise<void> | void;
run: (adapter: TreecrdtAdapter) => Promise<void | { extra?: Record<string, unknown> }>;
cleanup?: () => Promise<void> | void;
Expand Down Expand Up @@ -47,33 +44,21 @@ export async function runBenchmark(
): Promise<BenchmarkResult> {
const totalOps = workload.totalOps ?? -1;

const envIterations = envInt("BENCH_ITERATIONS");
const envWarmup = envInt("BENCH_WARMUP");
const rawIterations = Math.max(1, workload.iterations ?? envIterations ?? 1);
const minIterationsForTiny =
totalOps >= 1 && totalOps <= 100 ? 10 : totalOps <= 1000 ? 7 : 1;
const iterations = Math.max(minIterationsForTiny, rawIterations);
const warmupIterations = Math.max(0, workload.warmupIterations ?? envWarmup ?? (iterations > 1 ? 1 : 0));

const samplesMs: number[] = [];
const adapter = await adapterFactory();
let durationMs: number;
let lastExtra: Record<string, unknown> | undefined;

for (let i = 0; i < warmupIterations + iterations; i += 1) {
const adapter = await adapterFactory();
try {
if (workload.prepare) await workload.prepare();
const start = performance.now();
const runResult = await workload.run(adapter);
const end = performance.now();
if (runResult && typeof runResult === "object" && runResult.extra) lastExtra = runResult.extra;
if (workload.cleanup) await workload.cleanup();
if (i >= warmupIterations) samplesMs.push(end - start);
} finally {
if (adapter.close) await adapter.close();
}
try {
if (workload.prepare) await workload.prepare();
const start = performance.now();
const runResult = await workload.run(adapter);
const end = performance.now();
durationMs = end - start;
if (runResult && typeof runResult === "object" && runResult.extra) lastExtra = runResult.extra;
if (workload.cleanup) await workload.cleanup();
} finally {
if (adapter.close) await adapter.close();
}

const durationMs = quantile(samplesMs, 0.5);
const opsPerSec =
totalOps > 0 && durationMs > 0
? (totalOps / durationMs) * 1000
Expand All @@ -85,18 +70,7 @@ export async function runBenchmark(
totalOps,
durationMs,
opsPerSec,
extra:
lastExtra || samplesMs.length > 1
? {
...(lastExtra ?? {}),
iterations,
warmupIterations,
samplesMs,
p95Ms: quantile(samplesMs, 0.95),
minMs: Math.min(...samplesMs),
maxMs: Math.max(...samplesMs),
}
: undefined,
extra: lastExtra ?? undefined,
};
}

Expand Down Expand Up @@ -278,6 +252,5 @@ export async function runWorkloads(
}

export { DEFAULT_BENCH_SIZES, WORKLOAD_NAMES, type WorkloadName } from "./workloads.js";
export { benchTiming } from "./timing.js";
export * from "./sync.js";
export * from "./stats.js";
9 changes: 0 additions & 9 deletions packages/treecrdt-benchmark/src/sync.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import type { Operation, OperationKind, ReplicaId } from "@treecrdt/interface";
import { nodeIdToBytes16 } from "@treecrdt/interface/ids";
import { envIntList } from "./stats.js";
import { benchTiming } from "./timing.js";

export type SyncBenchWorkload =
| "sync-all"
Expand All @@ -27,14 +26,6 @@ export function syncBenchRootChildrenSizesFromEnv(): number[] {
return envIntList("SYNC_BENCH_ROOT_CHILDREN_SIZES") ?? Array.from(DEFAULT_SYNC_BENCH_ROOT_CHILDREN_SIZES);
}

export function syncBenchTiming(opts: { defaultIterations?: number } = {}): { iterations: number; warmupIterations: number } {
return benchTiming({
iterationsEnv: ["SYNC_BENCH_ITERATIONS", "BENCH_ITERATIONS"],
warmupEnv: ["SYNC_BENCH_WARMUP", "BENCH_WARMUP"],
defaultIterations: opts.defaultIterations ?? 10,
});
}

export type SyncFilter =
| { all: Record<string, never> }
| { children: { parent: Uint8Array } };
Expand Down
25 changes: 0 additions & 25 deletions packages/treecrdt-benchmark/src/timing.ts
View file
Open in desktop

This file was deleted.

36 changes: 8 additions & 28 deletions packages/treecrdt-core/benches/core.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::time::Instant;

use treecrdt_core::{Lamport, LamportClock, MemoryStorage, NodeId, ReplicaId, TreeCrdt};

const BENCH_CONFIG: &[(u64, u64)] = &[(100, 10), (1_000, 10), (10_000, 10)];
const BENCH_COUNTS: &[u64] = &[100, 1_000, 10_000];

#[derive(serde::Serialize)]
#[serde(rename_all = "camelCase")]
Expand All @@ -27,10 +27,6 @@ struct Output {
struct Extra {
count: u64,
mode: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
iterations: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
avg_duration_ms: Option<f64>,
}

fn hex_id(n: u64) -> NodeId {
Expand Down Expand Up @@ -69,42 +65,28 @@ fn run_benchmark(replica: &ReplicaId, count: u64) -> f64 {

fn main() {
let mut out_dir: Option<PathBuf> = None;
let mut custom_config: Option<Vec<(u64, u64)>> = None;
let mut custom_counts: Option<Vec<u64>> = None;
for arg in env::args().skip(1) {
if let Some(val) = arg.strip_prefix("--count=") {
let count = val.parse().unwrap_or(500);
custom_config = Some(vec![(count, 1)]);
custom_counts = Some(vec![count]);
} else if let Some(val) = arg.strip_prefix("--counts=") {
let parsed: Vec<(u64, u64)> = val
.split(',')
.filter_map(|s| s.trim().parse::<u64>().ok())
.map(|c| (c, 1))
.collect();
let parsed: Vec<u64> = val.split(',').filter_map(|s| s.trim().parse().ok()).collect();
if !parsed.is_empty() {
custom_config = Some(parsed);
custom_counts = Some(parsed);
}
} else if let Some(val) = arg.strip_prefix("--out-dir=") {
out_dir = Some(PathBuf::from(val));
}
}

let config = custom_config.as_deref().unwrap_or(BENCH_CONFIG);
let counts = custom_counts.as_deref().unwrap_or(BENCH_COUNTS);
let out_dir = out_dir.unwrap_or_else(default_out_dir);
fs::create_dir_all(&out_dir).expect("mkdirs");

let replica = ReplicaId::new(b"core");
for &(count, iterations) in config {
let (duration_ms, iterations_opt, avg_duration_ms) = if iterations > 1 {
let mut durations: Vec<f64> =
(0..iterations).map(|_| run_benchmark(&replica, count)).collect();
durations.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let median = durations[durations.len() / 2];
(median, Some(iterations), Some(median))
} else {
// Single run
let duration = run_benchmark(&replica, count);
(duration, None, None)
};
for &count in counts {
let duration_ms = run_benchmark(&replica, count);

let workload_name = format!("insert-move-{}", count);
let out_path = out_dir.join(format!("memory-{}.json", workload_name));
Expand All @@ -125,8 +107,6 @@ fn main() {
extra: Extra {
count,
mode: "sequential",
iterations: iterations_opt,
avg_duration_ms,
},
source_file: Some(out_path.display().to_string()),
};
Expand Down
74 changes: 17 additions & 57 deletions packages/treecrdt-sqlite-node/scripts/bench-sync.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import {
SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE,
SYNC_BENCH_DEFAULT_MAX_CODEWORDS,
maxLamport,
quantile,
type SyncBenchWorkload,
} from "@treecrdt/benchmark";
import { repoRootFromImportMeta, writeResult } from "@treecrdt/benchmark/node";
Expand All @@ -28,58 +27,33 @@ import {
} from "../dist/index.js";

type StorageKind = "memory" | "file";
type ConfigEntry = [number, number];

const SYNC_BENCH_CONFIG: ReadonlyArray<ConfigEntry> = [
[100, 10],
[1_000, 5],
[10_000, 10],
];
const SYNC_BENCH_COUNTS: readonly number[] = [100, 1_000, 10_000];
const SYNC_BENCH_ROOT_COUNTS: readonly number[] = [1110];

const SYNC_BENCH_ROOT_CONFIG: ReadonlyArray<ConfigEntry> = [[1110, 10]];

function envInt(name: string): number | undefined {
const raw = process.env[name];
if (raw == null || raw === "") return undefined;
const n = Number(raw);
return Number.isFinite(n) ? n : undefined;
}

function parseConfigFromArgv(argv: string[]): Array<ConfigEntry> | null {
let customConfig: Array<ConfigEntry> | null = null;
const defaultIterations = Math.max(1, envInt("BENCH_ITERATIONS") ?? 1);
function parseCountsFromArgv(argv: string[]): number[] | null {
for (const arg of argv) {
if (arg.startsWith("--count=")) {
const val = arg.slice("--count=".length).trim();
const count = val ? Number(val) : 500;
customConfig = [[Number.isFinite(count) && count > 0 ? count : 500, defaultIterations]];
break;
return [Number.isFinite(count) && count > 0 ? count : 500];
}
if (arg.startsWith("--counts=")) {
const vals = arg
const nums = arg
.slice("--counts=".length)
.split(",")
.map((s) => s.trim())
.filter((s) => s.length > 0);
const parsed = vals
.map((s) => {
const n = Number(s);
return Number.isFinite(n) && n > 0 ? n : null;
})
.filter((n): n is number => n != null)
.map((c) => [c, defaultIterations] as ConfigEntry);
if (parsed.length > 0) customConfig = parsed;
break;
.map((s) => Number(s.trim()))
.filter((n) => Number.isFinite(n) && n > 0);
if (nums.length > 0) return nums;
}
}
return customConfig;
return null;
}

type BenchCase = {
storage: StorageKind;
workload: SyncBenchWorkload;
size: number;
iterations: number;
};

type SyncBenchResult = {
Expand Down Expand Up @@ -211,15 +185,7 @@ async function runBenchCase(
benchCase: BenchCase
): Promise<SyncBenchResult> {
const bench = buildSyncBenchCase({ workload: benchCase.workload, size: benchCase.size });
const { size, iterations } = benchCase;

const samplesMs: number[] = [];
for (let i = 0; i < iterations; i += 1) {
samplesMs.push(await runBenchOnce(repoRoot, benchCase, bench));
}

const durationMs =
iterations > 1 ? quantile(samplesMs, 0.5) : samplesMs[0] ?? 0;
const durationMs = await runBenchOnce(repoRoot, benchCase, bench);
const opsPerSec = durationMs > 0 ? (bench.totalOps / durationMs) * 1000 : Infinity;

return {
Expand All @@ -229,15 +195,9 @@ async function runBenchCase(
opsPerSec,
extra: {
...bench.extra,
count: size,
count: benchCase.size,
codewordsPerMessage: SYNC_BENCH_DEFAULT_CODEWORDS_PER_MESSAGE,
maxCodewords: SYNC_BENCH_DEFAULT_MAX_CODEWORDS,
iterations: iterations > 1 ? iterations : undefined,
avgDurationMs: iterations > 1 ? durationMs : undefined,
samplesMs,
p95Ms: quantile(samplesMs, 0.95),
minMs: Math.min(...samplesMs),
maxMs: Math.max(...samplesMs),
},
};
}
Expand All @@ -246,19 +206,19 @@ async function main() {
const argv = process.argv.slice(2);
const repoRoot = repoRootFromImportMeta(import.meta.url, 3);

const config = parseConfigFromArgv(argv) ?? [...SYNC_BENCH_CONFIG];
const rootConfig = [...SYNC_BENCH_ROOT_CONFIG];
const counts = parseCountsFromArgv(argv) ?? [...SYNC_BENCH_COUNTS];
const rootCounts = [...SYNC_BENCH_ROOT_COUNTS];

const cases: BenchCase[] = [];
for (const storage of ["memory", "file"] as const) {
for (const workload of DEFAULT_SYNC_BENCH_WORKLOADS) {
for (const [size, iterations] of config) {
cases.push({ storage, workload, size, iterations });
for (const size of counts) {
cases.push({ storage, workload, size });
}
}
for (const workload of DEFAULT_SYNC_BENCH_ROOT_CHILDREN_WORKLOADS) {
for (const [size, iterations] of rootConfig) {
cases.push({ storage, workload, size, iterations });
for (const size of rootCounts) {
cases.push({ storage, workload, size });
}
}
}
Expand Down
Loading