From 3ab870d6ed495fcefb452ffed3f44eec6fb89b38 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 12:23:19 +0100 Subject: [PATCH 01/13] Add Basic Blocking Locust perf test Adds pi_basic_blocking_locust, which measures the same blocking-write workload as pi_basic_blocking but drives it with locust rather than piccolo, so the number of concurrent clients can be varied. The load is defined in tests/infra/basicperf_locustfile.py and uses FastHttpUser, since HttpUser cannot drive enough requests per second to saturate the service. tests/basicperf_locust.py owns the network, runs locust against it, and converts locust statistics into bencher metrics (throughput, latency, memory). The key space helper shared with basicperf.py moves to tests/infra/key_space.py, since basicperf.py can only be imported from tests/infra. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- CMakeLists.txt | 20 +++ tests/basicperf_locust.py | 206 ++++++++++++++++++++++++++++ tests/infra/basicperf.py | 22 +-- tests/infra/basicperf_locustfile.py | 95 +++++++++++++ tests/infra/key_space.py | 31 +++++ 5 files changed, 356 insertions(+), 18 deletions(-) create mode 100644 tests/basicperf_locust.py create mode 100644 tests/infra/basicperf_locustfile.py create mode 100644 tests/infra/key_space.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 07e6dfa7f1d9..af6f4bfda90a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1482,6 +1482,26 @@ if(BUILD_TESTS) "1,write,300000,primary" ) + # Same workload as pi_basic_blocking, driven by locust rather than piccolo, + # so that the number of concurrent clients can be varied. + add_e2e_test( + NAME pi_basic_blocking_locust + PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/basicperf_locust.py + LABEL perf + CONFIGURATIONS perf + ADDITIONAL_ARGS + --package + "samples/apps/basic/basic" + --perf-label + "Basic Blocking Locust" + --users + 128 + --spawn-rate + 128 + --run-time-s + 20 + ) + if(WORKER_THREADS) add_piccolo_test( NAME pi_basic_mt diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py new file mode 100644 index 000000000000..8cf992e3be13 --- /dev/null +++ b/tests/basicperf_locust.py @@ -0,0 +1,206 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +""" +"Basic Blocking Locust" benchmark. + +Measures the throughput of blocking writes (PUT /records/blocking/{key}), which +only return once the transaction has committed. This covers the same ground as +the piccolo-driven "Basic Blocking" benchmark, but drives load with locust +instead, so that the client count can be ramped up and the workload described +in Python rather than in a pre-generated parquet file. + +The load itself is defined in infra/basicperf_locustfile.py. This script owns +the network, runs locust against it, and converts locust's statistics into +bencher metrics. +""" + +import argparse +import csv +import os +import subprocess + +import infra.bencher +import infra.e2e_args +import infra.interfaces +import infra.key_space +import infra.network +import infra.proc +from loguru import logger as LOG + +LOCUST_FILE_NAME = "basicperf_locustfile.py" + +# Prefix for the CSV files locust writes. Locust appends _stats.csv, +# _failures.csv, and so on. +CSV_PREFIX = "locust" + +# Name of the row holding totals across all request types in locust's stats CSV. +AGGREGATED_ROW_NAME = "Aggregated" + + +def locust_file_path() -> str: + return os.path.join( + os.path.dirname(os.path.realpath(__file__)), "infra", LOCUST_FILE_NAME + ) + + +def run_locust(args, network, primary) -> dict: + """Run locust to completion against the given node, and return the + aggregated statistics it recorded.""" + csv_prefix = os.path.join(network.common_dir, CSV_PREFIX) + + session_auth = primary.session_auth("user0")["session_auth"] + host = "https://" + infra.interfaces.make_address( + primary.get_public_rpc_host(), primary.get_public_rpc_port() + ) + + cmd = ["locust"] + cmd += ["--headless"] + cmd += ["--locustfile", locust_file_path()] + cmd += ["--host", host] + + # Client authentication + cmd += ["--ca", primary.session_ca()["ca"]] + cmd += ["--cert", session_auth.cert] + cmd += ["--key", session_auth.key] + + cmd += ["--key-space-size", f"{args.key_space_size}"] + + # Load profile + cmd += ["--users", f"{args.users}"] + cmd += ["--spawn-rate", f"{args.spawn_rate}"] + cmd += ["--run-time", f"{args.run_time_s}s"] + + # A single locust process cannot saturate the service, because it drives + # all of its users from one thread. Fork enough workers to keep the client + # from being the bottleneck. + cmd += ["--processes", f"{args.locust_processes}"] + + # Discard everything recorded while users were still being spawned, so the + # reported numbers describe steady state at the full user count. + cmd += ["--reset-stats"] + + cmd += ["--csv", csv_prefix] + + LOG.info(f"Starting locust: {' '.join(cmd)}") + # Locust exits non-zero if any request failed, which should fail the test. + subprocess.run(cmd, check=True) + + return read_aggregated_stats(f"{csv_prefix}_stats.csv") + + +def read_aggregated_stats(stats_path: str) -> dict: + with open(stats_path, "r") as f: + rows = list(csv.DictReader(f)) + + for row in rows: + if row["Name"] == AGGREGATED_ROW_NAME: + return row + + raise RuntimeError(f"No {AGGREGATED_ROW_NAME} row found in {stats_path}") + + +def run(args): + LOG.info(f"Starting nodes on {args.nodes}") + with infra.network.network( + args.nodes, args.binary_dir, args.debug_nodes, pdb=args.pdb + ) as network: + network.start_and_open(args) + + primary, _ = network.find_primary() + + infra.key_space.create_and_fill_key_space(args.key_space_size, primary) + + stats = run_locust(args, network, primary) + + request_count = int(stats["Request Count"]) + failure_count = int(stats["Failure Count"]) + throughput = float(stats["Requests/s"]) + # Locust reports response times in milliseconds. + median_latency_ms = float(stats["Median Response Time"]) + p99_latency_ms = float(stats["99%"]) + min_latency_ms = float(stats["Min Response Time"]) + + LOG.info( + f"{request_count} requests ({failure_count} failures) " + f"=> {throughput:.1f} tx/s" + ) + LOG.info( + f"Latency: min={min_latency_ms:.1f}ms p50={median_latency_ms:.1f}ms " + f"p99={p99_latency_ms:.1f}ms" + ) + + if request_count == 0: + raise RuntimeError("Locust recorded no requests") + + # Locust should already have exited non-zero, but do not report a + # throughput figure built from failed requests under any circumstances. + if failure_count != 0: + raise RuntimeError( + f"Locust recorded {failure_count} failures out of {request_count} requests" + ) + + mem = infra.proc.get_proc_memory_stats(primary.remote.remote.proc.pid) + + network.stop_all_nodes() + + bf = infra.bencher.Bencher() + bf.set(args.perf_label, infra.bencher.Throughput(round(throughput, 1))) + bf.set( + args.perf_label, + infra.bencher.Latency( + value=median_latency_ms, + high_value=p99_latency_ms, + low_value=min_latency_ms, + ), + ) + if mem is not None: + bf.set_memory(args.perf_label, mem) + + +def cli_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--users", + help="Number of concurrent locust users, each sending one blocking write at a time", + type=int, + default=128, + ) + parser.add_argument( + "--spawn-rate", + help="Number of users to start per second", + type=int, + default=128, + ) + parser.add_argument( + "--run-time-s", + help="Duration of the load, in seconds, excluding the time taken to spawn users", + type=int, + default=20, + ) + parser.add_argument( + "--locust-processes", + help="Number of locust worker processes to fork", + type=int, + default=4, + ) + parser.add_argument( + "--key-space-size", + help="Size of the key space which is pre-populated and written to", + type=int, + default=1000, + ) + return infra.e2e_args.cli_args( + parser=parser, accept_unknown=False, ledger_chunk_bytes_override="5MB" + ) + + +if __name__ == "__main__": + args = cli_args() + # A single node is enough: the benchmark measures the time taken to commit + # on the primary, and additional nodes only add replication cost. + args.nodes = infra.e2e_args.min_nodes(args, f=0) + + run(args) diff --git a/tests/infra/basicperf.py b/tests/infra/basicperf.py index 7bb518517723..327cfb96c187 100644 --- a/tests/infra/basicperf.py +++ b/tests/infra/basicperf.py @@ -3,7 +3,6 @@ import argparse import datetime import hashlib -import http import json import os import random @@ -20,6 +19,7 @@ import infra.bencher import infra.e2e_args import infra.jwt_issuer +import infra.key_space import infra.proc import infra.remote_client @@ -214,22 +214,6 @@ def __call__( ) -def create_and_fill_key_space(size: int, primary: infra.node.Node) -> list[str]: - LOG.info(f"Creating and filling key space of size {size}") - space = [f"{i}" for i in range(size)] - mapping = {key: f"{hashlib.sha256(key.encode()).hexdigest()}" for key in space} - with primary.client("user0") as c: - r = c.post("/records", mapping) - assert r.status_code == http.HTTPStatus.NO_CONTENT, r - # Quick sanity check - for j in [0, -1]: - r = c.get(f"/records/{space[j]}") - assert r.status_code == http.HTTPStatus.OK, r - assert r.body.text() == mapping[space[j]], r - LOG.info("Key space created and filled") - return space - - def replace_primary(network, host, old_primary, snapshots_dir, statistics): LOG.info(f"Set up new node: {host}") node = network.create_node(host) @@ -297,7 +281,9 @@ def run(args): jwt = jwt_issuer.issue_jwt() additional_headers["Authorization"] = f"Bearer {jwt}" - key_space = create_and_fill_key_space(args.key_space_size, primary) + key_space = infra.key_space.create_and_fill_key_space( + args.key_space_size, primary + ) clients = [] client_idx = 0 diff --git a/tests/infra/basicperf_locustfile.py b/tests/infra/basicperf_locustfile.py new file mode 100644 index 000000000000..d773c9836bf3 --- /dev/null +++ b/tests/infra/basicperf_locustfile.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +""" +Locust workload for the "Basic Blocking Locust" benchmark. + +Each user issues blocking writes (PUT /records/blocking/{key}) one at a time, +waiting for each response before sending the next. That endpoint only responds +once the transaction has committed, so a single user's rate is bounded by +commit latency, and total throughput is driven by the number of users. + +FastHttpUser (geventhttpclient) is used rather than HttpUser (requests), +because the latter cannot drive enough requests per second to saturate the +service. +""" + +import hashlib +import random +import ssl + +from locust import constant, events, task +from locust.contrib.fasthttp import FastHttpUser + +# All requests are reported under a single name, so that locust aggregates them +# into one statistics entry rather than one per key. +REQUEST_NAME = "PUT /records/blocking/{key}" + +DEFAULT_KEY_SPACE_SIZE = 1000 + +EXPECTED_STATUS = 204 + +# Bodies are fixed per key, so they are built once per process and shared by +# every user in it, rather than rebuilt per user or per request. +_bodies: list[str] = [] + + +def _get_bodies(key_space_size: int) -> list[str]: + global _bodies + if len(_bodies) != key_space_size: + _bodies = [ + hashlib.sha256(f"{i}".encode()).hexdigest() for i in range(key_space_size) + ] + return _bodies + + +@events.init_command_line_parser.add_listener +def init_parser(parser): + parser.add_argument("--ca", help="Path to service certificate", required=True) + parser.add_argument("--cert", help="Path to client certificate", required=True) + parser.add_argument("--key", help="Path to client private key", required=True) + parser.add_argument( + "--key-space-size", + help="Number of distinct keys written to", + type=int, + default=DEFAULT_KEY_SPACE_SIZE, + ) + + +class BlockingWriter(FastHttpUser): + # Send the next request as soon as the previous response arrives. Each + # response already waits for commit, so no additional pacing is wanted. + wait_time = constant(0) + + # Verify the service certificate, rather than skipping verification as + # FastHttpUser does by default. + insecure = False + + def __init__(self, environment): + super().__init__(environment) + self.key_space_size = environment.parsed_options.key_space_size + self.bodies = _get_bodies(self.key_space_size) + + def ssl_context_factory(self): + opts = self.environment.parsed_options + context = ssl.create_default_context(cafile=opts.ca) + context.load_cert_chain(certfile=opts.cert, keyfile=opts.key) + return context + + @task + def blocking_write(self): + index = random.randrange(self.key_space_size) + with self.client.put( + f"/records/blocking/{index}", + data=self.bodies[index], + headers={"content-type": "text/plain"}, + name=REQUEST_NAME, + catch_response=True, + ) as response: + # Anything other than the expected status is a failure, including + # the 5xx returned when a transaction is invalidated, and the 0 + # reported when the connection itself failed. + if response.status_code == EXPECTED_STATUS: + response.success() + else: + response.failure(f"Unexpected status {response.status_code}") diff --git a/tests/infra/key_space.py b/tests/infra/key_space.py new file mode 100644 index 000000000000..f89084c0881b --- /dev/null +++ b/tests/infra/key_space.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import hashlib +import http + +from loguru import logger as LOG + +import infra.node + + +def key_body(key: str) -> str: + """The value written for a given key. Fixed per key, so that a write is + idempotent and the value can be recomputed without storing it.""" + return hashlib.sha256(key.encode()).hexdigest() + + +def create_and_fill_key_space(size: int, primary: infra.node.Node) -> list[str]: + LOG.info(f"Creating and filling key space of size {size}") + space = [f"{i}" for i in range(size)] + mapping = {key: key_body(key) for key in space} + with primary.client("user0") as c: + r = c.post("/records", mapping) + assert r.status_code == http.HTTPStatus.NO_CONTENT, r + # Quick sanity check + for j in [0, -1]: + r = c.get(f"/records/{space[j]}") + assert r.status_code == http.HTTPStatus.OK, r + assert r.body.text() == mapping[space[j]], r + LOG.info("Key space created and filled") + return space From 14f49abcadb90e22c673de5f0f5d8ffb1192155f Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 14:53:08 +0100 Subject: [PATCH 02/13] Normalise new test files to LF line endings The three files added by this branch were committed with CRLF, unlike every other Python file under tests/. No functional change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- tests/basicperf_locust.py | 412 ++++++++++++++-------------- tests/infra/basicperf_locustfile.py | 190 ++++++------- tests/infra/key_space.py | 62 ++--- 3 files changed, 332 insertions(+), 332 deletions(-) diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py index 8cf992e3be13..4da291e893d6 100644 --- a/tests/basicperf_locust.py +++ b/tests/basicperf_locust.py @@ -1,206 +1,206 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. - -""" -"Basic Blocking Locust" benchmark. - -Measures the throughput of blocking writes (PUT /records/blocking/{key}), which -only return once the transaction has committed. This covers the same ground as -the piccolo-driven "Basic Blocking" benchmark, but drives load with locust -instead, so that the client count can be ramped up and the workload described -in Python rather than in a pre-generated parquet file. - -The load itself is defined in infra/basicperf_locustfile.py. This script owns -the network, runs locust against it, and converts locust's statistics into -bencher metrics. -""" - -import argparse -import csv -import os -import subprocess - -import infra.bencher -import infra.e2e_args -import infra.interfaces -import infra.key_space -import infra.network -import infra.proc -from loguru import logger as LOG - -LOCUST_FILE_NAME = "basicperf_locustfile.py" - -# Prefix for the CSV files locust writes. Locust appends _stats.csv, -# _failures.csv, and so on. -CSV_PREFIX = "locust" - -# Name of the row holding totals across all request types in locust's stats CSV. -AGGREGATED_ROW_NAME = "Aggregated" - - -def locust_file_path() -> str: - return os.path.join( - os.path.dirname(os.path.realpath(__file__)), "infra", LOCUST_FILE_NAME - ) - - -def run_locust(args, network, primary) -> dict: - """Run locust to completion against the given node, and return the - aggregated statistics it recorded.""" - csv_prefix = os.path.join(network.common_dir, CSV_PREFIX) - - session_auth = primary.session_auth("user0")["session_auth"] - host = "https://" + infra.interfaces.make_address( - primary.get_public_rpc_host(), primary.get_public_rpc_port() - ) - - cmd = ["locust"] - cmd += ["--headless"] - cmd += ["--locustfile", locust_file_path()] - cmd += ["--host", host] - - # Client authentication - cmd += ["--ca", primary.session_ca()["ca"]] - cmd += ["--cert", session_auth.cert] - cmd += ["--key", session_auth.key] - - cmd += ["--key-space-size", f"{args.key_space_size}"] - - # Load profile - cmd += ["--users", f"{args.users}"] - cmd += ["--spawn-rate", f"{args.spawn_rate}"] - cmd += ["--run-time", f"{args.run_time_s}s"] - - # A single locust process cannot saturate the service, because it drives - # all of its users from one thread. Fork enough workers to keep the client - # from being the bottleneck. - cmd += ["--processes", f"{args.locust_processes}"] - - # Discard everything recorded while users were still being spawned, so the - # reported numbers describe steady state at the full user count. - cmd += ["--reset-stats"] - - cmd += ["--csv", csv_prefix] - - LOG.info(f"Starting locust: {' '.join(cmd)}") - # Locust exits non-zero if any request failed, which should fail the test. - subprocess.run(cmd, check=True) - - return read_aggregated_stats(f"{csv_prefix}_stats.csv") - - -def read_aggregated_stats(stats_path: str) -> dict: - with open(stats_path, "r") as f: - rows = list(csv.DictReader(f)) - - for row in rows: - if row["Name"] == AGGREGATED_ROW_NAME: - return row - - raise RuntimeError(f"No {AGGREGATED_ROW_NAME} row found in {stats_path}") - - -def run(args): - LOG.info(f"Starting nodes on {args.nodes}") - with infra.network.network( - args.nodes, args.binary_dir, args.debug_nodes, pdb=args.pdb - ) as network: - network.start_and_open(args) - - primary, _ = network.find_primary() - - infra.key_space.create_and_fill_key_space(args.key_space_size, primary) - - stats = run_locust(args, network, primary) - - request_count = int(stats["Request Count"]) - failure_count = int(stats["Failure Count"]) - throughput = float(stats["Requests/s"]) - # Locust reports response times in milliseconds. - median_latency_ms = float(stats["Median Response Time"]) - p99_latency_ms = float(stats["99%"]) - min_latency_ms = float(stats["Min Response Time"]) - - LOG.info( - f"{request_count} requests ({failure_count} failures) " - f"=> {throughput:.1f} tx/s" - ) - LOG.info( - f"Latency: min={min_latency_ms:.1f}ms p50={median_latency_ms:.1f}ms " - f"p99={p99_latency_ms:.1f}ms" - ) - - if request_count == 0: - raise RuntimeError("Locust recorded no requests") - - # Locust should already have exited non-zero, but do not report a - # throughput figure built from failed requests under any circumstances. - if failure_count != 0: - raise RuntimeError( - f"Locust recorded {failure_count} failures out of {request_count} requests" - ) - - mem = infra.proc.get_proc_memory_stats(primary.remote.remote.proc.pid) - - network.stop_all_nodes() - - bf = infra.bencher.Bencher() - bf.set(args.perf_label, infra.bencher.Throughput(round(throughput, 1))) - bf.set( - args.perf_label, - infra.bencher.Latency( - value=median_latency_ms, - high_value=p99_latency_ms, - low_value=min_latency_ms, - ), - ) - if mem is not None: - bf.set_memory(args.perf_label, mem) - - -def cli_args(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--users", - help="Number of concurrent locust users, each sending one blocking write at a time", - type=int, - default=128, - ) - parser.add_argument( - "--spawn-rate", - help="Number of users to start per second", - type=int, - default=128, - ) - parser.add_argument( - "--run-time-s", - help="Duration of the load, in seconds, excluding the time taken to spawn users", - type=int, - default=20, - ) - parser.add_argument( - "--locust-processes", - help="Number of locust worker processes to fork", - type=int, - default=4, - ) - parser.add_argument( - "--key-space-size", - help="Size of the key space which is pre-populated and written to", - type=int, - default=1000, - ) - return infra.e2e_args.cli_args( - parser=parser, accept_unknown=False, ledger_chunk_bytes_override="5MB" - ) - - -if __name__ == "__main__": - args = cli_args() - # A single node is enough: the benchmark measures the time taken to commit - # on the primary, and additional nodes only add replication cost. - args.nodes = infra.e2e_args.min_nodes(args, f=0) - - run(args) +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +""" +"Basic Blocking Locust" benchmark. + +Measures the throughput of blocking writes (PUT /records/blocking/{key}), which +only return once the transaction has committed. This covers the same ground as +the piccolo-driven "Basic Blocking" benchmark, but drives load with locust +instead, so that the client count can be ramped up and the workload described +in Python rather than in a pre-generated parquet file. + +The load itself is defined in infra/basicperf_locustfile.py. This script owns +the network, runs locust against it, and converts locust's statistics into +bencher metrics. +""" + +import argparse +import csv +import os +import subprocess + +import infra.bencher +import infra.e2e_args +import infra.interfaces +import infra.key_space +import infra.network +import infra.proc +from loguru import logger as LOG + +LOCUST_FILE_NAME = "basicperf_locustfile.py" + +# Prefix for the CSV files locust writes. Locust appends _stats.csv, +# _failures.csv, and so on. +CSV_PREFIX = "locust" + +# Name of the row holding totals across all request types in locust's stats CSV. +AGGREGATED_ROW_NAME = "Aggregated" + + +def locust_file_path() -> str: + return os.path.join( + os.path.dirname(os.path.realpath(__file__)), "infra", LOCUST_FILE_NAME + ) + + +def run_locust(args, network, primary) -> dict: + """Run locust to completion against the given node, and return the + aggregated statistics it recorded.""" + csv_prefix = os.path.join(network.common_dir, CSV_PREFIX) + + session_auth = primary.session_auth("user0")["session_auth"] + host = "https://" + infra.interfaces.make_address( + primary.get_public_rpc_host(), primary.get_public_rpc_port() + ) + + cmd = ["locust"] + cmd += ["--headless"] + cmd += ["--locustfile", locust_file_path()] + cmd += ["--host", host] + + # Client authentication + cmd += ["--ca", primary.session_ca()["ca"]] + cmd += ["--cert", session_auth.cert] + cmd += ["--key", session_auth.key] + + cmd += ["--key-space-size", f"{args.key_space_size}"] + + # Load profile + cmd += ["--users", f"{args.users}"] + cmd += ["--spawn-rate", f"{args.spawn_rate}"] + cmd += ["--run-time", f"{args.run_time_s}s"] + + # A single locust process cannot saturate the service, because it drives + # all of its users from one thread. Fork enough workers to keep the client + # from being the bottleneck. + cmd += ["--processes", f"{args.locust_processes}"] + + # Discard everything recorded while users were still being spawned, so the + # reported numbers describe steady state at the full user count. + cmd += ["--reset-stats"] + + cmd += ["--csv", csv_prefix] + + LOG.info(f"Starting locust: {' '.join(cmd)}") + # Locust exits non-zero if any request failed, which should fail the test. + subprocess.run(cmd, check=True) + + return read_aggregated_stats(f"{csv_prefix}_stats.csv") + + +def read_aggregated_stats(stats_path: str) -> dict: + with open(stats_path, "r") as f: + rows = list(csv.DictReader(f)) + + for row in rows: + if row["Name"] == AGGREGATED_ROW_NAME: + return row + + raise RuntimeError(f"No {AGGREGATED_ROW_NAME} row found in {stats_path}") + + +def run(args): + LOG.info(f"Starting nodes on {args.nodes}") + with infra.network.network( + args.nodes, args.binary_dir, args.debug_nodes, pdb=args.pdb + ) as network: + network.start_and_open(args) + + primary, _ = network.find_primary() + + infra.key_space.create_and_fill_key_space(args.key_space_size, primary) + + stats = run_locust(args, network, primary) + + request_count = int(stats["Request Count"]) + failure_count = int(stats["Failure Count"]) + throughput = float(stats["Requests/s"]) + # Locust reports response times in milliseconds. + median_latency_ms = float(stats["Median Response Time"]) + p99_latency_ms = float(stats["99%"]) + min_latency_ms = float(stats["Min Response Time"]) + + LOG.info( + f"{request_count} requests ({failure_count} failures) " + f"=> {throughput:.1f} tx/s" + ) + LOG.info( + f"Latency: min={min_latency_ms:.1f}ms p50={median_latency_ms:.1f}ms " + f"p99={p99_latency_ms:.1f}ms" + ) + + if request_count == 0: + raise RuntimeError("Locust recorded no requests") + + # Locust should already have exited non-zero, but do not report a + # throughput figure built from failed requests under any circumstances. + if failure_count != 0: + raise RuntimeError( + f"Locust recorded {failure_count} failures out of {request_count} requests" + ) + + mem = infra.proc.get_proc_memory_stats(primary.remote.remote.proc.pid) + + network.stop_all_nodes() + + bf = infra.bencher.Bencher() + bf.set(args.perf_label, infra.bencher.Throughput(round(throughput, 1))) + bf.set( + args.perf_label, + infra.bencher.Latency( + value=median_latency_ms, + high_value=p99_latency_ms, + low_value=min_latency_ms, + ), + ) + if mem is not None: + bf.set_memory(args.perf_label, mem) + + +def cli_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--users", + help="Number of concurrent locust users, each sending one blocking write at a time", + type=int, + default=128, + ) + parser.add_argument( + "--spawn-rate", + help="Number of users to start per second", + type=int, + default=128, + ) + parser.add_argument( + "--run-time-s", + help="Duration of the load, in seconds, excluding the time taken to spawn users", + type=int, + default=20, + ) + parser.add_argument( + "--locust-processes", + help="Number of locust worker processes to fork", + type=int, + default=4, + ) + parser.add_argument( + "--key-space-size", + help="Size of the key space which is pre-populated and written to", + type=int, + default=1000, + ) + return infra.e2e_args.cli_args( + parser=parser, accept_unknown=False, ledger_chunk_bytes_override="5MB" + ) + + +if __name__ == "__main__": + args = cli_args() + # A single node is enough: the benchmark measures the time taken to commit + # on the primary, and additional nodes only add replication cost. + args.nodes = infra.e2e_args.min_nodes(args, f=0) + + run(args) diff --git a/tests/infra/basicperf_locustfile.py b/tests/infra/basicperf_locustfile.py index d773c9836bf3..7bd9f7ab7d79 100644 --- a/tests/infra/basicperf_locustfile.py +++ b/tests/infra/basicperf_locustfile.py @@ -1,95 +1,95 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. - -""" -Locust workload for the "Basic Blocking Locust" benchmark. - -Each user issues blocking writes (PUT /records/blocking/{key}) one at a time, -waiting for each response before sending the next. That endpoint only responds -once the transaction has committed, so a single user's rate is bounded by -commit latency, and total throughput is driven by the number of users. - -FastHttpUser (geventhttpclient) is used rather than HttpUser (requests), -because the latter cannot drive enough requests per second to saturate the -service. -""" - -import hashlib -import random -import ssl - -from locust import constant, events, task -from locust.contrib.fasthttp import FastHttpUser - -# All requests are reported under a single name, so that locust aggregates them -# into one statistics entry rather than one per key. -REQUEST_NAME = "PUT /records/blocking/{key}" - -DEFAULT_KEY_SPACE_SIZE = 1000 - -EXPECTED_STATUS = 204 - -# Bodies are fixed per key, so they are built once per process and shared by -# every user in it, rather than rebuilt per user or per request. -_bodies: list[str] = [] - - -def _get_bodies(key_space_size: int) -> list[str]: - global _bodies - if len(_bodies) != key_space_size: - _bodies = [ - hashlib.sha256(f"{i}".encode()).hexdigest() for i in range(key_space_size) - ] - return _bodies - - -@events.init_command_line_parser.add_listener -def init_parser(parser): - parser.add_argument("--ca", help="Path to service certificate", required=True) - parser.add_argument("--cert", help="Path to client certificate", required=True) - parser.add_argument("--key", help="Path to client private key", required=True) - parser.add_argument( - "--key-space-size", - help="Number of distinct keys written to", - type=int, - default=DEFAULT_KEY_SPACE_SIZE, - ) - - -class BlockingWriter(FastHttpUser): - # Send the next request as soon as the previous response arrives. Each - # response already waits for commit, so no additional pacing is wanted. - wait_time = constant(0) - - # Verify the service certificate, rather than skipping verification as - # FastHttpUser does by default. - insecure = False - - def __init__(self, environment): - super().__init__(environment) - self.key_space_size = environment.parsed_options.key_space_size - self.bodies = _get_bodies(self.key_space_size) - - def ssl_context_factory(self): - opts = self.environment.parsed_options - context = ssl.create_default_context(cafile=opts.ca) - context.load_cert_chain(certfile=opts.cert, keyfile=opts.key) - return context - - @task - def blocking_write(self): - index = random.randrange(self.key_space_size) - with self.client.put( - f"/records/blocking/{index}", - data=self.bodies[index], - headers={"content-type": "text/plain"}, - name=REQUEST_NAME, - catch_response=True, - ) as response: - # Anything other than the expected status is a failure, including - # the 5xx returned when a transaction is invalidated, and the 0 - # reported when the connection itself failed. - if response.status_code == EXPECTED_STATUS: - response.success() - else: - response.failure(f"Unexpected status {response.status_code}") +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +""" +Locust workload for the "Basic Blocking Locust" benchmark. + +Each user issues blocking writes (PUT /records/blocking/{key}) one at a time, +waiting for each response before sending the next. That endpoint only responds +once the transaction has committed, so a single user's rate is bounded by +commit latency, and total throughput is driven by the number of users. + +FastHttpUser (geventhttpclient) is used rather than HttpUser (requests), +because the latter cannot drive enough requests per second to saturate the +service. +""" + +import hashlib +import random +import ssl + +from locust import constant, events, task +from locust.contrib.fasthttp import FastHttpUser + +# All requests are reported under a single name, so that locust aggregates them +# into one statistics entry rather than one per key. +REQUEST_NAME = "PUT /records/blocking/{key}" + +DEFAULT_KEY_SPACE_SIZE = 1000 + +EXPECTED_STATUS = 204 + +# Bodies are fixed per key, so they are built once per process and shared by +# every user in it, rather than rebuilt per user or per request. +_bodies: list[str] = [] + + +def _get_bodies(key_space_size: int) -> list[str]: + global _bodies + if len(_bodies) != key_space_size: + _bodies = [ + hashlib.sha256(f"{i}".encode()).hexdigest() for i in range(key_space_size) + ] + return _bodies + + +@events.init_command_line_parser.add_listener +def init_parser(parser): + parser.add_argument("--ca", help="Path to service certificate", required=True) + parser.add_argument("--cert", help="Path to client certificate", required=True) + parser.add_argument("--key", help="Path to client private key", required=True) + parser.add_argument( + "--key-space-size", + help="Number of distinct keys written to", + type=int, + default=DEFAULT_KEY_SPACE_SIZE, + ) + + +class BlockingWriter(FastHttpUser): + # Send the next request as soon as the previous response arrives. Each + # response already waits for commit, so no additional pacing is wanted. + wait_time = constant(0) + + # Verify the service certificate, rather than skipping verification as + # FastHttpUser does by default. + insecure = False + + def __init__(self, environment): + super().__init__(environment) + self.key_space_size = environment.parsed_options.key_space_size + self.bodies = _get_bodies(self.key_space_size) + + def ssl_context_factory(self): + opts = self.environment.parsed_options + context = ssl.create_default_context(cafile=opts.ca) + context.load_cert_chain(certfile=opts.cert, keyfile=opts.key) + return context + + @task + def blocking_write(self): + index = random.randrange(self.key_space_size) + with self.client.put( + f"/records/blocking/{index}", + data=self.bodies[index], + headers={"content-type": "text/plain"}, + name=REQUEST_NAME, + catch_response=True, + ) as response: + # Anything other than the expected status is a failure, including + # the 5xx returned when a transaction is invalidated, and the 0 + # reported when the connection itself failed. + if response.status_code == EXPECTED_STATUS: + response.success() + else: + response.failure(f"Unexpected status {response.status_code}") diff --git a/tests/infra/key_space.py b/tests/infra/key_space.py index f89084c0881b..d7aebcaa7ec2 100644 --- a/tests/infra/key_space.py +++ b/tests/infra/key_space.py @@ -1,31 +1,31 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. - -import hashlib -import http - -from loguru import logger as LOG - -import infra.node - - -def key_body(key: str) -> str: - """The value written for a given key. Fixed per key, so that a write is - idempotent and the value can be recomputed without storing it.""" - return hashlib.sha256(key.encode()).hexdigest() - - -def create_and_fill_key_space(size: int, primary: infra.node.Node) -> list[str]: - LOG.info(f"Creating and filling key space of size {size}") - space = [f"{i}" for i in range(size)] - mapping = {key: key_body(key) for key in space} - with primary.client("user0") as c: - r = c.post("/records", mapping) - assert r.status_code == http.HTTPStatus.NO_CONTENT, r - # Quick sanity check - for j in [0, -1]: - r = c.get(f"/records/{space[j]}") - assert r.status_code == http.HTTPStatus.OK, r - assert r.body.text() == mapping[space[j]], r - LOG.info("Key space created and filled") - return space +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import hashlib +import http + +from loguru import logger as LOG + +import infra.node + + +def key_body(key: str) -> str: + """The value written for a given key. Fixed per key, so that a write is + idempotent and the value can be recomputed without storing it.""" + return hashlib.sha256(key.encode()).hexdigest() + + +def create_and_fill_key_space(size: int, primary: infra.node.Node) -> list[str]: + LOG.info(f"Creating and filling key space of size {size}") + space = [f"{i}" for i in range(size)] + mapping = {key: key_body(key) for key in space} + with primary.client("user0") as c: + r = c.post("/records", mapping) + assert r.status_code == http.HTTPStatus.NO_CONTENT, r + # Quick sanity check + for j in [0, -1]: + r = c.get(f"/records/{space[j]}") + assert r.status_code == http.HTTPStatus.OK, r + assert r.body.text() == mapping[space[j]], r + LOG.info("Key space created and filled") + return space From 12b0454a2f24070ce5243cd3836fd62e3399014e Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 14:53:30 +0100 Subject: [PATCH 03/13] Measure for a fixed window after users have spawned Locust --run-time starts counting when locust starts, so it includes the ramp, and --reset-stats discards the statistics gathered during the ramp without extending the deadline. The measurement window was therefore shorter than requested, and shrank as the spawn rate was lowered, until it disappeared entirely. This mattered because varying the client count is the point of this test: at --users 128 --spawn-rate 4 the run ended mid-ramp and reported 127 tx/s instead of the ~1250 tx/s that 128 users actually sustain, and did so without failing. Start the shutdown timer from locust spawning_complete instead, so the window is the same length whatever the spawn rate is, and rename --run-time-s to --measure-time-s to describe what it now does. --run-time is kept as a backstop against a run which never finishes spawning. Also fail, rather than report, when a run ends without having spawned all users, or when the window measured is shorter than the one asked for. Both produce plausible looking figures that do not describe steady state. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- CMakeLists.txt | 2 +- tests/basicperf_locust.py | 38 ++++++++++++++++++-- tests/infra/basicperf_locustfile.py | 56 +++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index af6f4bfda90a..4559bd1148da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1498,7 +1498,7 @@ if(BUILD_TESTS) 128 --spawn-rate 128 - --run-time-s + --measure-time-s 20 ) diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py index 4da291e893d6..ee5d2e9aed3d 100644 --- a/tests/basicperf_locust.py +++ b/tests/basicperf_locust.py @@ -17,6 +17,7 @@ import argparse import csv +import math import os import subprocess @@ -37,6 +38,14 @@ # Name of the row holding totals across all request types in locust's stats CSV. AGGREGATED_ROW_NAME = "Aggregated" +# Slack allowed on top of the expected spawn and measurement time before the +# run is considered stuck and killed by locust's own --run-time. +RUN_TIME_MARGIN_S = 60 + +# Shortest measurement window, as a fraction of the one requested, which is +# still accepted as describing steady state. +MIN_MEASURED_FRACTION = 0.9 + def locust_file_path() -> str: return os.path.join( @@ -69,7 +78,16 @@ def run_locust(args, network, primary) -> dict: # Load profile cmd += ["--users", f"{args.users}"] cmd += ["--spawn-rate", f"{args.spawn_rate}"] - cmd += ["--run-time", f"{args.run_time_s}s"] + cmd += ["--measure-time-s", f"{args.measure_time_s}"] + + # The run is normally ended by the locustfile, a fixed time after the last + # user has spawned. This is only a backstop against a run which never + # finishes spawning, so it is deliberately generous: locust's --run-time + # includes the ramp, and pre-empting the real deadline would silently + # shorten the measurement window. + spawn_time_s = math.ceil(args.users / args.spawn_rate) + run_time_ceiling_s = spawn_time_s + args.measure_time_s + RUN_TIME_MARGIN_S + cmd += ["--run-time", f"{run_time_ceiling_s}s"] # A single locust process cannot saturate the service, because it drives # all of its users from one thread. Fork enough workers to keep the client @@ -133,6 +151,20 @@ def run(args): if request_count == 0: raise RuntimeError("Locust recorded no requests") + # Locust reports throughput over the window its statistics cover, so + # the two together give the length of that window. Check it against the + # window that was asked for: a short window still yields a plausible + # looking throughput, so without this a truncated run would be reported + # as a valid result. + measured_duration_s = request_count / throughput + if measured_duration_s < args.measure_time_s * MIN_MEASURED_FRACTION: + raise RuntimeError( + f"Measured over {measured_duration_s:.1f}s, but expected " + f"{args.measure_time_s}s. The run was cut short, so this " + "result does not describe steady state." + ) + LOG.info(f"Measured over {measured_duration_s:.1f}s") + # Locust should already have exited non-zero, but do not report a # throughput figure built from failed requests under any circumstances. if failure_count != 0: @@ -175,8 +207,8 @@ def cli_args(): default=128, ) parser.add_argument( - "--run-time-s", - help="Duration of the load, in seconds, excluding the time taken to spawn users", + "--measure-time-s", + help="Seconds to measure for, once all users have spawned", type=int, default=20, ) diff --git a/tests/infra/basicperf_locustfile.py b/tests/infra/basicperf_locustfile.py index 7bd9f7ab7d79..f1d8cfc4c893 100644 --- a/tests/infra/basicperf_locustfile.py +++ b/tests/infra/basicperf_locustfile.py @@ -15,11 +15,14 @@ """ import hashlib +import logging import random import ssl +import gevent from locust import constant, events, task from locust.contrib.fasthttp import FastHttpUser +from locust.runners import WorkerRunner # All requests are reported under a single name, so that locust aggregates them # into one statistics entry rather than one per key. @@ -27,8 +30,12 @@ DEFAULT_KEY_SPACE_SIZE = 1000 +DEFAULT_MEASURE_TIME_S = 20 + EXPECTED_STATUS = 204 +LOG = logging.getLogger(__name__) + # Bodies are fixed per key, so they are built once per process and shared by # every user in it, rather than rebuilt per user or per request. _bodies: list[str] = [] @@ -54,6 +61,55 @@ def init_parser(parser): type=int, default=DEFAULT_KEY_SPACE_SIZE, ) + parser.add_argument( + "--measure-time-s", + help="Seconds to keep running once all users have spawned", + type=int, + default=DEFAULT_MEASURE_TIME_S, + ) + + +@events.init.add_listener +def on_init(environment, **_kwargs): + """Stop the run a fixed time after the last user has spawned. + + Locust's own --run-time starts counting when locust starts, so it includes + the ramp. --reset-stats discards the statistics gathered during the ramp + but does not extend the deadline, so the further the ramp is stretched, the + smaller the steady-state window becomes, until it disappears entirely. + Timing from spawning_complete instead keeps the measurement window the same + length whatever the spawn rate is. + """ + # In distributed mode the master tells the workers to fire this event too, + # but only the master decides when the run ends. + if isinstance(environment.runner, WorkerRunner): + return + + spawning_completed = False + + def stop_after_measurement_window(**_kwargs): + nonlocal spawning_completed + spawning_completed = True + # Statistics are reset by --reset-stats on this same event, so the + # window measured here is exactly the window reported. + gevent.spawn_later( + environment.parsed_options.measure_time_s, environment.runner.quit + ) + + def check_spawning_completed(**_kwargs): + # Reaching the end without spawning having completed means the run was + # ended by the --run-time backstop mid-ramp. The statistics then + # describe a partial ramp, but still look like a plausible result, so + # say so and exit non-zero rather than reporting them. + if not spawning_completed: + LOG.error( + "Run ended before all users had spawned. " + "The statistics do not describe steady state." + ) + environment.process_exit_code = 1 + + environment.events.spawning_complete.add_listener(stop_after_measurement_window) + environment.events.quitting.add_listener(check_spawning_completed) class BlockingWriter(FastHttpUser): From ed7a1e6f777ab543ae1d77a2e7d44bb09a82b2c2 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 15:00:06 +0100 Subject: [PATCH 04/13] Bind the locust master to a free port Locust workers reach the master on port 5557 by default, so a second locust run anywhere on the same machine fails to bind. Pick a free port per run instead, via the existing infra.net helper. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- tests/basicperf_locust.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py index ee5d2e9aed3d..487f652aebba 100644 --- a/tests/basicperf_locust.py +++ b/tests/basicperf_locust.py @@ -25,6 +25,7 @@ import infra.e2e_args import infra.interfaces import infra.key_space +import infra.net import infra.network import infra.proc from loguru import logger as LOG @@ -94,6 +95,13 @@ def run_locust(args, network, primary) -> dict: # from being the bottleneck. cmd += ["--processes", f"{args.locust_processes}"] + # Workers reach the master over TCP, on a fixed port 5557 by default. Pick + # a free one instead, so that a second locust run on the same machine, from + # another test or another checkout, does not fail to bind. + master_port = infra.net.probably_free_local_port("localhost") + cmd += ["--master-bind-port", f"{master_port}"] + cmd += ["--master-port", f"{master_port}"] + # Discard everything recorded while users were still being spawned, so the # reported numbers describe steady state at the full user count. cmd += ["--reset-stats"] From eabdb9d976a6be3e0080be5fcb8042bfc429484a Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 15:58:35 +0100 Subject: [PATCH 05/13] Fail with a diagnosis when locust reports N/A Locust writes N/A rather than a number in the statistics CSV when it has too few samples to compute a percentile. float() then raised a bare ValueError, after the network had already been stopped, losing the run with no indication of what had gone wrong. Read the numeric columns through a helper which reports the column and value, and says that the run did not gather enough data. Found by a run which produced almost no samples. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- tests/basicperf_locust.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py index 487f652aebba..9eb55631ba17 100644 --- a/tests/basicperf_locust.py +++ b/tests/basicperf_locust.py @@ -126,6 +126,25 @@ def read_aggregated_stats(stats_path: str) -> dict: raise RuntimeError(f"No {AGGREGATED_ROW_NAME} row found in {stats_path}") +def stat_as_float(stats: dict, column: str) -> float: + """Read one numeric column from locust's statistics. + + Locust writes "N/A" rather than a number when it has too few samples to + produce a percentile, so a run which barely gathered any data would + otherwise fail here with a bare ValueError, after the network has already + been torn down and the evidence lost. + """ + value = stats[column] + try: + return float(value) + except ValueError as e: + raise RuntimeError( + f"Locust reported {column!r} as {value!r} rather than a number, " + "which means it had too few samples to compute it. The run did " + "not gather enough data to describe steady state." + ) from e + + def run(args): LOG.info(f"Starting nodes on {args.nodes}") with infra.network.network( @@ -141,11 +160,11 @@ def run(args): request_count = int(stats["Request Count"]) failure_count = int(stats["Failure Count"]) - throughput = float(stats["Requests/s"]) + throughput = stat_as_float(stats, "Requests/s") # Locust reports response times in milliseconds. - median_latency_ms = float(stats["Median Response Time"]) - p99_latency_ms = float(stats["99%"]) - min_latency_ms = float(stats["Min Response Time"]) + median_latency_ms = stat_as_float(stats, "Median Response Time") + p99_latency_ms = stat_as_float(stats, "99%") + min_latency_ms = stat_as_float(stats, "Min Response Time") LOG.info( f"{request_count} requests ({failure_count} failures) " From 0edff69a468b74aa81e26199d790995808b1bbe7 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 16:28:13 +0100 Subject: [PATCH 06/13] Snapshot as rarely as the piccolo perf tests do add_piccolo_test passes --snapshot-tx-interval 10000 for every piccolo perf test. add_e2e_test does not pass it at all, and e2e_args defaults it to 10, which is sensible for functional tests but not for a benchmark. This test was therefore writing and fsyncing a ~213KB snapshot every 10 transactions for the whole run, which measures the disk rather than the service, and makes the figure incomparable with Basic Blocking. Found by the vegeta comparison work, which hit the same defect. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4559bd1148da..7ef32815a14e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1494,6 +1494,11 @@ if(BUILD_TESTS) "samples/apps/basic/basic" --perf-label "Basic Blocking Locust" + # add_piccolo_test passes this for every piccolo perf test, but + # add_e2e_test does not, and the default is 10. Without it this + # benchmark snapshots every 10 transactions and measures the disk. + --snapshot-tx-interval + 10000 --users 128 --spawn-rate From d4162fbc8b9c6713d24abfb97560add01f0b2d2c Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 17:09:59 +0100 Subject: [PATCH 07/13] Measure at 5ms, 100ms and 1s signature intervals Blocking writes return once their transaction commits, and commit cannot outpace the signature interval, so a single interval only measures one regime. Sweeping three separates them: at 1s and 100ms the workload is latency-bound and throughput is simply the client count divided by the interval, while at 5ms the node becomes the limit and the benchmark measures capacity instead. Each interval gets its own network, since the interval is fixed in the node configuration at startup. consensus_update_timeout_ms moves with it, as in commit_latency.py, because commit cannot be observed faster than the primary sends updates. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- CMakeLists.txt | 6 ++++ tests/basicperf_locust.py | 64 +++++++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ef32815a14e..6564d76bd123 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1505,6 +1505,12 @@ if(BUILD_TESTS) 128 --measure-time-s 20 + # Blocking writes return on commit, and commit cannot outpace the + # signature interval, so this sweeps from latency-bound to capacity-bound. + --sig-ms-intervals + 5 + 100 + 1000 ) if(WORKER_THREADS) diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py index 9eb55631ba17..6268f997dd6e 100644 --- a/tests/basicperf_locust.py +++ b/tests/basicperf_locust.py @@ -145,8 +145,18 @@ def stat_as_float(stats: dict, column: str) -> float: ) from e -def run(args): - LOG.info(f"Starting nodes on {args.nodes}") +def measure(args, sig_ms_interval: int) -> dict: + """Run the workload against a fresh network with the given signature + interval, and return the statistics locust recorded.""" + args.sig_ms_interval = sig_ms_interval + # A response is only sent once its transaction commits, and commit cannot + # be observed any faster than the primary sends consensus updates. Move + # that in step with the signature interval, as commit_latency.py does, + # otherwise the shorter intervals are gated by the 100ms default and the + # setting has no effect. + args.consensus_update_timeout_ms = sig_ms_interval + + LOG.info(f"Starting nodes on {args.nodes} with {sig_ms_interval}ms signatures") with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, pdb=args.pdb ) as network: @@ -203,18 +213,44 @@ def run(args): network.stop_all_nodes() - bf = infra.bencher.Bencher() - bf.set(args.perf_label, infra.bencher.Throughput(round(throughput, 1))) + return { + "throughput": throughput, + "median_latency_ms": median_latency_ms, + "p99_latency_ms": p99_latency_ms, + "min_latency_ms": min_latency_ms, + "memory": mem, + } + + +def run(args): + # Each interval needs its own network, since the signature interval is + # fixed in the node's configuration at startup. + results = {} + for sig_ms_interval in args.sig_ms_intervals: + results[sig_ms_interval] = measure(args, sig_ms_interval) + + bf = infra.bencher.Bencher() + for sig_ms_interval, result in results.items(): + label = f"{args.perf_label} (sig_ms_interval={sig_ms_interval}ms)" + bf.set(label, infra.bencher.Throughput(round(result["throughput"], 1))) bf.set( - args.perf_label, + label, infra.bencher.Latency( - value=median_latency_ms, - high_value=p99_latency_ms, - low_value=min_latency_ms, + value=result["median_latency_ms"], + high_value=result["p99_latency_ms"], + low_value=result["min_latency_ms"], ), ) - if mem is not None: - bf.set_memory(args.perf_label, mem) + if result["memory"] is not None: + bf.set_memory(label, result["memory"]) + + LOG.info("Summary:") + for sig_ms_interval, result in results.items(): + LOG.info( + f" {sig_ms_interval:>5}ms signatures: " + f"{result['throughput']:>9.1f} tx/s, " + f"p50 {result['median_latency_ms']:.1f}ms" + ) def cli_args(): @@ -245,6 +281,14 @@ def cli_args(): type=int, default=4, ) + parser.add_argument( + "--sig-ms-intervals", + help="Signature intervals, in milliseconds, to measure the workload at. " + "Each is measured against its own network.", + type=int, + nargs="+", + default=[5, 100, 1000], + ) parser.add_argument( "--key-space-size", help="Size of the key space which is pre-populated and written to", From c07d93428d0a0d5b53495b0dcadc598f8157acb6 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 17:56:25 +0100 Subject: [PATCH 08/13] Plot benchmarks which main has no history for A benchmark added by a branch has no main runs to build an EWMA baseline from, so render_chart skipped it entirely. That made a new benchmark invisible on the very pull request which adds it, which is when it is most worth seeing. Plot such a benchmark against the branch's own earliest run instead, so its movement across the branch's runs is visible, and mark it as new. It carries no standard deviation band and is never coloured as an improvement or a regression, because there is nothing on main to compare it against. Borrowing a related benchmark's baseline was considered and rejected: an axis normalised against something which measures a different thing shows a difference which is not a change in CCF, and because the chart scale follows the largest axis, one such axis compresses every other benchmark into illegibility. Also truncate long axis labels in the middle rather than at the end. Benchmarks measured at several settings differ only in their suffix, so truncating the end left the 100ms and 1000ms axes indistinguishable. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- scripts/perf_compare_radar.py | 85 ++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 15 deletions(-) diff --git a/scripts/perf_compare_radar.py b/scripts/perf_compare_radar.py index ab43797f0fad..c1cad4e63038 100644 --- a/scripts/perf_compare_radar.py +++ b/scripts/perf_compare_radar.py @@ -248,19 +248,46 @@ def axis_label_color(percent: float, higher_is_better: bool, within_noise: bool) return LABEL_GOOD if improved else LABEL_BAD +NEW_BENCHMARK_MARKER = "new" + + +def shorten_label(label: str, max_length: int) -> str: + """Shorten a label to fit, keeping both ends. + + Benchmarks measured at several settings differ only in their suffix, for + example the interval in "Basic Blocking Locust 100ms", so truncating the + end would render such axes indistinguishable from one another. + """ + if len(label) <= max_length: + return label + if max_length <= 3: + return label[:max_length] + budget = max_length - 3 + head = budget // 2 + tail = budget - head + return f"{label[:head]}...{label[-tail:]}" + + def axis_label( - benchmark: str, value: float, percent: float, unit: str, within_noise: bool + benchmark: str, + value: float, + percent: float, + unit: str, + within_noise: bool, + is_new: bool = False, ) -> str: """Shorten benchmark labels and include the branch real value and delta.""" label = SIG_MS_INTERVAL_RE.sub(r" \1", benchmark) - suffix = ( - f": {metric_label_value(value, unit)} " - f"{format_delta_percent(percent, within_noise)}" - ) - max_label_length = MAX_AXIS_LABEL_LENGTH - len(suffix) - if len(label) <= max_label_length: - return f"{label}{suffix}" - return f"{label[:max_label_length - 3]}...{suffix}" + if is_new: + # There is no baseline to express a difference against, so report the + # value alone and say why. + suffix = f": {metric_label_value(value, unit)} ({NEW_BENCHMARK_MARKER})" + else: + suffix = ( + f": {metric_label_value(value, unit)} " + f"{format_delta_percent(percent, within_noise)}" + ) + return f"{shorten_label(label, MAX_AXIS_LABEL_LENGTH - len(suffix))}{suffix}" def normalized_percent(value: float, baseline: float) -> float: @@ -319,19 +346,39 @@ def render_mermaid_radar_chart( for data in trend if (value := metric_value(data, benchmark, metric)) is not None ] - if not main_values: - continue - baseline = ewma(main_values) + # A benchmark added by this branch has no main history to compare + # against. Rather than drop it, which would make a new benchmark + # invisible on the very PR that adds it, plot it against this branch's + # own earliest run so that movement across branch runs is still + # visible, and mark it so it is not mistaken for a main comparison. + is_new = not main_values + if is_new: + branch_values = [ + value + for data in branch_runs + if (value := metric_value(data, benchmark, metric)) is not None + ] + if not branch_values: + continue + baseline = branch_values[0] + sigma = 0.0 + else: + baseline = ewma(main_values) + sigma = statistics.pstdev(main_values) if len(main_values) > 1 else 0.0 + if baseline <= 0: continue - sigma = statistics.pstdev(main_values) if len(main_values) > 1 else 0.0 branch_percent = normalized_percent(branch_value, baseline) sigma_percent = normalized_percent(sigma, baseline) - within_noise = within_noise_band(branch_percent, sigma_percent) + # Without a main baseline there is no improvement or regression to + # report, so such an axis is never coloured as either. + within_noise = ( + True if is_new else within_noise_band(branch_percent, sigma_percent) + ) axes.append( - f"b{index}[{mermaid_label(axis_label(benchmark, branch_value, branch_percent, unit, within_noise))}]" + f"b{index}[{mermaid_label(axis_label(benchmark, branch_value, branch_percent, unit, within_noise, is_new))}]" ) axis_colors.append( axis_label_color(branch_percent, higher_better, within_noise) @@ -507,6 +554,14 @@ def render_comparison( "Higher is better for throughput and rate, lower for latency and memory._" ), "", + ( + f"_A benchmark marked ({NEW_BENCHMARK_MARKER}) does not exist on `main` yet, " + "so it has no baseline to be compared against. It is plotted against this " + "branch's own earliest run instead, which shows how it moved across the " + "branch's runs but says nothing about `main`, and it is never coloured as " + "an improvement or a regression._" + ), + "", "", "", ] From 70c396ee7bb2c3a67bcdea8d1eb5f4b1b63fa852 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 18:04:36 +0100 Subject: [PATCH 09/13] Drive 320 clients from 10 locust processes The blocking workload is latency-bound at the longer signature intervals, so throughput there is set by the client count: 320 clients raises the 100ms point from ~1260 to ~3100 tx/s and the 1s point from ~130 to ~314, both within a few percent of clients divided by interval. Ten sending processes rather than four keeps locust from becoming the limit while driving that many clients, since each process drives all of its users from a single thread. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- CMakeLists.txt | 6 ++++-- tests/basicperf_locust.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6564d76bd123..6675f5225ce9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1500,9 +1500,11 @@ if(BUILD_TESTS) --snapshot-tx-interval 10000 --users - 128 + 320 --spawn-rate - 128 + 320 + --locust-processes + 10 --measure-time-s 20 # Blocking writes return on commit, and commit cannot outpace the diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py index 6268f997dd6e..c2c5cb3bd2a7 100644 --- a/tests/basicperf_locust.py +++ b/tests/basicperf_locust.py @@ -261,13 +261,13 @@ def cli_args(): "--users", help="Number of concurrent locust users, each sending one blocking write at a time", type=int, - default=128, + default=320, ) parser.add_argument( "--spawn-rate", help="Number of users to start per second", type=int, - default=128, + default=320, ) parser.add_argument( "--measure-time-s", @@ -279,7 +279,7 @@ def cli_args(): "--locust-processes", help="Number of locust worker processes to fork", type=int, - default=4, + default=10, ) parser.add_argument( "--sig-ms-intervals", From f23fce184e85da2d5a38b9e919cbf80aee619560 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 18:27:55 +0100 Subject: [PATCH 10/13] Measure the fastest signatures at 2ms rather than 5ms The shortest interval is the point at which the node, rather than the signature timer, becomes the limit, so it is the one which measures capacity. Shortening it to 2ms pushes further past the latency-bound regime. The node ticks every 1ms in these tests, so a 2ms interval is representable. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- CMakeLists.txt | 2 +- tests/basicperf_locust.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6675f5225ce9..0e3bd13d0fb3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1510,7 +1510,7 @@ if(BUILD_TESTS) # Blocking writes return on commit, and commit cannot outpace the # signature interval, so this sweeps from latency-bound to capacity-bound. --sig-ms-intervals - 5 + 2 100 1000 ) diff --git a/tests/basicperf_locust.py b/tests/basicperf_locust.py index c2c5cb3bd2a7..8ca189690e68 100644 --- a/tests/basicperf_locust.py +++ b/tests/basicperf_locust.py @@ -287,7 +287,7 @@ def cli_args(): "Each is measured against its own network.", type=int, nargs="+", - default=[5, 100, 1000], + default=[2, 100, 1000], ) parser.add_argument( "--key-space-size", From 038d9825d9a949073512226f2fbabf5dd09310c9 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 19:15:31 +0100 Subject: [PATCH 11/13] Normalize new benchmarks against the branch's first run The previous version reported a benchmark absent from main by its value alone, marked (new), with no percentage. That read differently from every other axis, and the marker made the label long enough to be truncated, which is exactly what it should not have been for benchmarks whose names differ only in a suffix. Treat such a benchmark like any other axis instead, using this branch's earliest run as its reference in place of the main EWMA baseline, so it is normalized, labelled and coloured identically. The chart description records that the reference differs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- scripts/perf_compare_radar.py | 59 ++++++++++++----------------------- 1 file changed, 20 insertions(+), 39 deletions(-) diff --git a/scripts/perf_compare_radar.py b/scripts/perf_compare_radar.py index c1cad4e63038..49285c65f995 100644 --- a/scripts/perf_compare_radar.py +++ b/scripts/perf_compare_radar.py @@ -248,9 +248,6 @@ def axis_label_color(percent: float, higher_is_better: bool, within_noise: bool) return LABEL_GOOD if improved else LABEL_BAD -NEW_BENCHMARK_MARKER = "new" - - def shorten_label(label: str, max_length: int) -> str: """Shorten a label to fit, keeping both ends. @@ -269,24 +266,14 @@ def shorten_label(label: str, max_length: int) -> str: def axis_label( - benchmark: str, - value: float, - percent: float, - unit: str, - within_noise: bool, - is_new: bool = False, + benchmark: str, value: float, percent: float, unit: str, within_noise: bool ) -> str: """Shorten benchmark labels and include the branch real value and delta.""" label = SIG_MS_INTERVAL_RE.sub(r" \1", benchmark) - if is_new: - # There is no baseline to express a difference against, so report the - # value alone and say why. - suffix = f": {metric_label_value(value, unit)} ({NEW_BENCHMARK_MARKER})" - else: - suffix = ( - f": {metric_label_value(value, unit)} " - f"{format_delta_percent(percent, within_noise)}" - ) + suffix = ( + f": {metric_label_value(value, unit)} " + f"{format_delta_percent(percent, within_noise)}" + ) return f"{shorten_label(label, MAX_AXIS_LABEL_LENGTH - len(suffix))}{suffix}" @@ -347,13 +334,15 @@ def render_mermaid_radar_chart( if (value := metric_value(data, benchmark, metric)) is not None ] - # A benchmark added by this branch has no main history to compare - # against. Rather than drop it, which would make a new benchmark - # invisible on the very PR that adds it, plot it against this branch's - # own earliest run so that movement across branch runs is still - # visible, and mark it so it is not mistaken for a main comparison. - is_new = not main_values - if is_new: + # A benchmark added by this branch has no main runs to build a baseline + # from. Rather than drop it, which would make a new benchmark invisible + # on the very pull request which adds it, use this branch's own earliest + # run as the reference, so the axis is normalised and plotted exactly + # like every other one. + if main_values: + baseline = ewma(main_values) + sigma = statistics.pstdev(main_values) if len(main_values) > 1 else 0.0 + else: branch_values = [ value for data in branch_runs @@ -363,22 +352,15 @@ def render_mermaid_radar_chart( continue baseline = branch_values[0] sigma = 0.0 - else: - baseline = ewma(main_values) - sigma = statistics.pstdev(main_values) if len(main_values) > 1 else 0.0 if baseline <= 0: continue branch_percent = normalized_percent(branch_value, baseline) sigma_percent = normalized_percent(sigma, baseline) - # Without a main baseline there is no improvement or regression to - # report, so such an axis is never coloured as either. - within_noise = ( - True if is_new else within_noise_band(branch_percent, sigma_percent) - ) + within_noise = within_noise_band(branch_percent, sigma_percent) axes.append( - f"b{index}[{mermaid_label(axis_label(benchmark, branch_value, branch_percent, unit, within_noise, is_new))}]" + f"b{index}[{mermaid_label(axis_label(benchmark, branch_value, branch_percent, unit, within_noise))}]" ) axis_colors.append( axis_label_color(branch_percent, higher_better, within_noise) @@ -555,11 +537,10 @@ def render_comparison( ), "", ( - f"_A benchmark marked ({NEW_BENCHMARK_MARKER}) does not exist on `main` yet, " - "so it has no baseline to be compared against. It is plotted against this " - "branch's own earliest run instead, which shows how it moved across the " - "branch's runs but says nothing about `main`, and it is never coloured as " - "an improvement or a regression._" + "_A benchmark which does not exist on `main` yet has no baseline of its " + "own, so this branch's earliest run is used as its reference instead. " + "Its axis is normalized and plotted like any other, but the comparison " + "is against this branch rather than against `main`._" ), "", "", From 90fe7be9316b4f8ee78fa8b4a8dbebcba0992bc8 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 19:36:11 +0100 Subject: [PATCH 12/13] Elide the middles of words to fit long axis labels Truncating the middle of the whole label cut out several words at once. Elide word by word from the left instead, keeping each word's first and last letter and replacing the middle with a single ellipsis character, so a label degrades gradually and the last word, which is what distinguishes one setting of a benchmark from another, stays readable longest. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- scripts/perf_compare_radar.py | 46 +++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/scripts/perf_compare_radar.py b/scripts/perf_compare_radar.py index 49285c65f995..1c7212689427 100644 --- a/scripts/perf_compare_radar.py +++ b/scripts/perf_compare_radar.py @@ -248,21 +248,47 @@ def axis_label_color(percent: float, higher_is_better: bool, within_noise: bool) return LABEL_GOOD if improved else LABEL_BAD +# U+2026 HORIZONTAL ELLIPSIS. One character wide, so eliding the middle of a +# word with it costs a single column rather than the three of "...". +ELLIPSIS = "\u2026" + +# Eliding a shorter word saves nothing, since "abc" and "a" + ELLIPSIS + "c" +# are both three characters. +MIN_ELIDABLE_WORD_LENGTH = 4 + + +def elide_word(word: str) -> str: + """Replace the middle of a word with a single ellipsis character.""" + return f"{word[0]}{ELLIPSIS}{word[-1]}" + + def shorten_label(label: str, max_length: int) -> str: - """Shorten a label to fit, keeping both ends. + """Shorten a label to fit by eliding the middles of its words. - Benchmarks measured at several settings differ only in their suffix, for - example the interval in "Basic Blocking Locust 100ms", so truncating the - end would render such axes indistinguishable from one another. + Words are elided from left to right, each keeping its first and last letter, + until the label fits. Benchmarks measured at several settings differ only in + their last word, for example the interval in "Basic Blocking Locust 100ms", + so eliding from the left keeps the part which tells them apart readable for + as long as possible. """ if len(label) <= max_length: return label - if max_length <= 3: - return label[:max_length] - budget = max_length - 3 - head = budget // 2 - tail = budget - head - return f"{label[:head]}...{label[-tail:]}" + + words = label.split(" ") + for index, word in enumerate(words): + if len(word) < MIN_ELIDABLE_WORD_LENGTH: + continue + words[index] = elide_word(word) + elided = " ".join(words) + if len(elided) <= max_length: + return elided + + # Every word is elided and it still does not fit. Keep the end, which is + # what distinguishes one setting of a benchmark from another. + elided = " ".join(words) + if max_length <= 1: + return elided[:max_length] + return ELLIPSIS + elided[len(elided) - (max_length - 1) :] def axis_label( From dce652f5db1b262cd30860703b64b06d906c9117 Mon Sep 17 00:00:00 2001 From: Amaury Chamayou Date: Fri, 14 Aug 2026 19:56:25 +0100 Subject: [PATCH 13/13] Give new benchmarks a band like every other axis A benchmark with no main history had its standard deviation hardcoded to zero, so all four band curves collapsed to a single point at the baseline while every other axis carried a spread. That left a visible pinch in the band and, because nothing fell inside a zero-width noise threshold, any movement between branch runs was coloured as an improvement or a regression. Measure its spread the same way as for a benchmark with main history, across the runs available, which for such a benchmark are the branch's own. The radial zoom already covered these axes, since their values were always part of the data the scale is fitted to. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2da7bd10-ceb0-41a4-b6c4-d574595d91c9 --- scripts/perf_compare_radar.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/perf_compare_radar.py b/scripts/perf_compare_radar.py index 1c7212689427..d698093314af 100644 --- a/scripts/perf_compare_radar.py +++ b/scripts/perf_compare_radar.py @@ -377,7 +377,11 @@ def render_mermaid_radar_chart( if not branch_values: continue baseline = branch_values[0] - sigma = 0.0 + # Spread is measured the same way as for a benchmark with main + # history, from the runs available, so that such an axis carries a + # band and a noise threshold like every other one rather than + # collapsing to a point at the baseline. + sigma = statistics.pstdev(branch_values) if len(branch_values) > 1 else 0.0 if baseline <= 0: continue @@ -564,9 +568,10 @@ def render_comparison( "", ( "_A benchmark which does not exist on `main` yet has no baseline of its " - "own, so this branch's earliest run is used as its reference instead. " - "Its axis is normalized and plotted like any other, but the comparison " - "is against this branch rather than against `main`._" + "own, so this branch's earliest run is used as its reference and its band " + "is measured across this branch's runs. Its axis is normalized, scaled and " + "coloured like any other, but the comparison is against this branch rather " + "than against `main`._" ), "", "",