From b1ed3e045ded1b55f01f0d87eda4587be07bc37c Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Wed, 15 Jul 2026 19:20:23 +0300 Subject: [PATCH 01/13] implement memory monitoring for bench --- Client/bench.py | 68 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/Client/bench.py b/Client/bench.py index 33d7d7a46..4d3f4042d 100644 --- a/Client/bench.py +++ b/Client/bench.py @@ -33,7 +33,9 @@ import queue import re import subprocess -import sys +import threading +import time +import psutil ## Local imports must only use "import x", never "from x import ..." @@ -79,7 +81,7 @@ def single_core_bench(binary, outqueue): except: # Signal an error with (None, None) outqueue.put((None, None)) -def multi_core_bench(binary, threads): +def multi_core_bench(binary, threads, monitor_memory=False): outqueue = multiprocessing.Queue() @@ -92,26 +94,46 @@ def multi_core_bench(binary, threads): for process in processes: process.start() + stop_event = None + monitor = None + monitor_result = {'peak': 0} + + if monitor_memory: + pids = [process.pid for process in processes] + stop_event = threading.Event() + monitor = threading.Thread(target=monitor_peak_memory, args=(pids, stop_event, monitor_result)) + monitor.start() + try: # Every process deposits exactly one result into the Queue - return [outqueue.get(timeout=MAX_BENCH_TIME_SECONDS) for ii in range(threads)] + results = [outqueue.get(timeout=MAX_BENCH_TIME_SECONDS) for _ in range(threads)] + return results, monitor_result['peak'] if monitor_memory else None except queue.Empty: # Force kill the engine, thus causing the processes to finish utils.kill_process_by_name(binary) raise utils.OpenBenchBadBenchException('[%s] Bench Exceeded Max Duration' % (binary)) finally: # Join everything to avoid zombie processes + if monitor_memory: + stop_event.set() + monitor.join() for process in processes: process.join() -def run_benchmark(binary, threads, sets, expected=None): +def run_benchmark(binary, threads, sets, expected=None, monitor_memory=False): engine = os.path.basename(binary) + peak_memory = 0 benches, speeds = [], [] - for ii in range(sets): - for bench, speed in multi_core_bench(binary, threads): + for _ in range(sets): + results, peak_memory_run = multi_core_bench(binary, threads, monitor_memory) + + for bench, speed in results: benches.append(bench); speeds.append(speed) + if monitor_memory: + peak_memory = max(peak_memory, peak_memory_run) + if None in benches or None in speeds: raise utils.OpenBenchBadBenchException('[%s] Failed to Execute Benchmark' % (engine)) @@ -121,4 +143,36 @@ def run_benchmark(binary, threads, sets, expected=None): if expected and expected != benches[0]: raise utils.OpenBenchBadBenchException('[%s] Wrong Bench: %d' % (engine, benches[0])) - return sum(speeds) // len(speeds), benches[0] + return sum(speeds) // len(speeds), benches[0], peak_memory + +def sample_engine_memory(workers): + + total = 0 + for worker in workers: + try: # Engines are direct children of the multiprocessing workers + for engine in worker.children(recursive=True): + try: + info = engine.memory_full_info() + total += getattr(info, 'pss', info.uss) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + return total + +def monitor_peak_memory(worker_pids, stop_event, result): + + MEMORY_SAMPLE_SECONDS = 0.1 + + workers = [] + for pid in worker_pids: + try: workers.append(psutil.Process(pid)) + except psutil.NoSuchProcess: pass + + peak = 0 + while not stop_event.is_set(): + peak = max(peak, sample_engine_memory(workers)) + time.sleep(MEMORY_SAMPLE_SECONDS) + + result['peak'] = max(peak, sample_engine_memory(workers)) From 9cd6a4052d553c11b58c13ca5c445748c20b3709 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:30:57 +0300 Subject: [PATCH 02/13] add memory-limit option --- Client/worker.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/Client/worker.py b/Client/worker.py index 7f90ec4da..c1b3c28a8 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -104,16 +104,17 @@ def __init__(self, args): def process_args(self, args): # Extract all of the options - self.username = args.username - self.password = args.password - self.server = args.server - self.threads = int(args.threads) if args.threads != 'auto' else self.physical_cores - self.sockets = int(args.nsockets) - self.identity = args.identity if args.identity else 'None' - self.syzygy_path = args.syzygy if args.syzygy else None - self.fleet = args.fleet if args.fleet else False - self.noisy = args.noisy if args.noisy else False - self.focus = args.focus if args.focus else [] + self.username = args.username + self.password = args.password + self.server = args.server + self.threads = int(args.threads) if args.threads != 'auto' else self.physical_cores + self.sockets = int(args.nsockets) + self.identity = args.identity if args.identity else 'None' + self.syzygy_path = args.syzygy if args.syzygy else None + self.fleet = args.fleet if args.fleet else False + self.noisy = args.noisy if args.noisy else False + self.focus = args.focus if args.focus else [] + self.memory_limit = int(args.memory_limit) if args.memory_limit else None def check_requirements(self): @@ -155,6 +156,7 @@ def init_client(self): def validate_setup(self): + assert self.memory_limit is None or IS_LINUX assert self.threads >= self.sockets assert self.threads % self.sockets == 0 assert min(self.threads, self.sockets) >= 1 @@ -1205,7 +1207,7 @@ def safe_run_benchmarks(config, branch, engine): try: print('\nRunning %dx Benchmarks for %s' % (config.threads, name)) - speed, nodes = bench.run_benchmark(binary, config.threads, 1, expected) + speed, nodes, peak_memory = bench.run_benchmark(binary, config.threads, 1, expected, config.memory_limit) except utils.OpenBenchBadBenchException as error: ServerReporter.report_bad_bench(config, error.message) @@ -1213,6 +1215,11 @@ def safe_run_benchmarks(config, branch, engine): print('Bench for %s is %d' % (name, nodes)) print('Speed for %s is %d' % (name, speed)) + + if config.memory_limit: + megabyte = 1024 * 1024 + print('\nPeak memory for %s is %.2f MB' % (name, peak_memory / megabyte)) + return speed @@ -1307,13 +1314,14 @@ def parse_arguments(client_args): ) # Arguments specific to worker.py - p.add_argument('-T', '--threads' , help='Total Threads' , required=True ) - p.add_argument('-N', '--nsockets', help='Number of Sockets' , required=True ) - p.add_argument('-I', '--identity', help='Machine pseudonym' , required=False ) - p.add_argument( '--syzygy' , help='Syzygy WDL' , required=False ) - p.add_argument( '--fleet' , help='Fleet Mode' , action='store_true') - p.add_argument( '--noisy' , help='Reject time-based workloads' , action='store_true') - p.add_argument( '--focus' , help='Prefer certain engine(s)' , nargs='+' ) + p.add_argument('-T', '--threads' , help='Total Threads' , required=True ) + p.add_argument('-N', '--nsockets' , help='Number of Sockets' , required=True ) + p.add_argument('-I', '--identity' , help='Machine pseudonym' , required=False ) + p.add_argument( '--syzygy' , help='Syzygy WDL' , required=False ) + p.add_argument( '--fleet' , help='Fleet Mode' , action='store_true') + p.add_argument( '--noisy' , help='Reject time-based workloads' , action='store_true') + p.add_argument( '--focus' , help='Prefer certain engine(s)' , nargs='+' ) + p.add_argument( '--memory-limit', help='Memory limit in MB (Linux only)', required=False ) # Ignore unknown arguments ( from client ) worker_args, unknown = p.parse_known_args() From 1b57e96c5080a9109e959ac1686276aef1652731 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:01:01 +0300 Subject: [PATCH 03/13] fix result propagation --- Client/bench.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Client/bench.py b/Client/bench.py index 4d3f4042d..5c4a6e659 100644 --- a/Client/bench.py +++ b/Client/bench.py @@ -96,17 +96,16 @@ def multi_core_bench(binary, threads, monitor_memory=False): stop_event = None monitor = None - monitor_result = {'peak': 0} + monitor_result = {} if monitor_memory: - pids = [process.pid for process in processes] - stop_event = threading.Event() - monitor = threading.Thread(target=monitor_peak_memory, args=(pids, stop_event, monitor_result)) + pids = [process.pid for process in processes] + stop_event = threading.Event() + monitor = threading.Thread(target=monitor_peak_memory, args=(pids, stop_event, monitor_result)) monitor.start() try: # Every process deposits exactly one result into the Queue results = [outqueue.get(timeout=MAX_BENCH_TIME_SECONDS) for _ in range(threads)] - return results, monitor_result['peak'] if monitor_memory else None except queue.Empty: # Force kill the engine, thus causing the processes to finish utils.kill_process_by_name(binary) @@ -119,6 +118,8 @@ def multi_core_bench(binary, threads, monitor_memory=False): for process in processes: process.join() + return results, monitor_result.get('peak', 0) + def run_benchmark(binary, threads, sets, expected=None, monitor_memory=False): engine = os.path.basename(binary) From 371fe1a770e9d829fec1fe764b0ef25920e453d1 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:01:52 +0300 Subject: [PATCH 04/13] add a new exception type --- Client/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Client/utils.py b/Client/utils.py index b2ff2ad37..70d1defac 100644 --- a/Client/utils.py +++ b/Client/utils.py @@ -85,6 +85,11 @@ def __init__(self): self.message = '' super().__init__(self.message) +class OpenBenchInsufficientMemoryException(Exception): + def __init__(self, message): + self.message = message + super().__init__(self.message) + def kill_process_by_name(process_name): From f37e6b4bcfc6989dcea0c56158ed3d33020b78f4 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:09:28 +0300 Subject: [PATCH 05/13] estimate and blocklist if exceeds --- Client/worker.py | 49 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/Client/worker.py b/Client/worker.py index c1b3c28a8..997d53377 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -306,6 +306,17 @@ def report_bad_bench(config, error): return ServerReporter.report(config, 'clientBenchError', payload) + @staticmethod + def report_insufficient_memory(config, error): + + payload = { + 'test_id' : config.workload['test']['id'], + 'error' : error, + 'logs' : error, + } + + return ServerReporter.report(config, 'clientSubmitError', payload) + @staticmethod def report_results(config, batches): @@ -871,10 +882,21 @@ def find_pgn_error(reason, command): def determine_scale_factor(config, dev_name, base_name): # Run the benchmarks and compute the scaling NPS value - dev_nps = safe_run_benchmarks(config, 'dev' , dev_name ) - base_nps = safe_run_benchmarks(config, 'base', base_name) + dev_nps , dev_peak = safe_run_benchmarks(config, 'dev' , dev_name ) + base_nps, base_peak = safe_run_benchmarks(config, 'base', base_name) ServerReporter.report_nps(config, dev_nps, base_nps) + if config.memory_limit: + required = estimate_required_memory_mb(config, dev_peak, base_peak) + print('\nEstimated memory required: %.2f MB' % required) + + if required > config.memory_limit: + error = '[Error] Insufficient memory to run this Workload (required: %.2f MB, limit: %.2f MB)' % (required, config.memory_limit) + + config.blacklist.append(config.workload['test']['id']) + ServerReporter.report_insufficient_memory(config, error) + raise utils.OpenBenchInsufficientMemoryException(error) + dev_factor = base_factor = None # Scaling is only done relative to the Dev Engine @@ -898,6 +920,27 @@ def determine_scale_factor(config, dev_name, base_name): return factor +def estimate_required_memory_mb(config, dev_peak, base_peak): + + MEGABYTE = 1024 * 1024 + + # TODO: Actually get the default Hash from the engine + BENCH_HASH_MB = 16 + + MEMORY_RESERVED_BYTES = 1024 * MEGABYTE + MEMORY_SAFETY_FACTOR = 1.25 + + def estimate_memory_per_engine(branch, peak): + hash_mb = int(re.search(r'Hash=(\d+)', config.workload['test'][branch]['options']).group(1)) + hash_delta = max(0, hash_mb - BENCH_HASH_MB) * MEGABYTE + return peak / config.threads + hash_delta + + runner_cnt = config.workload['distribution']['runner-count'] + concurrency_per = config.workload['distribution']['concurrency-per'] + memory_per_pair = (estimate_memory_per_engine('dev', dev_peak) + estimate_memory_per_engine('base', base_peak)) + + return int(runner_cnt * concurrency_per * memory_per_pair * MEMORY_SAFETY_FACTOR + MEMORY_RESERVED_BYTES) // MEGABYTE + ## Functions interacting with the OpenBench server that establish the initial ## connection and then make simple requests to retrieve Workloads as json objects @@ -1220,7 +1263,7 @@ def safe_run_benchmarks(config, branch, engine): megabyte = 1024 * 1024 print('\nPeak memory for %s is %.2f MB' % (name, peak_memory / megabyte)) - return speed + return speed, peak_memory def build_runner_command(config, dev_cmd, base_cmd, scale_factor, timestamp, runner_idx): From ee5e8ca588f01db55c4548b9c1d97934d3e281e1 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:11:43 +0300 Subject: [PATCH 06/13] remove newline --- Client/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Client/worker.py b/Client/worker.py index 997d53377..d05b5e632 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -1261,7 +1261,7 @@ def safe_run_benchmarks(config, branch, engine): if config.memory_limit: megabyte = 1024 * 1024 - print('\nPeak memory for %s is %.2f MB' % (name, peak_memory / megabyte)) + print('Peak memory for %s is %.2f MB' % (name, peak_memory / megabyte)) return speed, peak_memory From a98b054c6c2a2e463c8ebd06e670b3b772251b3d Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:22:59 +0300 Subject: [PATCH 07/13] extract engine default hash --- Client/worker.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/Client/worker.py b/Client/worker.py index d05b5e632..703229fbd 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -887,7 +887,7 @@ def determine_scale_factor(config, dev_name, base_name): ServerReporter.report_nps(config, dev_nps, base_nps) if config.memory_limit: - required = estimate_required_memory_mb(config, dev_peak, base_peak) + required = estimate_required_memory_mb(config, dev_name, dev_peak, base_name, base_peak) print('\nEstimated memory required: %.2f MB' % required) if required > config.memory_limit: @@ -920,27 +920,41 @@ def determine_scale_factor(config, dev_name, base_name): return factor -def estimate_required_memory_mb(config, dev_peak, base_peak): +def estimate_required_memory_mb(config, dev_name, dev_peak, base_name, base_peak): MEGABYTE = 1024 * 1024 - # TODO: Actually get the default Hash from the engine - BENCH_HASH_MB = 16 - MEMORY_RESERVED_BYTES = 1024 * MEGABYTE MEMORY_SAFETY_FACTOR = 1.25 - def estimate_memory_per_engine(branch, peak): - hash_mb = int(re.search(r'Hash=(\d+)', config.workload['test'][branch]['options']).group(1)) - hash_delta = max(0, hash_mb - BENCH_HASH_MB) * MEGABYTE + def estimate_memory_per_engine(branch, engine, peak): + bench_hash = get_engine_default_hash(os.path.join('Engines', engine)) + workload_hash = int(re.search(r'Hash=(\d+)', config.workload['test'][branch]['options']).group(1)) + hash_delta = max(0, workload_hash - bench_hash) * MEGABYTE return peak / config.threads + hash_delta runner_cnt = config.workload['distribution']['runner-count'] concurrency_per = config.workload['distribution']['concurrency-per'] - memory_per_pair = (estimate_memory_per_engine('dev', dev_peak) + estimate_memory_per_engine('base', base_peak)) + memory_per_pair = (estimate_memory_per_engine('dev', dev_name, dev_peak) + estimate_memory_per_engine('base', base_name, base_peak)) return int(runner_cnt * concurrency_per * memory_per_pair * MEMORY_SAFETY_FACTOR + MEMORY_RESERVED_BYTES) // MEGABYTE +def get_engine_default_hash(binary): + + DEFAULT_BENCH_HASH_MB = 16 + + process = Popen(['./%s' % binary], stdin=PIPE, stdout=PIPE, stderr=STDOUT) + + try: + stdout = process.communicate(input=b'uci\nquit\n', timeout=10)[0].decode('utf-8', 'ignore') + match = re.search(r'option name Hash type spin default (\d+)', stdout, re.IGNORECASE) + return int(match.group(1)) + except subprocess.TimeoutExpired: + process.kill() + + print('[Warning] Unable to determine default Hash size for %s' % (binary)) + return DEFAULT_BENCH_HASH_MB + ## Functions interacting with the OpenBench server that establish the initial ## connection and then make simple requests to retrieve Workloads as json objects From 769102b27d5abc8a9eef9c56a7cf79d61c4bb1bb Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:44:24 +0300 Subject: [PATCH 08/13] extract MEGABYTE constant --- Client/worker.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Client/worker.py b/Client/worker.py index 703229fbd..0e6f5338a 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -69,6 +69,7 @@ IS_WINDOWS = platform.system() == 'Windows' # Don't touch this IS_LINUX = platform.system() != 'Windows' # Don't touch this +MEGABYTE = 1024 * 1024 class Configuration: @@ -922,8 +923,6 @@ def determine_scale_factor(config, dev_name, base_name): def estimate_required_memory_mb(config, dev_name, dev_peak, base_name, base_peak): - MEGABYTE = 1024 * 1024 - MEMORY_RESERVED_BYTES = 1024 * MEGABYTE MEMORY_SAFETY_FACTOR = 1.25 @@ -1274,8 +1273,7 @@ def safe_run_benchmarks(config, branch, engine): print('Speed for %s is %d' % (name, speed)) if config.memory_limit: - megabyte = 1024 * 1024 - print('Peak memory for %s is %.2f MB' % (name, peak_memory / megabyte)) + print('Peak memory for %s is %.2f MB' % (name, peak_memory / MEGABYTE)) return speed, peak_memory From 12e55d96276bceb6a9fd81f918ee9f2069c9117c Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:46:38 +0300 Subject: [PATCH 09/13] reorder a bit --- Client/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Client/worker.py b/Client/worker.py index 0e6f5338a..594adfaf7 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -110,12 +110,12 @@ def process_args(self, args): self.server = args.server self.threads = int(args.threads) if args.threads != 'auto' else self.physical_cores self.sockets = int(args.nsockets) + self.memory_limit = int(args.memory_limit) if args.memory_limit else None self.identity = args.identity if args.identity else 'None' self.syzygy_path = args.syzygy if args.syzygy else None self.fleet = args.fleet if args.fleet else False self.noisy = args.noisy if args.noisy else False self.focus = args.focus if args.focus else [] - self.memory_limit = int(args.memory_limit) if args.memory_limit else None def check_requirements(self): @@ -1372,11 +1372,11 @@ def parse_arguments(client_args): p.add_argument('-T', '--threads' , help='Total Threads' , required=True ) p.add_argument('-N', '--nsockets' , help='Number of Sockets' , required=True ) p.add_argument('-I', '--identity' , help='Machine pseudonym' , required=False ) + p.add_argument( '--memory-limit', help='Memory limit in MB (Linux only)', required=False ) p.add_argument( '--syzygy' , help='Syzygy WDL' , required=False ) p.add_argument( '--fleet' , help='Fleet Mode' , action='store_true') p.add_argument( '--noisy' , help='Reject time-based workloads' , action='store_true') p.add_argument( '--focus' , help='Prefer certain engine(s)' , nargs='+' ) - p.add_argument( '--memory-limit', help='Memory limit in MB (Linux only)', required=False ) # Ignore unknown arguments ( from client ) worker_args, unknown = p.parse_known_args() From 43ddaf179eaa6961808674be63627c0c93ae5fe4 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 19:51:21 +0300 Subject: [PATCH 10/13] normalize memory tracking to kbs --- Client/bench.py | 20 ++++++++++---------- Client/worker.py | 33 ++++++++++++++++----------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/Client/bench.py b/Client/bench.py index 5c4a6e659..462148e69 100644 --- a/Client/bench.py +++ b/Client/bench.py @@ -124,16 +124,16 @@ def run_benchmark(binary, threads, sets, expected=None, monitor_memory=False): engine = os.path.basename(binary) - peak_memory = 0 + peak_memory_kb = 0 benches, speeds = [], [] for _ in range(sets): - results, peak_memory_run = multi_core_bench(binary, threads, monitor_memory) + results, peak_memory_kb_run = multi_core_bench(binary, threads, monitor_memory) for bench, speed in results: benches.append(bench); speeds.append(speed) if monitor_memory: - peak_memory = max(peak_memory, peak_memory_run) + peak_memory_kb = max(peak_memory_kb, peak_memory_kb_run) if None in benches or None in speeds: raise utils.OpenBenchBadBenchException('[%s] Failed to Execute Benchmark' % (engine)) @@ -144,23 +144,23 @@ def run_benchmark(binary, threads, sets, expected=None, monitor_memory=False): if expected and expected != benches[0]: raise utils.OpenBenchBadBenchException('[%s] Wrong Bench: %d' % (engine, benches[0])) - return sum(speeds) // len(speeds), benches[0], peak_memory + return sum(speeds) // len(speeds), benches[0], peak_memory_kb def sample_engine_memory(workers): - total = 0 + peak_kb = 0 for worker in workers: try: # Engines are direct children of the multiprocessing workers for engine in worker.children(recursive=True): try: info = engine.memory_full_info() - total += getattr(info, 'pss', info.uss) + peak_kb += getattr(info, 'pss', info.uss) // 1024 except (psutil.NoSuchProcess, psutil.AccessDenied): pass except (psutil.NoSuchProcess, psutil.AccessDenied): pass - return total + return peak_kb def monitor_peak_memory(worker_pids, stop_event, result): @@ -171,9 +171,9 @@ def monitor_peak_memory(worker_pids, stop_event, result): try: workers.append(psutil.Process(pid)) except psutil.NoSuchProcess: pass - peak = 0 + peak_kb = 0 while not stop_event.is_set(): - peak = max(peak, sample_engine_memory(workers)) + peak_kb = max(peak_kb, sample_engine_memory(workers)) time.sleep(MEMORY_SAMPLE_SECONDS) - result['peak'] = max(peak, sample_engine_memory(workers)) + result['peak'] = max(peak_kb, sample_engine_memory(workers)) diff --git a/Client/worker.py b/Client/worker.py index 594adfaf7..738261d8e 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -69,8 +69,6 @@ IS_WINDOWS = platform.system() == 'Windows' # Don't touch this IS_LINUX = platform.system() != 'Windows' # Don't touch this -MEGABYTE = 1024 * 1024 - class Configuration: ## Handles configuring the worker with the server. This means collecting @@ -888,11 +886,11 @@ def determine_scale_factor(config, dev_name, base_name): ServerReporter.report_nps(config, dev_nps, base_nps) if config.memory_limit: - required = estimate_required_memory_mb(config, dev_name, dev_peak, base_name, base_peak) - print('\nEstimated memory required: %.2f MB' % required) + required_mb = estimate_required_memory_kb(config, dev_name, dev_peak, base_name, base_peak) / 1024 + print('\nEstimated memory required: %.2f MB' % required_mb) - if required > config.memory_limit: - error = '[Error] Insufficient memory to run this Workload (required: %.2f MB, limit: %.2f MB)' % (required, config.memory_limit) + if required_mb > config.memory_limit: + error = '[Error] Insufficient memory to run this Workload (required: %.2f MB, limit: %.2f MB)' % (required_mb, config.memory_limit) config.blacklist.append(config.workload['test']['id']) ServerReporter.report_insufficient_memory(config, error) @@ -921,22 +919,23 @@ def determine_scale_factor(config, dev_name, base_name): return factor -def estimate_required_memory_mb(config, dev_name, dev_peak, base_name, base_peak): +def estimate_required_memory_kb(config, dev_name, dev_peak, base_name, base_peak): - MEMORY_RESERVED_BYTES = 1024 * MEGABYTE + # Reserve 1GB for Python, Fastchess, etc. + MEMORY_RESERVED_KB = 1024**2 MEMORY_SAFETY_FACTOR = 1.25 def estimate_memory_per_engine(branch, engine, peak): bench_hash = get_engine_default_hash(os.path.join('Engines', engine)) workload_hash = int(re.search(r'Hash=(\d+)', config.workload['test'][branch]['options']).group(1)) - hash_delta = max(0, workload_hash - bench_hash) * MEGABYTE - return peak / config.threads + hash_delta + hash_delta = max(0, workload_hash - bench_hash) + return peak / config.threads + 1024 * hash_delta - runner_cnt = config.workload['distribution']['runner-count'] - concurrency_per = config.workload['distribution']['concurrency-per'] - memory_per_pair = (estimate_memory_per_engine('dev', dev_name, dev_peak) + estimate_memory_per_engine('base', base_name, base_peak)) + runner_cnt = config.workload['distribution']['runner-count'] + concurrency_per = config.workload['distribution']['concurrency-per'] + memory_per_pair_kb = (estimate_memory_per_engine('dev', dev_name, dev_peak) + estimate_memory_per_engine('base', base_name, base_peak)) - return int(runner_cnt * concurrency_per * memory_per_pair * MEMORY_SAFETY_FACTOR + MEMORY_RESERVED_BYTES) // MEGABYTE + return int(runner_cnt * concurrency_per * memory_per_pair_kb * MEMORY_SAFETY_FACTOR + MEMORY_RESERVED_KB) def get_engine_default_hash(binary): @@ -1263,7 +1262,7 @@ def safe_run_benchmarks(config, branch, engine): try: print('\nRunning %dx Benchmarks for %s' % (config.threads, name)) - speed, nodes, peak_memory = bench.run_benchmark(binary, config.threads, 1, expected, config.memory_limit) + speed, nodes, peak_memory_kb = bench.run_benchmark(binary, config.threads, 1, expected, config.memory_limit) except utils.OpenBenchBadBenchException as error: ServerReporter.report_bad_bench(config, error.message) @@ -1273,9 +1272,9 @@ def safe_run_benchmarks(config, branch, engine): print('Speed for %s is %d' % (name, speed)) if config.memory_limit: - print('Peak memory for %s is %.2f MB' % (name, peak_memory / MEGABYTE)) + print('Peak memory for %s is %.2f MB' % (name, peak_memory_kb / 1024)) - return speed, peak_memory + return speed, peak_memory_kb def build_runner_command(config, dev_cmd, base_cmd, scale_factor, timestamp, runner_idx): From e91a7e3da61ffc68330f522fc3854cc05f5f9838 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 19:58:18 +0300 Subject: [PATCH 11/13] wire it up to bench_engine.py --- Scripts/bench_engine.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Scripts/bench_engine.py b/Scripts/bench_engine.py index 544f3d205..2acaab0c3 100755 --- a/Scripts/bench_engine.py +++ b/Scripts/bench_engine.py @@ -36,9 +36,13 @@ p.add_argument('-E', '--engine' , help='Relative path to Binary', required=True) p.add_argument('-T', '--threads' , help='Concurrent Benchmarks', required=True, type=int) p.add_argument('-S', '--sets' , help='Benchmark Sample Count', required=True, type=int) + p.add_argument('-M', '--memory' , help='Sample peak memory', action='store_true') args = p.parse_args() - speed, bench = run_benchmark(args.engine, args.threads, args.sets) + speed, bench, peak_kb = run_benchmark(args.engine, args.threads, args.sets, monitor_memory=args.memory) print('Bench for %s is %d' % (args.engine, bench)) print('Speed for %s is %d' % (args.engine, speed)) + + if args.memory: + print('Peak memory for %s is %.2f MB' % (args.engine, peak_kb / 1024)) From a1ca90f852537fe24c91c215edef9488e2e8758e Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:10:33 +0300 Subject: [PATCH 12/13] wire it up to bench_all.py --- Scripts/bench_all.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Scripts/bench_all.py b/Scripts/bench_all.py index 0c247e3c2..75826bf73 100755 --- a/Scripts/bench_all.py +++ b/Scripts/bench_all.py @@ -91,6 +91,7 @@ def get_engine(engine, config): parser.add_argument('--engines', help='List of specific engines', nargs='+') parser.add_argument('--threads', help='Concurrent Benchmarks', required=True, type=int) parser.add_argument('--sets' , help='Benchmark Sample Count', required=True, type=int) + parser.add_argument('--memory' , help='Sample peak memory', action='store_true') args = credentialed_cmdline_args(parser) # Get the build info, and default network info, for all applicable engines @@ -121,6 +122,7 @@ def get_engine(engine, config): # Pretty Formatting max_length = max(len(engine) for engine in engines) print_format = '%-' + str(max_length) + 's %8d nps %10d nodes in %6.3f seconds' + print_format_mem = '%-' + str(max_length) + 's %8d nps %10d nodes in %6.3f seconds and %6.2f MB peak memory' for engine in engines: @@ -133,8 +135,11 @@ def get_engine(engine, config): continue try: - nps, nodes = run_benchmark(binary, args.threads, args.sets) - print (print_format % (engine, nps, nodes, nodes / max(1e-6, nps))) + nps, nodes, peak_kb = run_benchmark(binary, args.threads, args.sets, monitor_memory=args.memory) + if args.memory: + print (print_format_mem % (engine, nps, nodes, nodes / max(1e-6, nps), peak_kb / 1024)) + else: + print (print_format % (engine, nps, nodes, nodes / max(1e-6, nps))) except OpenBenchBadBenchException as error: print ('%s: %s' % (engine, error)) From d5012015c1c9e2f6c591090e63cb305484ab4dd4 Mon Sep 17 00:00:00 2001 From: Arseniy Surkov <93079612+codedeliveryservice@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:14:20 +0300 Subject: [PATCH 13/13] minor cleanups --- Client/bench.py | 4 ++-- Client/worker.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Client/bench.py b/Client/bench.py index 462148e69..b4a4eaf7b 100644 --- a/Client/bench.py +++ b/Client/bench.py @@ -127,13 +127,13 @@ def run_benchmark(binary, threads, sets, expected=None, monitor_memory=False): peak_memory_kb = 0 benches, speeds = [], [] for _ in range(sets): - results, peak_memory_kb_run = multi_core_bench(binary, threads, monitor_memory) + results, peak_memory_run_kb = multi_core_bench(binary, threads, monitor_memory) for bench, speed in results: benches.append(bench); speeds.append(speed) if monitor_memory: - peak_memory_kb = max(peak_memory_kb, peak_memory_kb_run) + peak_memory_kb = max(peak_memory_kb, peak_memory_run_kb) if None in benches or None in speeds: raise utils.OpenBenchBadBenchException('[%s] Failed to Execute Benchmark' % (engine)) diff --git a/Client/worker.py b/Client/worker.py index 738261d8e..7a7d9f675 100644 --- a/Client/worker.py +++ b/Client/worker.py @@ -923,19 +923,19 @@ def estimate_required_memory_kb(config, dev_name, dev_peak, base_name, base_peak # Reserve 1GB for Python, Fastchess, etc. MEMORY_RESERVED_KB = 1024**2 - MEMORY_SAFETY_FACTOR = 1.25 + MEMORY_OVERHEAD_FACTOR = 1.25 def estimate_memory_per_engine(branch, engine, peak): bench_hash = get_engine_default_hash(os.path.join('Engines', engine)) workload_hash = int(re.search(r'Hash=(\d+)', config.workload['test'][branch]['options']).group(1)) - hash_delta = max(0, workload_hash - bench_hash) - return peak / config.threads + 1024 * hash_delta + hash_delta = 1024 * max(0, workload_hash - bench_hash) + return peak / config.threads + hash_delta runner_cnt = config.workload['distribution']['runner-count'] concurrency_per = config.workload['distribution']['concurrency-per'] memory_per_pair_kb = (estimate_memory_per_engine('dev', dev_name, dev_peak) + estimate_memory_per_engine('base', base_name, base_peak)) - return int(runner_cnt * concurrency_per * memory_per_pair_kb * MEMORY_SAFETY_FACTOR + MEMORY_RESERVED_KB) + return int(runner_cnt * concurrency_per * memory_per_pair_kb * MEMORY_OVERHEAD_FACTOR + MEMORY_RESERVED_KB) def get_engine_default_hash(binary):