From eda8a74ec25591a555e6b66f725537f73a6aa664 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Fri, 21 Aug 2026 10:38:53 +0200 Subject: [PATCH 01/17] decrease --heavy_traffic_concurrent_workers for yugabyte --- build/dev/probe_locally.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/dev/probe_locally.sh b/build/dev/probe_locally.sh index 16e7ffaba..34e91fc56 100755 --- a/build/dev/probe_locally.sh +++ b/build/dev/probe_locally.sh @@ -59,6 +59,12 @@ if [[ "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_time_based_notification_index PROBER_EXTRA_ARGS+=(--scd-time-based-notification-index true) fi +# The heavy traffic concurrency tests are flaky when running against Yugabyte +# We reduce the concurrent worker count for that backend to 5 +if [[ "${COMPOSE_PROFILES:-}" == *"yugabyte"* ]]; then + PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 5) +fi + if ! docker run --rm --link "$OAUTH_CONTAINER":oauth \ --link "$CORE_SERVICE_CONTAINER":core-service \ --network dss_sandbox-default \ From 91f57ef5427868237ae7e4a5f6e316ae2f381eb0 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Fri, 21 Aug 2026 11:46:04 +0200 Subject: [PATCH 02/17] adjust doc and add flag --- build/dev/probe_locally.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build/dev/probe_locally.sh b/build/dev/probe_locally.sh index 34e91fc56..d5b33f898 100755 --- a/build/dev/probe_locally.sh +++ b/build/dev/probe_locally.sh @@ -59,10 +59,11 @@ if [[ "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_time_based_notification_index PROBER_EXTRA_ARGS+=(--scd-time-based-notification-index true) fi -# The heavy traffic concurrency tests are flaky when running against Yugabyte -# We reduce the concurrent worker count for that backend to 5 -if [[ "${COMPOSE_PROFILES:-}" == *"yugabyte"* ]]; then - PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 5) +# Following the extension of performance testing in https://github.com/interuss/dss/pull/1618 +# The Yugabyte test with the -enable_scd_hash_lock flag is flaky, so we reduce the +# concurrent worker count to 8 for that specific configuration. +if [[ "${COMPOSE_PROFILES:-}" == *"yugabyte"* && "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_scd_hash_lock"* ]]; then + PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 8) fi if ! docker run --rm --link "$OAUTH_CONTAINER":oauth \ From 673030f25bf0545d92442a4ad0ddd10bff90db03 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Fri, 21 Aug 2026 12:05:18 +0200 Subject: [PATCH 03/17] improve documentation --- build/dev/probe_locally.sh | 2 +- docs/operations/performances.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build/dev/probe_locally.sh b/build/dev/probe_locally.sh index d5b33f898..5fe59b4ba 100755 --- a/build/dev/probe_locally.sh +++ b/build/dev/probe_locally.sh @@ -60,7 +60,7 @@ if [[ "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_time_based_notification_index fi # Following the extension of performance testing in https://github.com/interuss/dss/pull/1618 -# The Yugabyte test with the -enable_scd_hash_lock flag is flaky, so we reduce the +# The Yugabyte test with the -enable_scd_hash_lock flag faces intermittent load-sensitive test failures, so we reduce the # concurrent worker count to 8 for that specific configuration. if [[ "${COMPOSE_PROFILES:-}" == *"yugabyte"* && "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_scd_hash_lock"* ]]; then PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 8) diff --git a/docs/operations/performances.md b/docs/operations/performances.md index fe7135134..95140ebaf 100644 --- a/docs/operations/performances.md +++ b/docs/operations/performances.md @@ -51,6 +51,8 @@ This should be better than global lock, as long as cells used don't collide and The number of locks (65535) is a compromise between lock contention (the more locks, the less unrelated cells share the same one) and the size of the `scd_locks` table. It is fixed and cannot be changed. +Note that since [#1618](https://github.com/interuss/dss/issues/1618), the `--heavy_traffic_concurrent_workers` flag needs to be reduced on the CI when testing with the `-enable_scd_hash_lock` configuration against Yugabyte due to intermittent load-sensitive test failures. + ## The time-based notification index option !!! danger From e1ac39fd756f7b08a9eba2c376e97df3f761afaf Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Fri, 21 Aug 2026 12:07:36 +0200 Subject: [PATCH 04/17] fix performances.md note --- docs/operations/performances.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/performances.md b/docs/operations/performances.md index 95140ebaf..0cf4c6e29 100644 --- a/docs/operations/performances.md +++ b/docs/operations/performances.md @@ -51,7 +51,7 @@ This should be better than global lock, as long as cells used don't collide and The number of locks (65535) is a compromise between lock contention (the more locks, the less unrelated cells share the same one) and the size of the `scd_locks` table. It is fixed and cannot be changed. -Note that since [#1618](https://github.com/interuss/dss/issues/1618), the `--heavy_traffic_concurrent_workers` flag needs to be reduced on the CI when testing with the `-enable_scd_hash_lock` configuration against Yugabyte due to intermittent load-sensitive test failures. +Note that this flag creates intermittent load-sensitive test failures when used with Yugabyte. See PR [#1659](https://github.com/interuss/dss/pull/1659) for more details. ## The time-based notification index option From d5364d0da656911e5aaa6ebe8a11b74234032b59 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Fri, 21 Aug 2026 12:17:09 +0200 Subject: [PATCH 05/17] decrease to 5 --- build/dev/probe_locally.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/dev/probe_locally.sh b/build/dev/probe_locally.sh index 5fe59b4ba..1e656a3c4 100755 --- a/build/dev/probe_locally.sh +++ b/build/dev/probe_locally.sh @@ -61,9 +61,9 @@ fi # Following the extension of performance testing in https://github.com/interuss/dss/pull/1618 # The Yugabyte test with the -enable_scd_hash_lock flag faces intermittent load-sensitive test failures, so we reduce the -# concurrent worker count to 8 for that specific configuration. +# concurrent worker count to 5 for that specific configuration. if [[ "${COMPOSE_PROFILES:-}" == *"yugabyte"* && "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_scd_hash_lock"* ]]; then - PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 8) + PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 5) fi if ! docker run --rm --link "$OAUTH_CONTAINER":oauth \ From db0a0f08be5c1b2df17ce5765395c689b8d295dd Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Mon, 31 Aug 2026 13:59:42 +0200 Subject: [PATCH 06/17] decrease to 2 --- build/dev/probe_locally.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/dev/probe_locally.sh b/build/dev/probe_locally.sh index 1e656a3c4..871b80ee2 100755 --- a/build/dev/probe_locally.sh +++ b/build/dev/probe_locally.sh @@ -61,9 +61,9 @@ fi # Following the extension of performance testing in https://github.com/interuss/dss/pull/1618 # The Yugabyte test with the -enable_scd_hash_lock flag faces intermittent load-sensitive test failures, so we reduce the -# concurrent worker count to 5 for that specific configuration. +# concurrent worker count to 2 for that specific configuration. if [[ "${COMPOSE_PROFILES:-}" == *"yugabyte"* && "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_scd_hash_lock"* ]]; then - PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 5) + PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 2) fi if ! docker run --rm --link "$OAUTH_CONTAINER":oauth \ From a3ae0e579298ad46e8455e2577844f09e00729c4 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Mon, 31 Aug 2026 14:13:31 +0200 Subject: [PATCH 07/17] restart From 257b312ea5dab04096f9f6d14fd4b50d84b7d03e Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Mon, 31 Aug 2026 14:16:56 +0200 Subject: [PATCH 08/17] decrease to 1 --- build/dev/probe_locally.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/dev/probe_locally.sh b/build/dev/probe_locally.sh index 871b80ee2..a4022d927 100755 --- a/build/dev/probe_locally.sh +++ b/build/dev/probe_locally.sh @@ -61,9 +61,9 @@ fi # Following the extension of performance testing in https://github.com/interuss/dss/pull/1618 # The Yugabyte test with the -enable_scd_hash_lock flag faces intermittent load-sensitive test failures, so we reduce the -# concurrent worker count to 2 for that specific configuration. +# concurrent worker count to 1 for that specific configuration. if [[ "${COMPOSE_PROFILES:-}" == *"yugabyte"* && "${CORE_SERVICE_EXTRA_FLAGS:-}" == *"-enable_scd_hash_lock"* ]]; then - PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 2) + PROBER_EXTRA_ARGS+=(--heavy_traffic_concurrent_workers 1) fi if ! docker run --rm --link "$OAUTH_CONTAINER":oauth \ From ed1e7cf928ed1e9a91d3d23c91b02f29ea76262f Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Mon, 31 Aug 2026 16:26:24 +0200 Subject: [PATCH 09/17] ci run 2 From 22ab871bb32d293576ee70e1b872e03809671654 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Mon, 31 Aug 2026 16:39:51 +0200 Subject: [PATCH 10/17] ci run 3 From 41be9dd5349447a169ea0e899bed39e2a9fc3200 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 2 Sep 2026 10:17:47 +0200 Subject: [PATCH 11/17] ci run 4 From 5d3386e4e5e8b158872a85ee2828b62f5098e947 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 2 Sep 2026 10:24:23 +0200 Subject: [PATCH 12/17] ci run 5 From aaab6ae896f08b3c7eada2d54e70c3228c4063ac Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 2 Sep 2026 10:32:45 +0200 Subject: [PATCH 13/17] ci run 6 From ba5fe7ee296a4c5670a9a0eeb90aa4250358a4a6 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 2 Sep 2026 10:41:18 +0200 Subject: [PATCH 14/17] ci run 7 From 709a2c5bcdd6d0472f12ab15441b4eebe2752f12 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 2 Sep 2026 11:15:17 +0200 Subject: [PATCH 15/17] ci run 8 From 5a347e77df904864182ac92333e7b20c9083b34d Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 2 Sep 2026 11:24:02 +0200 Subject: [PATCH 16/17] ci run 9 From 738d3735a084aeeb0189a858960a35bc9ebc7702 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 2 Sep 2026 12:12:58 +0200 Subject: [PATCH 17/17] ci run 10