From a7c0f1b6fb946857b500d08ac16f1f1731a5f0f9 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 16 Jul 2026 09:40:04 +0000 Subject: [PATCH 1/3] acc: run ssh/connect-serverless-cpu locally Mirror the connect-serverless-gpu local conversion for the bare `ssh connect` (serverless CPU) invocation: run against the in-process testserver, which backs the driver-proxy /ssh websocket with a real sshd, so the run asserts both the submitted bootstrap job and a full handshake + remote exec over the ws:// tunnel. Linux-only and skipped when sshd is absent, matching the sibling ssh tests. Unlike the GPU test, the CPU invocation passes no --name, so the CLI derives the connection name from a hash of the workspace host. That host is a random localhost port locally, so a Repls entry normalizes databricks-cpu- to keep the submitted job payload deterministic. The cloud branch (Connection successful over real serverless CPU) is preserved for when cloud is re-enabled. --- .../ssh/connect-serverless-cpu/known_hosts | 1 + .../ssh/connect-serverless-cpu/out.test.toml | 7 +++- .../ssh/connect-serverless-cpu/output.txt | 37 +++++++++++++++++++ acceptance/ssh/connect-serverless-cpu/script | 31 ++++++++++++++-- .../ssh/connect-serverless-cpu/test.toml | 27 +++++++++++++- 5 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 acceptance/ssh/connect-serverless-cpu/known_hosts diff --git a/acceptance/ssh/connect-serverless-cpu/known_hosts b/acceptance/ssh/connect-serverless-cpu/known_hosts new file mode 100644 index 00000000000..197ec818881 --- /dev/null +++ b/acceptance/ssh/connect-serverless-cpu/known_hosts @@ -0,0 +1 @@ +# not actually checked by tests; accept-new appends the ephemeral host key here diff --git a/acceptance/ssh/connect-serverless-cpu/out.test.toml b/acceptance/ssh/connect-serverless-cpu/out.test.toml index 0a085fdfda4..4cea3439153 100644 --- a/acceptance/ssh/connect-serverless-cpu/out.test.toml +++ b/acceptance/ssh/connect-serverless-cpu/out.test.toml @@ -1,4 +1,7 @@ -Local = false -Cloud = true +Local = true +Cloud = false RequiresUnityCatalog = true +GOOS.darwin = false +GOOS.linux = true +GOOS.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/ssh/connect-serverless-cpu/output.txt b/acceptance/ssh/connect-serverless-cpu/output.txt index e69de29bb2d..84a12431491 100644 --- a/acceptance/ssh/connect-serverless-cpu/output.txt +++ b/acceptance/ssh/connect-serverless-cpu/output.txt @@ -0,0 +1,37 @@ + +>>> print_requests.py //api/2.2/jobs/runs/submit +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/submit", + "body": { + "environments": [ + { + "environment_key": "ssh_tunnel_serverless", + "spec": { + "environment_version": "4" + } + } + ], + "run_name": "ssh-server-bootstrap-[CPU_CONN]", + "tasks": [ + { + "environment_key": "ssh_tunnel_serverless", + "notebook_task": { + "base_parameters": { + "authorizedKeySecretName": "client-public-key", + "maxClients": "10", + "secretScopeName": "[USERNAME]-[CPU_CONN]-ssh-tunnel-keys", + "serverless": "true", + "sessionId": "[CPU_CONN]", + "shutdownDelay": "10m0s", + "version": "[CLI_VERSION]" + }, + "notebook_path": "/Workspace/Users/[USERNAME]/.databricks/ssh-tunnel/[CLI_VERSION]/[CPU_CONN]/ssh-server-bootstrap" + }, + "task_key": "start_ssh_server", + "timeout_seconds": 86400 + } + ], + "timeout_seconds": 86400 + } +} diff --git a/acceptance/ssh/connect-serverless-cpu/script b/acceptance/ssh/connect-serverless-cpu/script index 841cf780493..c4eeccc0eee 100644 --- a/acceptance/ssh/connect-serverless-cpu/script +++ b/acceptance/ssh/connect-serverless-cpu/script @@ -1,6 +1,29 @@ -errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr +if [ -z "${CLOUD_ENV:-}" ]; then + # Local runs have no release artifacts and never execute the uploaded + # binaries, so stand in empty archives (dev name omits the version). + mkdir -p releases + : >releases/databricks_cli_linux_amd64.zip + : >releases/databricks_cli_linux_arm64.zip + CLI_RELEASES_DIR="$PWD/releases" +fi -if ! grep -q "Connection successful" out.stdout.txt; then - run_id=$(cat LOG.stderr | grep -o "Job submitted successfully with run ID: [0-9]*" | grep -o "[0-9]*$") - trace $CLI jobs get-run "$run_id" > LOG.job +if [ -n "${CLOUD_ENV:-}" ]; then + # On cloud serverless CPU compute runs the remote command over a full SSH + # handshake; dump the run on failure. + errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr + if ! grep -q "Connection successful" out.stdout.txt; then + run_id=$(cat LOG.stderr | grep -o "Job submitted successfully with run ID: [0-9]*" | grep -o "[0-9]*$") + trace $CLI jobs get-run "$run_id" > LOG.job + fi +else + # No compute locally: the test server backs the driver-proxy /ssh websocket with a + # real sshd, so this one run asserts both the bootstrap job and a full handshake + + # remote exec over the ws:// tunnel. Needs sshd (only task test-exp-ssh provisions + # it); the test server probes the same way, so skip where it's absent. + if [ -z "$(command -v sshd || ls /usr/sbin/sshd /usr/local/sbin/sshd /sbin/sshd 2>/dev/null)" ]; then + echo "SKIP_TEST sshd (openssh-server) not installed" + exit 0 + fi + errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR --user-known-hosts-file=known_hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr + trace print_requests.py //api/2.2/jobs/runs/submit fi diff --git a/acceptance/ssh/connect-serverless-cpu/test.toml b/acceptance/ssh/connect-serverless-cpu/test.toml index c7dfb393289..fbe2631f4d1 100644 --- a/acceptance/ssh/connect-serverless-cpu/test.toml +++ b/acceptance/ssh/connect-serverless-cpu/test.toml @@ -1,8 +1,31 @@ -Local = false -Cloud = true +Local = true +Cloud = false # Serverless requires Unity Catalog RequiresUnityCatalog = true +# Assert the serverless CPU bootstrap job the CLI submits on a local run. +RecordRequests = true + +# Local runs stage stand-in release archives here; not part of the golden output. +Ignore = [ + "releases", +] + +# Linux-only: the local run drives a real sshd, reliable only there (and where +# task test-exp-ssh provisions it). Absent GOOS keys default to enabled, so the +# other OSes must be disabled explicitly. +[GOOS] + linux = true + darwin = false + windows = false + +# The bare `ssh connect` (no --name) derives the connection name from a hash of +# the workspace host, which is a random localhost port locally. Normalize it so +# the submitted job payload is deterministic. +[[Repls]] +Old = 'databricks-cpu-[0-9a-f]{8}' +New = "[CPU_CONN]" + [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] From ca973b0f66cde8bf27c87a53eb04c8bd0c0cd3f7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 16 Jul 2026 09:57:23 +0000 Subject: [PATCH 2/3] acc: tighten connect-serverless-cpu comments --- acceptance/ssh/connect-serverless-cpu/script | 13 +++++-------- acceptance/ssh/connect-serverless-cpu/test.toml | 15 ++++++--------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/acceptance/ssh/connect-serverless-cpu/script b/acceptance/ssh/connect-serverless-cpu/script index c4eeccc0eee..c2650ab3fec 100644 --- a/acceptance/ssh/connect-serverless-cpu/script +++ b/acceptance/ssh/connect-serverless-cpu/script @@ -1,6 +1,5 @@ if [ -z "${CLOUD_ENV:-}" ]; then - # Local runs have no release artifacts and never execute the uploaded - # binaries, so stand in empty archives (dev name omits the version). + # No release artifacts locally and they're never executed; stand in empty archives. mkdir -p releases : >releases/databricks_cli_linux_amd64.zip : >releases/databricks_cli_linux_arm64.zip @@ -8,18 +7,16 @@ if [ -z "${CLOUD_ENV:-}" ]; then fi if [ -n "${CLOUD_ENV:-}" ]; then - # On cloud serverless CPU compute runs the remote command over a full SSH - # handshake; dump the run on failure. + # Cloud: real serverless CPU runs the command; dump the run on failure. errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr if ! grep -q "Connection successful" out.stdout.txt; then run_id=$(cat LOG.stderr | grep -o "Job submitted successfully with run ID: [0-9]*" | grep -o "[0-9]*$") trace $CLI jobs get-run "$run_id" > LOG.job fi else - # No compute locally: the test server backs the driver-proxy /ssh websocket with a - # real sshd, so this one run asserts both the bootstrap job and a full handshake + - # remote exec over the ws:// tunnel. Needs sshd (only task test-exp-ssh provisions - # it); the test server probes the same way, so skip where it's absent. + # Local: the test server backs the /ssh websocket with a real sshd, so this + # asserts the bootstrap job plus a full handshake over the tunnel. Skip when + # sshd is absent (only task test-exp-ssh provisions it). if [ -z "$(command -v sshd || ls /usr/sbin/sshd /usr/local/sbin/sshd /sbin/sshd 2>/dev/null)" ]; then echo "SKIP_TEST sshd (openssh-server) not installed" exit 0 diff --git a/acceptance/ssh/connect-serverless-cpu/test.toml b/acceptance/ssh/connect-serverless-cpu/test.toml index fbe2631f4d1..ca00098972c 100644 --- a/acceptance/ssh/connect-serverless-cpu/test.toml +++ b/acceptance/ssh/connect-serverless-cpu/test.toml @@ -1,28 +1,25 @@ Local = true Cloud = false -# Serverless requires Unity Catalog +# Serverless requires Unity Catalog. RequiresUnityCatalog = true -# Assert the serverless CPU bootstrap job the CLI submits on a local run. +# Record requests so we can assert the bootstrap job the CLI submits. RecordRequests = true -# Local runs stage stand-in release archives here; not part of the golden output. +# Stand-in release archives staged by the local run; not golden output. Ignore = [ "releases", ] -# Linux-only: the local run drives a real sshd, reliable only there (and where -# task test-exp-ssh provisions it). Absent GOOS keys default to enabled, so the -# other OSes must be disabled explicitly. +# The local run drives a real sshd, reliable only on Linux. [GOOS] linux = true darwin = false windows = false -# The bare `ssh connect` (no --name) derives the connection name from a hash of -# the workspace host, which is a random localhost port locally. Normalize it so -# the submitted job payload is deterministic. +# No --name: the connection name is a hash of the (random) local host, so +# normalize it to keep the submitted job payload deterministic. [[Repls]] Old = 'databricks-cpu-[0-9a-f]{8}' New = "[CPU_CONN]" From 7ffc99662516c8add64cbea226ab958026243b4f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 17 Jul 2026 11:18:04 +0000 Subject: [PATCH 3/3] acc: address review comments on connect-serverless-cpu script Merge the two CLOUD_ENV conditionals so the local setup lives in one branch, replace the ': >file' idiom with touch, and title the core 'ssh connect' command so it surfaces in the golden output. --- .../ssh/connect-serverless-cpu/output.txt | 2 ++ acceptance/ssh/connect-serverless-cpu/script | 17 +++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/acceptance/ssh/connect-serverless-cpu/output.txt b/acceptance/ssh/connect-serverless-cpu/output.txt index 84a12431491..5519f43fc58 100644 --- a/acceptance/ssh/connect-serverless-cpu/output.txt +++ b/acceptance/ssh/connect-serverless-cpu/output.txt @@ -1,4 +1,6 @@ +=== CLI ssh connect (local sshd) + >>> print_requests.py //api/2.2/jobs/runs/submit { "method": "POST", diff --git a/acceptance/ssh/connect-serverless-cpu/script b/acceptance/ssh/connect-serverless-cpu/script index c2650ab3fec..40bbb6debc2 100644 --- a/acceptance/ssh/connect-serverless-cpu/script +++ b/acceptance/ssh/connect-serverless-cpu/script @@ -1,13 +1,6 @@ -if [ -z "${CLOUD_ENV:-}" ]; then - # No release artifacts locally and they're never executed; stand in empty archives. - mkdir -p releases - : >releases/databricks_cli_linux_amd64.zip - : >releases/databricks_cli_linux_arm64.zip - CLI_RELEASES_DIR="$PWD/releases" -fi - if [ -n "${CLOUD_ENV:-}" ]; then # Cloud: real serverless CPU runs the command; dump the run on failure. + title "CLI ssh connect (serverless CPU)\n" errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr if ! grep -q "Connection successful" out.stdout.txt; then run_id=$(cat LOG.stderr | grep -o "Job submitted successfully with run ID: [0-9]*" | grep -o "[0-9]*$") @@ -21,6 +14,14 @@ else echo "SKIP_TEST sshd (openssh-server) not installed" exit 0 fi + + # No release artifacts locally and they're never executed; stand in empty archives. + mkdir -p releases + touch releases/databricks_cli_linux_amd64.zip + touch releases/databricks_cli_linux_arm64.zip + CLI_RELEASES_DIR="$PWD/releases" + + title "CLI ssh connect (local sshd)\n" errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR --user-known-hosts-file=known_hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr trace print_requests.py //api/2.2/jobs/runs/submit fi