diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e5781f88f4..e89a635fe0 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -27,6 +27,7 @@ import ( "unicode/utf8" "github.com/google/uuid" + "golang.org/x/sync/errgroup" "github.com/databricks/cli/acceptance/internal" "github.com/databricks/cli/internal/build" @@ -322,9 +323,13 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { } else if UseVersion != "" { version := UseVersion if version == "latest" { - version = resolveLatestVersion(t, buildDir) + v, err := resolvePrevReleasedVersion(cwd) + require.NoError(t, err) + version = v } - execPath = DownloadCLI(t, buildDir, version) + p, err := DownloadCLI(buildDir, version) + require.NoError(t, err) + execPath = p // For a downloaded release the version string is already known. cliVersion = version } else { @@ -350,9 +355,34 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { repls.SetPath(execPath, "[CLI]") if !inprocessMode { - cli293Path := DownloadCLI(t, buildDir, "0.293.0") + prevVersion, err := resolvePrevReleasedVersion(cwd) + require.NoError(t, err) + + // Download 0.293.0 and the N-2 released CLI concurrently: on a cold CI + // cache both downloads run in parallel; DownloadCLI's on-disk cache + // makes warm runs a no-op either way. + var ( + cli293Path, cliPrevPath string + g errgroup.Group + ) + g.Go(func() error { + p, err := DownloadCLI(buildDir, "0.293.0") + cli293Path = p + return err + }) + g.Go(func() error { + p, err := DownloadCLI(buildDir, prevVersion) + cliPrevPath = p + return err + }) + require.NoError(t, g.Wait()) + t.Setenv("CLI_293", cli293Path) repls.SetPath(cli293Path, "[CLI_293]") + + t.Setenv("CLI_PREV", cliPrevPath) + repls.SetPath(cliPrevPath, "[CLI_PREV]") + repls.Set(prevVersion, "[CLI_PREV_VERSION]") } paths := []string{ @@ -1301,41 +1331,53 @@ func CreateReleaseArtifact(t *testing.T, cwd, releasesDir, coverDir, osName, arc t.Logf("Created %s %s release: %s", osName, arch, zipPath) } -// resolveLatestVersion returns the latest released CLI version (e.g. "0.293.0"), -// using a file-based cache in buildDir valid for 1 hour. -func resolveLatestVersion(t *testing.T, buildDir string) string { - cachePath := filepath.Join(buildDir, "latest_version.txt") - if info, err := os.Stat(cachePath); err == nil && time.Since(info.ModTime()) < time.Hour { - data, err := os.ReadFile(cachePath) - require.NoError(t, err) - if version := strings.TrimSpace(string(data)); version != "" { - return version - } - } - - const url = "https://api.github.com/repos/databricks/cli/releases/latest" - resp, err := http.Get(url) - require.NoError(t, err) - defer resp.Body.Close() - require.Equal(t, http.StatusOK, resp.StatusCode, "failed to fetch %s: %s", url, resp.Status) +// releaseHeaderRegexp matches "## Release vX.Y.Z" lines in CHANGELOG.md. +var releaseHeaderRegexp = regexp.MustCompile(`^## Release v(\d+\.\d+\.\d+)`) - var release struct { - TagName string `json:"tag_name"` +// resolvePrevReleasedVersion returns the second-most-recent released CLI +// version from the repo's CHANGELOG.md (i.e. N-2 rather than N-1). +// +// Why N-2 and not N-1: the release workflow lands the CHANGELOG bump on main +// *before* the release tag is pushed, so between those two events the top +// entry names a version whose GitHub release archive does not exist yet. +// Tests that run in that window would 404. Skipping one entry back guarantees +// we always resolve to a version that has published binaries. +// +// Reading from CHANGELOG.md avoids a network call to api.github.com (rate +// limits, CI reachability) and pins the version to the tree at HEAD so every +// runner sees the same CLI_PREV. +func resolvePrevReleasedVersion(cwd string) (string, error) { + path := filepath.Join(cwd, "..", "CHANGELOG.md") + f, err := os.Open(path) + if err != nil { + return "", err } - require.NoError(t, json.NewDecoder(resp.Body).Decode(&release)) - version := strings.TrimPrefix(release.TagName, "v") - require.NotEmpty(t, version, "empty tag_name in GitHub latest release response") + defer f.Close() - require.NoError(t, os.WriteFile(cachePath, []byte(version), 0o644)) - return version + var versions []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if m := releaseHeaderRegexp.FindStringSubmatch(scanner.Text()); m != nil { + versions = append(versions, m[1]) + if len(versions) == 2 { + return versions[1], nil + } + } + } + if err := scanner.Err(); err != nil { + return "", err + } + return "", fmt.Errorf("expected at least 2 release headers in %s, found %d", path, len(versions)) } // DownloadCLI downloads a released CLI binary archive for the given version, // extracts the executable, and returns its path. -func DownloadCLI(t *testing.T, buildDir, version string) string { +func DownloadCLI(buildDir, version string) (string, error) { // Prepare target directory for this version versionDir := filepath.Join(buildDir, version) - require.NoError(t, os.MkdirAll(versionDir, 0o755)) + if err := os.MkdirAll(versionDir, 0o755); err != nil { + return "", err + } execName := "databricks" if runtime.GOOS == "windows" { @@ -1345,7 +1387,7 @@ func DownloadCLI(t *testing.T, buildDir, version string) string { // If already downloaded, reuse if _, err := os.Stat(execPath); err == nil { - return execPath + return execPath, nil } osName := runtime.GOOS @@ -1356,55 +1398,85 @@ func DownloadCLI(t *testing.T, buildDir, version string) string { url := fmt.Sprintf("https://github.com/databricks/cli/releases/download/v%s/%s", version, archiveName) zipPath := filepath.Join(versionDir, archiveName) - downloadToFile(t, url, zipPath) - extractFileFromZip(t, zipPath, execName, versionDir) + if err := downloadToFile(url, zipPath); err != nil { + return "", err + } + if err := extractFileFromZip(zipPath, execName, versionDir); err != nil { + return "", err + } - return execPath + return execPath, nil } // downloadToFile downloads contents from url into the given destination path -func downloadToFile(t *testing.T, url, destPath string) { - require.NoError(t, os.MkdirAll(filepath.Dir(destPath), 0o755)) +func downloadToFile(url, destPath string) error { + if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil { + return err + } out, err := os.Create(destPath) - require.NoError(t, err) + if err != nil { + return err + } defer out.Close() resp, err := http.Get(url) - require.NoError(t, err) + if err != nil { + return err + } defer resp.Body.Close() - require.Equal(t, http.StatusOK, resp.StatusCode, "failed to download %s: %s", url, resp.Status) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download %s: %s", url, resp.Status) + } _, err = io.Copy(out, resp.Body) - require.NoError(t, err) + return err } // extractFileFromZip finds a file by name inside a zip archive and writes it -// into destDir using the file's base name. Fails the test if not found. -func extractFileFromZip(t *testing.T, zipPath, fileName, destDir string) { +// into destDir using the file's base name. The destination is written atomically +// (temp file + rename) so a failed extract can't leave a truncated binary that +// DownloadCLI would later accept via os.Stat. +func extractFileFromZip(zipPath, fileName, destDir string) error { r, err := zip.OpenReader(zipPath) - require.NoError(t, err) + if err != nil { + return err + } defer r.Close() targetBase := filepath.Base(fileName) destPath := filepath.Join(destDir, targetBase) - var found bool for _, f := range r.File { if filepath.Base(f.Name) != targetBase { continue } rc, err := f.Open() - require.NoError(t, err) - defer rc.Close() - - out, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) - require.NoError(t, err) - _, err = io.Copy(out, rc) - _ = out.Close() - require.NoError(t, err) - found = true - break + if err != nil { + return err + } + tmp, err := os.CreateTemp(destDir, targetBase+".*.tmp") + if err != nil { + rc.Close() + return err + } + tmpPath := tmp.Name() + _, copyErr := io.Copy(tmp, rc) + closeErr := tmp.Close() + rc.Close() + if copyErr != nil || closeErr != nil { + os.Remove(tmpPath) + return errors.Join(copyErr, closeErr) + } + if err := os.Chmod(tmpPath, 0o755); err != nil { + os.Remove(tmpPath) + return err + } + if err := os.Rename(tmpPath, destPath); err != nil { + os.Remove(tmpPath) + return err + } + return nil } - require.True(t, found, "file %s not found in archive %s", targetBase, zipPath) + return fmt.Errorf("file %s not found in archive %s", targetBase, zipPath) } func copyFile(src, dst string) error { diff --git a/acceptance/bundle/invariant/continue_prev/out.test.toml b/acceptance/bundle/invariant/continue_prev/out.test.toml new file mode 100644 index 0000000000..93f3cede4e --- /dev/null +++ b/acceptance/bundle/invariant/continue_prev/out.test.toml @@ -0,0 +1,55 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/continue_prev/output.txt b/acceptance/bundle/invariant/continue_prev/output.txt new file mode 100644 index 0000000000..20e01313be --- /dev/null +++ b/acceptance/bundle/invariant/continue_prev/output.txt @@ -0,0 +1,4 @@ + +>>> [CLI_PREV] --version +Databricks CLI v[CLI_PREV_VERSION] +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/continue_prev/script b/acceptance/bundle/invariant/continue_prev/script new file mode 100644 index 0000000000..908792643b --- /dev/null +++ b/acceptance/bundle/invariant/continue_prev/script @@ -0,0 +1,41 @@ +# Invariant to test: current CLI can deploy on top of state produced by the previous release + +cp -r "$TESTDIR/../data/." . &> LOG.cp + +INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" +if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init +fi + +envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml + +cleanup() { + $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + + CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi +} + +trap cleanup EXIT + +# Deploy with previous released CLI to produce previous-version state +trace $CLI_PREV --version +$CLI_PREV bundle deploy &> LOG.deploy.prev +cat LOG.deploy.prev | contains.py '!panic' '!internal error' > /dev/null + +echo INPUT_CONFIG_OK + +# Deploy with current CLI on top of previous state +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + +# Verify no drift after current CLI deploy +$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err +cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null +verify_no_drift.py LOG.planjson + +$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan +cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null diff --git a/acceptance/bundle/invariant/continue_prev/test.toml b/acceptance/bundle/invariant/continue_prev/test.toml new file mode 100644 index 0000000000..18e13c56f8 --- /dev/null +++ b/acceptance/bundle/invariant/continue_prev/test.toml @@ -0,0 +1,13 @@ +# Computed volume_path (#5550) is not in CLI_PREV, so the seed deploy cannot +# resolve ${resources.volumes.*.volume_path}. Remove once #5550 has shipped +# in two releases (CLI_PREV is the N-2 release). Covered by no_drift on direct. +EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"] + +# postgres_project.yml.tmpl sets enable_pg_native_login: false, which triggers a +# state serialization bug on the previous CLI (fixed by #5782). +# Remove once #5782 has shipped in two releases (CLI_PREV is the N-2 release). +EnvMatrixExclude.no_postgres_project = ["INPUT_CONFIG=postgres_project.yml.tmpl"] + +# The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min +# per variant (two full deploys at 1000 tasks) without incremental coverage. +EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]