Add continue_prev invariant test against latest released CLI

acceptance/acceptance_test.go

@@ -27,6 +27,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/google/uuid"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/databricks/cli/acceptance/internal"
 	"github.com/databricks/cli/internal/testutil"
@@ -311,9 +312,13 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		} else if UseVersion != "" {
 			version := UseVersion
 			if version == "latest" {
-				version = resolveLatestVersion(t, buildDir)
+				v, err := resolvePrevReleasedVersion(cwd)
+				require.NoError(t, err)
+				version = v
 			}
-			execPath = DownloadCLI(t, buildDir, version)
+			p, err := DownloadCLI(buildDir, version)
+			require.NoError(t, err)
+			execPath = p
 		} else {
 			execPath = BuildCLI(t, buildDir, coverDir, runtime.GOOS, runtime.GOARCH)
 		}
@@ -330,9 +335,34 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 	repls.SetPath(execPath, "[CLI]")
 
 	if !inprocessMode {
-		cli293Path := DownloadCLI(t, buildDir, "0.293.0")
+		prevVersion, err := resolvePrevReleasedVersion(cwd)
+		require.NoError(t, err)
+
+		// Download 0.293.0 and the N-2 released CLI concurrently: on a cold CI
+		// cache both downloads run in parallel; DownloadCLI's on-disk cache
+		// makes warm runs a no-op either way.
+		var (
+			cli293Path, cliPrevPath string
+			g                       errgroup.Group
+		)
+		g.Go(func() error {
+			p, err := DownloadCLI(buildDir, "0.293.0")
+			cli293Path = p
+			return err
+		})
+		g.Go(func() error {
+			p, err := DownloadCLI(buildDir, prevVersion)
+			cliPrevPath = p
+			return err
+		})
+		require.NoError(t, g.Wait())
+
 		t.Setenv("CLI_293", cli293Path)
 		repls.SetPath(cli293Path, "[CLI_293]")
+
+		t.Setenv("CLI_PREV", cliPrevPath)
+		repls.SetPath(cliPrevPath, "[CLI_PREV]")
+		repls.Set(prevVersion, "[CLI_PREV_VERSION]")
 	}
 
 	paths := []string{
@@ -1224,41 +1254,53 @@ func CreateReleaseArtifact(t *testing.T, cwd, releasesDir, coverDir, osName, arc
 	t.Logf("Created %s %s release: %s", osName, arch, zipPath)
 }
 
-// resolveLatestVersion returns the latest released CLI version (e.g. "0.293.0"),
-// using a file-based cache in buildDir valid for 1 hour.
-func resolveLatestVersion(t *testing.T, buildDir string) string {
-	cachePath := filepath.Join(buildDir, "latest_version.txt")
-	if info, err := os.Stat(cachePath); err == nil && time.Since(info.ModTime()) < time.Hour {
-		data, err := os.ReadFile(cachePath)
-		require.NoError(t, err)
-		if version := strings.TrimSpace(string(data)); version != "" {
-			return version
-		}
-	}
-
-	const url = "https://api.github.com/repos/databricks/cli/releases/latest"
-	resp, err := http.Get(url)
-	require.NoError(t, err)
-	defer resp.Body.Close()
-	require.Equal(t, http.StatusOK, resp.StatusCode, "failed to fetch %s: %s", url, resp.Status)
+// releaseHeaderRegexp matches "## Release vX.Y.Z" lines in CHANGELOG.md.
+var releaseHeaderRegexp = regexp.MustCompile(`^## Release v(\d+\.\d+\.\d+)`)
 
-	var release struct {
-		TagName string `json:"tag_name"`
+// resolvePrevReleasedVersion returns the second-most-recent released CLI
+// version from the repo's CHANGELOG.md (i.e. N-2 rather than N-1).
+//
+// Why N-2 and not N-1: the release workflow lands the CHANGELOG bump on main
+// *before* the release tag is pushed, so between those two events the top
+// entry names a version whose GitHub release archive does not exist yet.
+// Tests that run in that window would 404. Skipping one entry back guarantees
+// we always resolve to a version that has published binaries.
+//
+// Reading from CHANGELOG.md avoids a network call to api.github.com (rate
+// limits, CI reachability) and pins the version to the tree at HEAD so every
+// runner sees the same CLI_PREV.
+func resolvePrevReleasedVersion(cwd string) (string, error) {
+	path := filepath.Join(cwd, "..", "CHANGELOG.md")
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
 	}
-	require.NoError(t, json.NewDecoder(resp.Body).Decode(&release))
-	version := strings.TrimPrefix(release.TagName, "v")
-	require.NotEmpty(t, version, "empty tag_name in GitHub latest release response")
+	defer f.Close()
 
-	require.NoError(t, os.WriteFile(cachePath, []byte(version), 0o644))
-	return version
+	var versions []string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		if m := releaseHeaderRegexp.FindStringSubmatch(scanner.Text()); m != nil {
+			versions = append(versions, m[1])
+			if len(versions) == 2 {
+				return versions[1], nil
+			}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+	return "", fmt.Errorf("expected at least 2 release headers in %s, found %d", path, len(versions))
 }
 
 // DownloadCLI downloads a released CLI binary archive for the given version,
 // extracts the executable, and returns its path.
-func DownloadCLI(t *testing.T, buildDir, version string) string {
+func DownloadCLI(buildDir, version string) (string, error) {
 	// Prepare target directory for this version
 	versionDir := filepath.Join(buildDir, version)
-	require.NoError(t, os.MkdirAll(versionDir, 0o755))
+	if err := os.MkdirAll(versionDir, 0o755); err != nil {
+		return "", err
+	}
 
 	execName := "databricks"
 	if runtime.GOOS == "windows" {
@@ -1268,7 +1310,7 @@ func DownloadCLI(t *testing.T, buildDir, version string) string {
 
 	// If already downloaded, reuse
 	if _, err := os.Stat(execPath); err == nil {
-		return execPath
+		return execPath, nil
 	}
 
 	osName := runtime.GOOS
@@ -1279,55 +1321,85 @@ func DownloadCLI(t *testing.T, buildDir, version string) string {
 	url := fmt.Sprintf("https://github.com/databricks/cli/releases/download/v%s/%s", version, archiveName)
 	zipPath := filepath.Join(versionDir, archiveName)
 
-	downloadToFile(t, url, zipPath)
-	extractFileFromZip(t, zipPath, execName, versionDir)
+	if err := downloadToFile(url, zipPath); err != nil {
+		return "", err
+	}
+	if err := extractFileFromZip(zipPath, execName, versionDir); err != nil {
+		return "", err
+	}
 
-	return execPath
+	return execPath, nil
 }
 
 // downloadToFile downloads contents from url into the given destination path
-func downloadToFile(t *testing.T, url, destPath string) {
-	require.NoError(t, os.MkdirAll(filepath.Dir(destPath), 0o755))
+func downloadToFile(url, destPath string) error {
+	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
+		return err
+	}
 	out, err := os.Create(destPath)
-	require.NoError(t, err)
+	if err != nil {
+		return err
+	}
 	defer out.Close()
 
 	resp, err := http.Get(url)
-	require.NoError(t, err)
+	if err != nil {
+		return err
+	}
 	defer resp.Body.Close()
-	require.Equal(t, http.StatusOK, resp.StatusCode, "failed to download %s: %s", url, resp.Status)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to download %s: %s", url, resp.Status)
+	}
 
 	_, err = io.Copy(out, resp.Body)
-	require.NoError(t, err)
+	return err
 }
 
 // extractFileFromZip finds a file by name inside a zip archive and writes it
-// into destDir using the file's base name. Fails the test if not found.
-func extractFileFromZip(t *testing.T, zipPath, fileName, destDir string) {
+// into destDir using the file's base name. The destination is written atomically
+// (temp file + rename) so a failed extract can't leave a truncated binary that
+// DownloadCLI would later accept via os.Stat.
+func extractFileFromZip(zipPath, fileName, destDir string) error {
 	r, err := zip.OpenReader(zipPath)
-	require.NoError(t, err)
+	if err != nil {
+		return err
+	}
 	defer r.Close()
 
 	targetBase := filepath.Base(fileName)
 	destPath := filepath.Join(destDir, targetBase)
-	var found bool
 	for _, f := range r.File {
 		if filepath.Base(f.Name) != targetBase {
 			continue
 		}
 		rc, err := f.Open()
-		require.NoError(t, err)
-		defer rc.Close()
-
-		out, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755)
-		require.NoError(t, err)
-		_, err = io.Copy(out, rc)
-		_ = out.Close()
-		require.NoError(t, err)
-		found = true
-		break
+		if err != nil {
+			return err
+		}
+		tmp, err := os.CreateTemp(destDir, targetBase+".*.tmp")
+		if err != nil {
+			rc.Close()
+			return err
+		}
+		tmpPath := tmp.Name()
+		_, copyErr := io.Copy(tmp, rc)
+		closeErr := tmp.Close()
+		rc.Close()
+		if copyErr != nil || closeErr != nil {
+			os.Remove(tmpPath)
+			return errors.Join(copyErr, closeErr)
+		}
+		if err := os.Chmod(tmpPath, 0o755); err != nil {
+			os.Remove(tmpPath)
+			return err
+		}
+		if err := os.Rename(tmpPath, destPath); err != nil {
+			os.Remove(tmpPath)
+			return err
+		}
+		return nil
 	}
-	require.True(t, found, "file %s not found in archive %s", targetBase, zipPath)
+	return fmt.Errorf("file %s not found in archive %s", targetBase, zipPath)
 }
 
 func copyFile(src, dst string) error {

acceptance/bundle/invariant/continue_prev/output.txt

@@ -0,0 +1,4 @@
+
+>>> [CLI_PREV] --version
+Databricks CLI v[CLI_PREV_VERSION]
+INPUT_CONFIG_OK

acceptance/bundle/invariant/continue_prev/script

@@ -0,0 +1,41 @@
+# Invariant to test: current CLI can deploy on top of state produced by the previous release
+
+cp -r "$TESTDIR/../data/." . &> LOG.cp
+
+INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh"
+if [ -f "$INIT_SCRIPT" ]; then
+    source "$INIT_SCRIPT" &> LOG.init
+fi
+
+envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml
+
+cleanup() {
+    $CLI bundle destroy --auto-approve &> LOG.destroy
+    cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null
+
+    CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh"
+    if [ -f "$CLEANUP_SCRIPT" ]; then
+        source "$CLEANUP_SCRIPT" &> LOG.cleanup
+    fi
+}
+
+trap cleanup EXIT
+
+# Deploy with previous released CLI to produce previous-version state
+trace $CLI_PREV --version
+$CLI_PREV bundle deploy &> LOG.deploy.prev
+cat LOG.deploy.prev | contains.py '!panic' '!internal error' > /dev/null
+
+echo INPUT_CONFIG_OK
+
+# Deploy with current CLI on top of previous state
+$CLI bundle deploy &> LOG.deploy
+cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null
+
+# Verify no drift after current CLI deploy
+$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err
+cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null
+verify_no_drift.py LOG.planjson
+
+$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
+cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null

acceptance/bundle/invariant/continue_prev/test.toml

@@ -0,0 +1,13 @@
+# Computed volume_path (#5550) is not in CLI_PREV, so the seed deploy cannot
+# resolve ${resources.volumes.*.volume_path}. Remove once #5550 has shipped
+# in two releases (CLI_PREV is the N-2 release). Covered by no_drift on direct.
+EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"]
+
+# postgres_project.yml.tmpl sets enable_pg_native_login: false, which triggers a
+# state serialization bug on the previous CLI (fixed by #5782).
+# Remove once #5782 has shipped in two releases (CLI_PREV is the N-2 release).
+EnvMatrixExclude.no_postgres_project = ["INPUT_CONFIG=postgres_project.yml.tmpl"]
+
+# The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min
+# per variant (two full deploys at 1000 tasks) without incremental coverage.
+EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]