From 223d3a09267c2c06ac41835e3647809ab7b99dd6 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Wed, 12 Aug 2026 16:44:09 +0200 Subject: [PATCH 1/4] localenv: write [tool.databricks.environment] version on serverless setup-local When `environments setup-local` provisions or regenerates pyproject.toml against a serverless target, write a `[tool.databricks.environment]` section carrying `environment_version` (the resolved serverless version). This lets the same project run interactively, in bundles, and in serverless jobs from one source of truth. The section is env-owned: it is refreshed in place on regeneration (preserving any inline comment and other user keys in the table) via the existing formatting-preserving merge, and appended when absent. Cluster targets leave the version empty, so the section is never written and any existing one is left untouched. DECO-27998 Co-authored-by: Isaac --- .../cli/setup-local-environment-version.md | 1 + .../localenv/constraints-only/output.txt | 2 +- .../localenv/job-serverless-check/output.txt | 1 + .../localenv/job-task-foreach/output.txt | 1 + .../localenv/merge-warnings-json/output.txt | 6 +- acceptance/localenv/merge-warnings/output.txt | 1 + .../localenv/serverless-check/output.txt | 1 + .../serverless-json-major-pin/output.txt | 2 +- .../localenv/serverless-json/output.txt | 2 +- libs/localenv/constraints.go | 5 + libs/localenv/merge.go | 75 +++++++++++-- libs/localenv/merge_test.go | 101 ++++++++++++++++++ libs/localenv/pipeline.go | 16 ++- libs/localenv/pipeline_test.go | 4 + libs/localenv/result.go | 9 ++ 15 files changed, 210 insertions(+), 17 deletions(-) create mode 100644 .nextchanges/cli/setup-local-environment-version.md diff --git a/.nextchanges/cli/setup-local-environment-version.md b/.nextchanges/cli/setup-local-environment-version.md new file mode 100644 index 00000000000..d5c3ba67644 --- /dev/null +++ b/.nextchanges/cli/setup-local-environment-version.md @@ -0,0 +1 @@ +`databricks environments setup-local` now records the resolved serverless environment version in a `[tool.databricks.environment]` section of the generated `pyproject.toml` on serverless targets, so the same project can run interactively, in bundles, and in serverless jobs from one source of truth. Cluster targets are unaffected. diff --git a/acceptance/localenv/constraints-only/output.txt b/acceptance/localenv/constraints-only/output.txt index a98729c6ac2..7fb5072d708 100644 --- a/acceptance/localenv/constraints-only/output.txt +++ b/acceptance/localenv/constraints-only/output.txt @@ -19,7 +19,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,15 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = []\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,18 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = []\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/job-serverless-check/output.txt b/acceptance/localenv/job-serverless-check/output.txt index 5c11f8707a9..1abe7d22d77 100644 --- a/acceptance/localenv/job-serverless-check/output.txt +++ b/acceptance/localenv/job-serverless-check/output.txt @@ -4,4 +4,5 @@ Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: tool.uv.constraint-dependencies changed region: databricks-connect + changed region: tool.databricks.environment Check complete. No files were modified. diff --git a/acceptance/localenv/job-task-foreach/output.txt b/acceptance/localenv/job-task-foreach/output.txt index 1b6110d05fd..760ddc579ea 100644 --- a/acceptance/localenv/job-task-foreach/output.txt +++ b/acceptance/localenv/job-task-foreach/output.txt @@ -4,4 +4,5 @@ Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: tool.uv.constraint-dependencies changed region: databricks-connect + changed region: tool.databricks.environment Check complete. No files were modified. diff --git a/acceptance/localenv/merge-warnings-json/output.txt b/acceptance/localenv/merge-warnings-json/output.txt index 6005d4f186a..28b242e3ab6 100644 --- a/acceptance/localenv/merge-warnings-json/output.txt +++ b/acceptance/localenv/merge-warnings-json/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,17 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n spark = [\"databricks-connect==15.0.0\"]\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { @@ -59,8 +59,8 @@ "message": "databricks-connect \"databricks-connect~=16.0.0\" is replaced by the environment's \"databricks-connect~=17.2.0\"" }, { - "code": "W_DBCONNECT_CONSOLIDATED", - "message": "databricks-connect \"databricks-connect==15.0.0\" in [dependency-groups].spark conflicts with the environment's \"databricks-connect~=17.2.0\" and is removed; it is managed in \"dev\"" + "code": "W_DBCONNECT_PIN_DUPLICATED", + "message": "databricks-connect \"databricks-connect==15.0.0\" is not rewritten by the merge; the environment's \"databricks-connect~=17.2.0\" sits in \"dev\" alongside it, and no version satisfies both" }, { "code": "W_USER_CONSTRAINT_CONFLICT", diff --git a/acceptance/localenv/merge-warnings/output.txt b/acceptance/localenv/merge-warnings/output.txt index d6ddf728c83..67372978cb8 100644 --- a/acceptance/localenv/merge-warnings/output.txt +++ b/acceptance/localenv/merge-warnings/output.txt @@ -8,5 +8,6 @@ warning: dependency "pandas==4.0.0" conflicts with the environment constraint "p Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: databricks-connect + changed region: tool.databricks.environment changed region: tool.uv.constraint-dependencies Check complete. No files were modified. diff --git a/acceptance/localenv/serverless-check/output.txt b/acceptance/localenv/serverless-check/output.txt index 0dae505cd2f..a62cdbbb2d8 100644 --- a/acceptance/localenv/serverless-check/output.txt +++ b/acceptance/localenv/serverless-check/output.txt @@ -4,4 +4,5 @@ Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: tool.uv.constraint-dependencies changed region: databricks-connect + changed region: tool.databricks.environment Check complete. No files were modified. diff --git a/acceptance/localenv/serverless-json-major-pin/output.txt b/acceptance/localenv/serverless-json-major-pin/output.txt index 8d04812bfcf..d610b89c3cb 100644 --- a/acceptance/localenv/serverless-json-major-pin/output.txt +++ b/acceptance/localenv/serverless-json-major-pin/output.txt @@ -19,7 +19,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,17 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.0\",\n+]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,20 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.0\",\n+]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/serverless-json/output.txt b/acceptance/localenv/serverless-json/output.txt index 4b46fda397d..040ae13a9b1 100644 --- a/acceptance/localenv/serverless-json/output.txt +++ b/acceptance/localenv/serverless-json/output.txt @@ -20,7 +20,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,17 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.2.0\",\n+]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,20 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.2.0\",\n+]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/libs/localenv/constraints.go b/libs/localenv/constraints.go index eae55263a6e..bee8ba723e0 100644 --- a/libs/localenv/constraints.go +++ b/libs/localenv/constraints.go @@ -74,6 +74,11 @@ type Constraints struct { DatabricksConnect string // ConstraintDeps is the list of entries from [tool.uv].constraint-dependencies. ConstraintDeps []string + // EnvironmentVersion is the serverless environment version written into + // [tool.databricks.environment].environment_version (e.g. "5"). It is not + // parsed from the artifact but set by the pipeline from the resolved compute + // target; it is empty for cluster targets, where the section is not managed. + EnvironmentVersion string } // cacheFileName maps an env key to a single, collision-free cache filename. diff --git a/libs/localenv/merge.go b/libs/localenv/merge.go index 795492ba876..4c547832e51 100644 --- a/libs/localenv/merge.go +++ b/libs/localenv/merge.go @@ -28,11 +28,20 @@ var ( // Region names reported back to the caller via MergeManaged's regions return value. const ( - regionRequiresPython = "requires-python" - regionDatabricksConnect = "databricks-connect" - regionToolUv = "tool.uv.constraint-dependencies" + regionRequiresPython = "requires-python" + regionDatabricksConnect = "databricks-connect" + regionToolUv = "tool.uv.constraint-dependencies" + regionDatabricksEnvironment = "tool.databricks.environment" ) +// databricksEnvironmentTable is the TOML table header that carries the +// serverless environment version, and environmentVersionRe matches the start of +// its managed environment_version assignment (capturing leading whitespace so it +// is preserved when the value is replaced). +const databricksEnvironmentTable = "[tool.databricks.environment]" + +var environmentVersionRe = regexp.MustCompile(`^(\s*)environment_version\s*=`) + var ( // tableHeaderRe matches a TOML table header line: a standard table like // "[project]" / "[tool.uv]" or an array-of-tables like "[[tool.uv.index]]", @@ -88,7 +97,7 @@ func planDBConnect(target []byte, c Constraints) dbconnectPlan { return dbconnectPlan{replacedDevPin: replaced, removed: removed} } -// MergeManaged applies the three managed transforms to target, preserving every other +// MergeManaged applies the managed transforms to target, preserving every other // byte (comments, ordering, whitespace). It returns the merged bytes and the list of // regions that actually changed. The operation is idempotent: feeding its own output // back in produces identical bytes. @@ -144,6 +153,11 @@ func MergeManaged(target []byte, c Constraints) (merged []byte, regions []string regions = append(regions, regionDatabricksConnect) } + lines, envChanged := mergeDatabricksEnvironment(lines, c.EnvironmentVersion) + if envChanged { + regions = append(regions, regionDatabricksEnvironment) + } + lines, uvChanged := mergeToolUv(lines, c.ConstraintDeps) if uvChanged { regions = append(regions, regionToolUv) @@ -239,6 +253,45 @@ func mergeRequiresPython(lines []string, value string) ([]string, bool) { return inserted, true } +// mergeDatabricksEnvironment pins environment_version to version within the +// env-owned [tool.databricks.environment] table: it replaces an existing value +// (preserving the line's indentation and any inline comment), inserts the key +// when the table exists without it, and appends the table when it is absent. +// An empty version (a cluster target) is a no-op — the section is only written +// for serverless targets, so an existing one is left untouched rather than +// removed. Returns whether the line slice changed. +func mergeDatabricksEnvironment(lines []string, version string) ([]string, bool) { + if version == "" { + return lines, false + } + + header, end, found := tableBounds(lines, databricksEnvironmentTable) + if !found { + return appendManagedBlock(lines, []string{databricksEnvironmentTable, fmt.Sprintf(`environment_version = "%s"`, version)}), true + } + + for i := header + 1; i < end; i++ { + m := environmentVersionRe.FindStringSubmatch(lines[i]) + if m == nil { + continue + } + // Only the value is managed; a trailing inline comment is user content. + replacement := fmt.Sprintf(`%senvironment_version = "%s"%s`, m[1], version, trailingComment(lines[i])) + if lines[i] == replacement { + return lines, false + } + lines[i] = replacement + return lines, true + } + + // Table exists but has no environment_version: insert directly under the header. + inserted := make([]string, 0, len(lines)+1) + inserted = append(inserted, lines[:header+1]...) + inserted = append(inserted, fmt.Sprintf(`environment_version = "%s"`, version)) + inserted = append(inserted, lines[header+1:]...) + return inserted, true +} + // trailingComment returns the inline TOML comment suffix of a line (including the // leading whitespace and "#"), or "" if there is none. It ignores "#" characters // inside a quoted string so a value like requires-python = ">=3.10 # x" is not @@ -1100,9 +1153,10 @@ func equalLines(a, b []string) bool { const freshProjectVersion = "0.0.0" // RenderFreshPyproject produces a complete managed pyproject.toml for a project that has -// none, with [project], [dependency-groups].dev (carrying the databricks-connect pin), and -// the marker-bracketed [tool.uv] constraint block. When c.DatabricksConnect is empty -// (constraints-only mode) the dev group is emitted empty rather than with a blank entry. +// none, with [project], [dependency-groups].dev (carrying the databricks-connect pin), the +// [tool.databricks.environment] section (serverless targets only), and the marker-bracketed +// [tool.uv] constraint block. When c.DatabricksConnect is empty (constraints-only mode) the +// dev group is emitted empty rather than with a blank entry. func RenderFreshPyproject(projectName string, c Constraints) []byte { var b strings.Builder b.WriteString("[project]\n") @@ -1120,6 +1174,13 @@ func RenderFreshPyproject(projectName string, c Constraints) []byte { b.WriteString("dev = []\n") } b.WriteString("\n") + // The serverless environment version is written only for serverless targets; + // a cluster target leaves EnvironmentVersion empty and omits the section. + if c.EnvironmentVersion != "" { + b.WriteString(databricksEnvironmentTable + "\n") + fmt.Fprintf(&b, "environment_version = %q\n", c.EnvironmentVersion) + b.WriteString("\n") + } for _, line := range renderToolUvBlock(c.ConstraintDeps, true) { b.WriteString(line) b.WriteString("\n") diff --git a/libs/localenv/merge_test.go b/libs/localenv/merge_test.go index 9e3ffbdb8f5..26e2438a7d0 100644 --- a/libs/localenv/merge_test.go +++ b/libs/localenv/merge_test.go @@ -452,12 +452,113 @@ func TestRenderFreshPyproject(t *testing.T) { assert.Contains(t, s, managedMarkerStart) assert.Contains(t, s, managedMarkerEnd) assert.Contains(t, s, "pydantic~=2.10.6") + // A cluster target (no EnvironmentVersion) writes no [tool.databricks.environment]. + assert.NotContains(t, s, "[tool.databricks.environment]") // A fresh render is itself a no-op under MergeManaged (already fully managed). merged, _, err := MergeManaged(out, testConstraints()) require.NoError(t, err) assert.Equal(t, s, string(merged)) } +func TestRenderFreshPyprojectServerlessWritesEnvironment(t *testing.T) { + c := testConstraints() + c.EnvironmentVersion = "5" + out := RenderFreshPyproject("demo", c) + s := string(out) + assert.Contains(t, s, "[tool.databricks.environment]") + assert.Contains(t, s, `environment_version = "5"`) + requireValidTOML(t, out) + // A fresh render is itself a no-op under MergeManaged (already fully managed). + merged, _, err := MergeManaged(out, c) + require.NoError(t, err) + assert.Equal(t, s, string(merged)) +} + +func TestMergeInsertsDatabricksEnvironmentWhenAbsent(t *testing.T) { + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "[tool.databricks.environment]") + assert.Contains(t, s, `environment_version = "5"`) + assert.Contains(t, regions, "tool.databricks.environment") + requireValidTOML(t, out) + // Idempotent. + twice, _, err := MergeManaged(out, c) + require.NoError(t, err) + assert.Equal(t, s, string(twice)) +} + +func TestMergeReplacesExistingEnvironmentVersionPreservingComment(t *testing.T) { + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] + +[tool.databricks.environment] +environment_version = "4" # pinned +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, `environment_version = "5" # pinned`) + assert.NotContains(t, s, `environment_version = "4"`) + assert.Contains(t, regions, "tool.databricks.environment") + // The table is refreshed in place, not duplicated. + assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) + requireValidTOML(t, out) +} + +func TestMergeInsertsEnvironmentVersionKeyWhenTableExists(t *testing.T) { + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] + +[tool.databricks.environment] +# user note +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, _, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, `environment_version = "5"`) + assert.Contains(t, s, "# user note") + assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) + requireValidTOML(t, out) +} + +func TestMergeEnvironmentNoopForClusterTarget(t *testing.T) { + // A cluster target leaves EnvironmentVersion empty: the section is never + // written, and an existing one is left untouched rather than removed. + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] + +[tool.databricks.environment] +environment_version = "4" +`) + c := testConstraints() // EnvironmentVersion == "": cluster target. + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + assert.Contains(t, string(out), `environment_version = "4"`) + assert.NotContains(t, regions, "tool.databricks.environment") +} + func TestMergeStripsMultiLineConstraintDepsWithBracketInFirstElement(t *testing.T) { // The user's stale constraint-dependencies is a multi-line array whose FIRST // element line contains a "]" inside an extras spec. A naive diff --git a/libs/localenv/pipeline.go b/libs/localenv/pipeline.go index 46a5507a9fa..096a27f278e 100644 --- a/libs/localenv/pipeline.go +++ b/libs/localenv/pipeline.go @@ -205,8 +205,10 @@ func (p *Pipeline) run(ctx context.Context) error { } // Phase: merge — compute the merged pyproject.toml (in-memory, no writes yet). + // The serverless environment version (empty for cluster targets) is written + // into [tool.databricks.environment] so the project also runs in serverless Jobs. p.report(ctx, PhaseMerge) - mergedBytes, greenfield, err := p.mergePlan(ctx, pyMinor, c, dbcPin) + mergedBytes, greenfield, err := p.mergePlan(ctx, pyMinor, c, dbcPin, compute.ServerlessEnvironmentVersion()) if err != nil { return err } @@ -277,8 +279,9 @@ func (p *Pipeline) backupPath() string { // mergePlan computes the merged pyproject.toml bytes (without writing to disk), // decides greenfield vs. existing, and builds the Plan (populated only under // --dry-run). dbcPin is the databricks-connect pin to inject, or "" in -// constraints-only mode. -func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, dbcPin string) (merged []byte, greenfield bool, err error) { +// constraints-only mode. envVersion is the serverless environment version to +// write into [tool.databricks.environment], or "" for a cluster target. +func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, dbcPin, envVersion string) (merged []byte, greenfield bool, err error) { pyproject := p.pyprojectPath() backup := p.backupPath() @@ -305,9 +308,11 @@ func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, greenfield = baseBytes == nil // The artifact drives the merge; in constraints-only mode we clear the - // databricks-connect pin so it is neither written nor asserted. + // databricks-connect pin so it is neither written nor asserted. envVersion is + // the resolved serverless version (empty for cluster targets). effective := *c effective.DatabricksConnect = dbcPin + effective.EnvironmentVersion = envVersion var changedRegions []string if greenfield { @@ -318,6 +323,9 @@ func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, if dbcPin != "" { changedRegions = append(changedRegions, regionDatabricksConnect) } + if envVersion != "" { + changedRegions = append(changedRegions, regionDatabricksEnvironment) + } } else { merged, changedRegions, err = MergeManaged(baseBytes, effective) if err != nil { diff --git a/libs/localenv/pipeline_test.go b/libs/localenv/pipeline_test.go index 5b007d7571d..c4748a31985 100644 --- a/libs/localenv/pipeline_test.go +++ b/libs/localenv/pipeline_test.go @@ -480,6 +480,10 @@ func TestPipelineGreenfieldCreatesNewPyproject(t *testing.T) { data, readErr := os.ReadFile(filepath.Join(dir, "pyproject.toml")) require.NoError(t, readErr) assert.Contains(t, string(data), `"databricks-connect~=17.2.0",`) + // A serverless target records the environment version so the same project + // also runs in serverless Jobs (DECO-27998). + assert.Contains(t, string(data), "[tool.databricks.environment]") + assert.Contains(t, string(data), `environment_version = "4"`) // No backup created when pyproject.toml did not previously exist. assert.NoFileExists(t, filepath.Join(dir, "pyproject.toml.bak")) } diff --git a/libs/localenv/result.go b/libs/localenv/result.go index 1e704bcc55d..7a360b7da98 100644 --- a/libs/localenv/result.go +++ b/libs/localenv/result.go @@ -192,6 +192,15 @@ func (c *ComputeInfo) Label() string { } } +// ServerlessEnvironmentVersion returns the bare serverless environment version +// (e.g. "5") to write into [tool.databricks.environment].environment_version. +// ServerlessVersion is normalized to "vN", so the leading "v" is dropped to +// match the documented bare-number form. It is empty for a cluster target +// (which leaves ServerlessVersion unset), where the section is not managed. +func (c *ComputeInfo) ServerlessEnvironmentVersion() string { + return strings.TrimPrefix(c.ServerlessVersion, "v") +} + // ResolvedInfo is the resolved environment definition (spec §6 "resolved"). // DBConnectVersion is omitted in constraints-only mode. type ResolvedInfo struct { From 4759a4866bfb33ca8ea7c5ac9c86afacda582dbb Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Wed, 12 Aug 2026 16:47:29 +0200 Subject: [PATCH 2/4] localenv: drop changelog fragment (setup-local not yet released) Co-authored-by: Isaac --- .nextchanges/cli/setup-local-environment-version.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .nextchanges/cli/setup-local-environment-version.md diff --git a/.nextchanges/cli/setup-local-environment-version.md b/.nextchanges/cli/setup-local-environment-version.md deleted file mode 100644 index d5c3ba67644..00000000000 --- a/.nextchanges/cli/setup-local-environment-version.md +++ /dev/null @@ -1 +0,0 @@ -`databricks environments setup-local` now records the resolved serverless environment version in a `[tool.databricks.environment]` section of the generated `pyproject.toml` on serverless targets, so the same project can run interactively, in bundles, and in serverless jobs from one source of truth. Cluster targets are unaffected. From f61bb7a4b7b40360b62721a1b0970f474591543d Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Wed, 12 Aug 2026 16:53:36 +0200 Subject: [PATCH 3/4] localenv: test setup-local environment merge on pre-feature managed files Cover the common upgrade path a code review flagged: a pyproject.toml a pre-feature CLI wrote for a serverless target already carries the managed [tool.uv] marker block but no [tool.databricks.environment] section. Assert the section is added without duplicating the marker block, the result is valid TOML, and a second merge is a no-op. Co-authored-by: Isaac --- libs/localenv/merge_test.go | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/libs/localenv/merge_test.go b/libs/localenv/merge_test.go index 26e2438a7d0..1fca28a69a3 100644 --- a/libs/localenv/merge_test.go +++ b/libs/localenv/merge_test.go @@ -559,6 +559,47 @@ environment_version = "4" assert.NotContains(t, regions, "tool.databricks.environment") } +func TestMergeAddsEnvironmentToPreFeatureManagedFile(t *testing.T) { + // The common upgrade path: a file a pre-feature CLI wrote for a serverless + // target already carries the managed [tool.uv] marker block but no + // [tool.databricks.environment] section. Re-running the new CLI must add the + // section, keep exactly one managed block, and stay idempotent. + in := []byte(`[project] +name = "demo" +version = "0.0.0" +requires-python = "==3.12.*" + +[dependency-groups] +dev = [ + "databricks-connect~=17.2.0", +] + +` + managedMarkerStart + ` +[tool.uv] +constraint-dependencies = [ + "pydantic~=2.10.6", + "anyio~=4.6.2", +] +` + managedMarkerEnd + ` +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "[tool.databricks.environment]") + assert.Contains(t, s, `environment_version = "5"`) + assert.Contains(t, regions, "tool.databricks.environment") + // The pre-existing managed [tool.uv] block is neither duplicated nor disturbed. + assert.Equal(t, 1, countOccurrences(s, managedMarkerStart)) + assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) + requireValidTOML(t, out) + // Idempotent on the upgraded file. + twice, _, err := MergeManaged(out, c) + require.NoError(t, err) + assert.Equal(t, s, string(twice)) +} + func TestMergeStripsMultiLineConstraintDepsWithBracketInFirstElement(t *testing.T) { // The user's stale constraint-dependencies is a multi-line array whose FIRST // element line contains a "]" inside an extras spec. A naive From 3f5940ed8e2252ebc986fb06d7682964c0c3312a Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Thu, 13 Aug 2026 14:06:45 +0200 Subject: [PATCH 4/4] localenv: address setup-local environment-version review feedback - Warn when a cluster run finds a stale [tool.databricks.environment] environment_version left over from an earlier serverless run (W_STALE_ENVIRONMENT_VERSION); the section is not managed for cluster targets, so the value would otherwise silently misdescribe the target. - Document why --constraints-only still records environment_version: it reflects the resolved compute target, not a managed dependency like databricks-connect. - Deduplicate the "v"-prefix stripping so Label() reuses ServerlessEnvironmentVersion(). - Tests reference the regionDatabricksEnvironment constant, and add an acceptance test covering the stale-version warning on a cluster target. Co-authored-by: Isaac --- .../cluster-stale-environment/out.test.toml | 2 + .../cluster-stale-environment/output.txt | 6 +++ .../localenv/cluster-stale-environment/script | 15 ++++++ .../cluster-stale-environment/test.toml | 36 +++++++++++++++ .../constraints-only-existing/output.txt | 2 +- .../output.txt | 2 +- .../dbconnect-consolidate-check/output.txt | 2 +- .../localenv/merge-warnings-json/output.txt | 6 +-- .../output.txt | 1 + libs/localenv/merge_test.go | 8 ++-- libs/localenv/pipeline.go | 6 +++ libs/localenv/result.go | 9 +++- libs/localenv/warnings.go | 24 ++++++++++ libs/localenv/warnings_test.go | 46 +++++++++++++++++++ 14 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 acceptance/localenv/cluster-stale-environment/out.test.toml create mode 100644 acceptance/localenv/cluster-stale-environment/output.txt create mode 100644 acceptance/localenv/cluster-stale-environment/script create mode 100644 acceptance/localenv/cluster-stale-environment/test.toml diff --git a/acceptance/localenv/cluster-stale-environment/out.test.toml b/acceptance/localenv/cluster-stale-environment/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/cluster-stale-environment/output.txt b/acceptance/localenv/cluster-stale-environment/output.txt new file mode 100644 index 00000000000..64581495875 --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] environments setup-local --cluster-name my-cluster --dry-run +warning: [tool.databricks.environment] environment_version "5" is left from a serverless target but the current target is a cluster; it is not updated +Plan: [TEST_TMP_DIR]/pyproject.toml + changed region: tool.uv.constraint-dependencies +Check complete. No files were modified. diff --git a/acceptance/localenv/cluster-stale-environment/script b/acceptance/localenv/cluster-stale-environment/script new file mode 100644 index 00000000000..2aeb709898c --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/script @@ -0,0 +1,15 @@ +# An existing project set up for serverless carries [tool.databricks.environment]. +# Re-running against a cluster target leaves it untouched but warns it is stale. +cat > pyproject.toml <<'PY' +[project] +name = "demo" +requires-python = ">=3.11" + +[dependency-groups] +dev = ["databricks-connect~=15.4.0"] + +[tool.databricks.environment] +environment_version = "5" +PY + +trace $CLI environments setup-local --cluster-name my-cluster --dry-run diff --git a/acceptance/localenv/cluster-stale-environment/test.toml b/acceptance/localenv/cluster-stale-environment/test.toml new file mode 100644 index 00000000000..1619602c515 --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/test.toml @@ -0,0 +1,36 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The script writes pyproject.toml as the merge input; --dry-run leaves it unchanged. +Ignore = ["pyproject.toml"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" + +# A cluster target does not manage [tool.databricks.environment], so a leftover +# serverless environment_version is reported as stale. +[[Server]] +Pattern = "GET /api/2.1/clusters/list" +Response.Body = ''' +{ + "clusters": [ + {"cluster_id": "cid-123", "cluster_name": "my-cluster", "spark_version": "15.4.x-scala2.12"} + ] +} +''' + +[[Server]] +Pattern = "GET /dbr/15.4.x-scala2.12/pyproject.toml" +Response.Body = ''' +[project] +requires-python = ">=3.11" + +[dependency-groups] +dev = ["databricks-connect~=15.4.0"] + +[tool.uv] +constraint-dependencies = ["pyarrow<19"] +''' + +[[Repls]] +Old = 'uv uv \S+(?: \([^)]+\))?' +New = 'uv [UV_VERSION]' diff --git a/acceptance/localenv/constraints-only-existing/output.txt b/acceptance/localenv/constraints-only-existing/output.txt index b17f420e4e5..365bc747bdc 100644 --- a/acceptance/localenv/constraints-only-existing/output.txt +++ b/acceptance/localenv/constraints-only-existing/output.txt @@ -20,7 +20,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,8 +1,15 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"databricks-connect==15.1.*\"]\n \n [dependency-groups]\n dev = [\"databricks-connect~=16.0\"]\n docs = [\"databricks-connect==15.0.0\"]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,8 +1,18 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"databricks-connect==15.1.*\"]\n \n [dependency-groups]\n dev = [\"databricks-connect~=16.0\"]\n docs = [\"databricks-connect==15.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/dbconnect-compatible-kept-check/output.txt b/acceptance/localenv/dbconnect-compatible-kept-check/output.txt index 3f36af3bce1..7b147c65964 100644 --- a/acceptance/localenv/dbconnect-compatible-kept-check/output.txt +++ b/acceptance/localenv/dbconnect-compatible-kept-check/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,6 +1,6 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-connect\u003e=15\",\n \"databricks-connect ; python_version \u003c '3.13'\",\n@@ -10,5 +10,11 @@\n extra = [\"databricks-connect\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n test = [\"databricks-connect\u003e=15,\u003c20\"]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,6 +1,6 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-connect\u003e=15\",\n \"databricks-connect ; python_version \u003c '3.13'\",\n@@ -10,5 +10,14 @@\n extra = [\"databricks-connect\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n test = [\"databricks-connect\u003e=15,\u003c20\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/dbconnect-consolidate-check/output.txt b/acceptance/localenv/dbconnect-consolidate-check/output.txt index 31cb766909f..0e94ce0430e 100644 --- a/acceptance/localenv/dbconnect-consolidate-check/output.txt +++ b/acceptance/localenv/dbconnect-consolidate-check/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,15 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-dlt\",\n- \"databricks-connect==15.1.*\",\n \"pytest\",\n ]\n \n [project.optional-dependencies]\n-extra = [\"databricks-connect==14.0.0\"]\n+extra = []\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n-docs = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n+docs = []\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,15 +1,23 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-dlt\",\n- \"databricks-connect==15.1.*\",\n \"pytest\",\n ]\n \n [project.optional-dependencies]\n-extra = [\"databricks-connect==14.0.0\"]\n+extra = []\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n-docs = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n+docs = []\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/merge-warnings-json/output.txt b/acceptance/localenv/merge-warnings-json/output.txt index 28b242e3ab6..2f11282d105 100644 --- a/acceptance/localenv/merge-warnings-json/output.txt +++ b/acceptance/localenv/merge-warnings-json/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n spark = [\"databricks-connect==15.0.0\"]\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { @@ -59,8 +59,8 @@ "message": "databricks-connect \"databricks-connect~=16.0.0\" is replaced by the environment's \"databricks-connect~=17.2.0\"" }, { - "code": "W_DBCONNECT_PIN_DUPLICATED", - "message": "databricks-connect \"databricks-connect==15.0.0\" is not rewritten by the merge; the environment's \"databricks-connect~=17.2.0\" sits in \"dev\" alongside it, and no version satisfies both" + "code": "W_DBCONNECT_CONSOLIDATED", + "message": "databricks-connect \"databricks-connect==15.0.0\" in [dependency-groups].spark conflicts with the environment's \"databricks-connect~=17.2.0\" and is removed; it is managed in \"dev\"" }, { "code": "W_USER_CONSTRAINT_CONFLICT", diff --git a/acceptance/localenv/wildcard-constraint-conflict-check/output.txt b/acceptance/localenv/wildcard-constraint-conflict-check/output.txt index a45244f373c..8fdf2ffeb1c 100644 --- a/acceptance/localenv/wildcard-constraint-conflict-check/output.txt +++ b/acceptance/localenv/wildcard-constraint-conflict-check/output.txt @@ -4,5 +4,6 @@ warning: requires-python ">=3.10" is replaced by the environment's ">=3.12" warning: dependency "pyarrow==18.*" conflicts with the environment constraint "pyarrow~=19.0" Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python + changed region: tool.databricks.environment changed region: tool.uv.constraint-dependencies Check complete. No files were modified. diff --git a/libs/localenv/merge_test.go b/libs/localenv/merge_test.go index 1fca28a69a3..96ced5dfc4f 100644 --- a/libs/localenv/merge_test.go +++ b/libs/localenv/merge_test.go @@ -488,7 +488,7 @@ dev = ["databricks-connect~=16.0.0"] s := string(out) assert.Contains(t, s, "[tool.databricks.environment]") assert.Contains(t, s, `environment_version = "5"`) - assert.Contains(t, regions, "tool.databricks.environment") + assert.Contains(t, regions, regionDatabricksEnvironment) requireValidTOML(t, out) // Idempotent. twice, _, err := MergeManaged(out, c) @@ -513,7 +513,7 @@ environment_version = "4" # pinned s := string(out) assert.Contains(t, s, `environment_version = "5" # pinned`) assert.NotContains(t, s, `environment_version = "4"`) - assert.Contains(t, regions, "tool.databricks.environment") + assert.Contains(t, regions, regionDatabricksEnvironment) // The table is refreshed in place, not duplicated. assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) requireValidTOML(t, out) @@ -556,7 +556,7 @@ environment_version = "4" out, regions, err := MergeManaged(in, c) require.NoError(t, err) assert.Contains(t, string(out), `environment_version = "4"`) - assert.NotContains(t, regions, "tool.databricks.environment") + assert.NotContains(t, regions, regionDatabricksEnvironment) } func TestMergeAddsEnvironmentToPreFeatureManagedFile(t *testing.T) { @@ -589,7 +589,7 @@ constraint-dependencies = [ s := string(out) assert.Contains(t, s, "[tool.databricks.environment]") assert.Contains(t, s, `environment_version = "5"`) - assert.Contains(t, regions, "tool.databricks.environment") + assert.Contains(t, regions, regionDatabricksEnvironment) // The pre-existing managed [tool.uv] block is neither duplicated nor disturbed. assert.Equal(t, 1, countOccurrences(s, managedMarkerStart)) assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) diff --git a/libs/localenv/pipeline.go b/libs/localenv/pipeline.go index 096a27f278e..39d13bb1b4b 100644 --- a/libs/localenv/pipeline.go +++ b/libs/localenv/pipeline.go @@ -310,6 +310,12 @@ func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, // The artifact drives the merge; in constraints-only mode we clear the // databricks-connect pin so it is neither written nor asserted. envVersion is // the resolved serverless version (empty for cluster targets). + // + // envVersion is deliberately NOT cleared in constraints-only mode: unlike the + // databricks-connect pin (a managed *dependency* the mode opts out of), the + // environment version records the resolved compute *target*, which the mode + // still resolves. Recording it keeps the target discoverable for VS Code and + // serverless Jobs even when dependency management is turned off. effective := *c effective.DatabricksConnect = dbcPin effective.EnvironmentVersion = envVersion diff --git a/libs/localenv/result.go b/libs/localenv/result.go index 7a360b7da98..99266128abc 100644 --- a/libs/localenv/result.go +++ b/libs/localenv/result.go @@ -178,7 +178,7 @@ func (c *ComputeInfo) Label() string { switch { case c.ServerlessVersion != "": // ServerlessVersion is normalized to "v4"; drop the "v" for display. - return "serverless " + strings.TrimPrefix(c.ServerlessVersion, "v") + return "serverless " + c.ServerlessEnvironmentVersion() case c.ClusterID != "": return "cluster " + c.ClusterID case c.SparkVersion != "": @@ -274,6 +274,13 @@ const ( // groups — since constraint-dependencies applies to the whole resolution. Emitted // only when the ranges are provably disjoint; ambiguous cases are not flagged. WarnUserConstraintConflict = "W_USER_CONSTRAINT_CONFLICT" + // WarnStaleEnvironmentVersion: the target is a cluster, which does not manage the + // serverless environment section, but the file carries a [tool.databricks.environment] + // environment_version left over from an earlier serverless run. The value is not + // updated (cluster targets are a no-op there), so it now describes a target the + // project is no longer set up for — worth surfacing because VS Code and serverless + // Jobs read that section as a source of truth. + WarnStaleEnvironmentVersion = "W_STALE_ENVIRONMENT_VERSION" ) // Result is the full outcome of a sync run and the root of the --json object diff --git a/libs/localenv/warnings.go b/libs/localenv/warnings.go index d6737c31926..5e1c15f2203 100644 --- a/libs/localenv/warnings.go +++ b/libs/localenv/warnings.go @@ -77,6 +77,17 @@ type userPyprojectTOML struct { // Every group is decoded, not just dev: uv locks all declared groups, so a pin in // any of them is subject to constraint-dependencies (see resolutionRequirements). DependencyGroups map[string]any `toml:"dependency-groups"` + // Tool.Databricks.Environment.EnvironmentVersion is the serverless version the + // merge manages; it is read here only to warn when it is left stale on a cluster + // target. A non-string value here is malformed and, like any decode error, yields + // no warnings rather than a crash. + Tool struct { + Databricks struct { + Environment struct { + EnvironmentVersion string `toml:"environment_version"` + } `toml:"environment"` + } `toml:"databricks"` + } `toml:"tool"` } // devGroup is the dependency group whose databricks-connect pin the merge manages. @@ -198,6 +209,19 @@ func detectMergeWarnings(userPyproject []byte, c Constraints, plan dbconnectPlan } warnings = append(warnings, constraintConflicts(survivors, c.ConstraintDeps)...) + + // A cluster target leaves c.EnvironmentVersion empty and does not manage the + // serverless environment section, so an environment_version left over from an + // earlier serverless run is neither refreshed nor removed. Warn that it is now + // stale rather than let it silently misdescribe the target to a downstream reader. + if c.EnvironmentVersion == "" { + if ev := strings.TrimSpace(p.Tool.Databricks.Environment.EnvironmentVersion); ev != "" { + warnings = append(warnings, Warning{ + Code: WarnStaleEnvironmentVersion, + Message: fmt.Sprintf("[tool.databricks.environment] environment_version %q is left from a serverless target but the current target is a cluster; it is not updated", ev), + }) + } + } return warnings } diff --git a/libs/localenv/warnings_test.go b/libs/localenv/warnings_test.go index 3fd6bb1e573..d2f4c07cfb7 100644 --- a/libs/localenv/warnings_test.go +++ b/libs/localenv/warnings_test.go @@ -72,6 +72,52 @@ dev = ["databricks-connect~=16.1.0"] assert.Empty(t, detectWarnings(user, c)) } +func TestDetectMergeWarningsStaleEnvironmentVersionOnClusterTarget(t *testing.T) { + // A cluster target (c.EnvironmentVersion == "") that finds a leftover serverless + // environment_version warns that the value is now stale. + user := []byte(`[project] +requires-python = "==3.12.*" + +[dependency-groups] +dev = ["databricks-connect~=18.0.0"] + +[tool.databricks.environment] +environment_version = "5" +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0", EnvironmentVersion: ""} + got := detectWarnings(user, c) + assert.Equal(t, []string{WarnStaleEnvironmentVersion}, codes(got)) + assert.Contains(t, got[0].Message, `"5"`) +} + +func TestDetectMergeWarningsNoStaleWarningForServerlessTarget(t *testing.T) { + // A serverless target manages the section (refreshes it), so an existing value + // is not stale and must not warn. + user := []byte(`[project] +requires-python = "==3.12.*" + +[dependency-groups] +dev = ["databricks-connect~=18.0.0"] + +[tool.databricks.environment] +environment_version = "4" +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0", EnvironmentVersion: "5"} + assert.NotContains(t, codes(detectWarnings(user, c)), WarnStaleEnvironmentVersion) +} + +func TestDetectMergeWarningsNoStaleWarningWhenSectionAbsent(t *testing.T) { + // A cluster target with no existing section has nothing to go stale. + user := []byte(`[project] +requires-python = "==3.12.*" + +[dependency-groups] +dev = ["databricks-connect~=18.0.0"] +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0", EnvironmentVersion: ""} + assert.NotContains(t, codes(detectWarnings(user, c)), WarnStaleEnvironmentVersion) +} + func TestDetectMergeWarningsUserConstraintConflict(t *testing.T) { user := []byte(`[project] requires-python = "==3.12.*"