diff --git a/acceptance/localenv/cluster-stale-environment/out.test.toml b/acceptance/localenv/cluster-stale-environment/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/cluster-stale-environment/output.txt b/acceptance/localenv/cluster-stale-environment/output.txt new file mode 100644 index 00000000000..64581495875 --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] environments setup-local --cluster-name my-cluster --dry-run +warning: [tool.databricks.environment] environment_version "5" is left from a serverless target but the current target is a cluster; it is not updated +Plan: [TEST_TMP_DIR]/pyproject.toml + changed region: tool.uv.constraint-dependencies +Check complete. No files were modified. diff --git a/acceptance/localenv/cluster-stale-environment/script b/acceptance/localenv/cluster-stale-environment/script new file mode 100644 index 00000000000..2aeb709898c --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/script @@ -0,0 +1,15 @@ +# An existing project set up for serverless carries [tool.databricks.environment]. +# Re-running against a cluster target leaves it untouched but warns it is stale. +cat > pyproject.toml <<'PY' +[project] +name = "demo" +requires-python = ">=3.11" + +[dependency-groups] +dev = ["databricks-connect~=15.4.0"] + +[tool.databricks.environment] +environment_version = "5" +PY + +trace $CLI environments setup-local --cluster-name my-cluster --dry-run diff --git a/acceptance/localenv/cluster-stale-environment/test.toml b/acceptance/localenv/cluster-stale-environment/test.toml new file mode 100644 index 00000000000..1619602c515 --- /dev/null +++ b/acceptance/localenv/cluster-stale-environment/test.toml @@ -0,0 +1,36 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The script writes pyproject.toml as the merge input; --dry-run leaves it unchanged. +Ignore = ["pyproject.toml"] + +[Env] +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" + +# A cluster target does not manage [tool.databricks.environment], so a leftover +# serverless environment_version is reported as stale. +[[Server]] +Pattern = "GET /api/2.1/clusters/list" +Response.Body = ''' +{ + "clusters": [ + {"cluster_id": "cid-123", "cluster_name": "my-cluster", "spark_version": "15.4.x-scala2.12"} + ] +} +''' + +[[Server]] +Pattern = "GET /dbr/15.4.x-scala2.12/pyproject.toml" +Response.Body = ''' +[project] +requires-python = ">=3.11" + +[dependency-groups] +dev = ["databricks-connect~=15.4.0"] + +[tool.uv] +constraint-dependencies = ["pyarrow<19"] +''' + +[[Repls]] +Old = 'uv uv \S+(?: \([^)]+\))?' +New = 'uv [UV_VERSION]' diff --git a/acceptance/localenv/constraints-only-existing/output.txt b/acceptance/localenv/constraints-only-existing/output.txt index b17f420e4e5..365bc747bdc 100644 --- a/acceptance/localenv/constraints-only-existing/output.txt +++ b/acceptance/localenv/constraints-only-existing/output.txt @@ -20,7 +20,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,8 +1,15 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"databricks-connect==15.1.*\"]\n \n [dependency-groups]\n dev = [\"databricks-connect~=16.0\"]\n docs = [\"databricks-connect==15.0.0\"]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,8 +1,18 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"databricks-connect==15.1.*\"]\n \n [dependency-groups]\n dev = [\"databricks-connect~=16.0\"]\n docs = [\"databricks-connect==15.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/constraints-only/output.txt b/acceptance/localenv/constraints-only/output.txt index a98729c6ac2..7fb5072d708 100644 --- a/acceptance/localenv/constraints-only/output.txt +++ b/acceptance/localenv/constraints-only/output.txt @@ -19,7 +19,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,15 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = []\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,18 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = []\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/dbconnect-compatible-kept-check/output.txt b/acceptance/localenv/dbconnect-compatible-kept-check/output.txt index 3f36af3bce1..7b147c65964 100644 --- a/acceptance/localenv/dbconnect-compatible-kept-check/output.txt +++ b/acceptance/localenv/dbconnect-compatible-kept-check/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,6 +1,6 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-connect\u003e=15\",\n \"databricks-connect ; python_version \u003c '3.13'\",\n@@ -10,5 +10,11 @@\n extra = [\"databricks-connect\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n test = [\"databricks-connect\u003e=15,\u003c20\"]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,6 +1,6 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-connect\u003e=15\",\n \"databricks-connect ; python_version \u003c '3.13'\",\n@@ -10,5 +10,14 @@\n extra = [\"databricks-connect\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n test = [\"databricks-connect\u003e=15,\u003c20\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/dbconnect-consolidate-check/output.txt b/acceptance/localenv/dbconnect-consolidate-check/output.txt index 31cb766909f..0e94ce0430e 100644 --- a/acceptance/localenv/dbconnect-consolidate-check/output.txt +++ b/acceptance/localenv/dbconnect-consolidate-check/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,15 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-dlt\",\n- \"databricks-connect==15.1.*\",\n \"pytest\",\n ]\n \n [project.optional-dependencies]\n-extra = [\"databricks-connect==14.0.0\"]\n+extra = []\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n-docs = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n+docs = []\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,15 +1,23 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\n \"databricks-dlt\",\n- \"databricks-connect==15.1.*\",\n \"pytest\",\n ]\n \n [project.optional-dependencies]\n-extra = [\"databricks-connect==14.0.0\"]\n+extra = []\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0\"]\n-docs = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\"]\n+docs = []\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/job-serverless-check/output.txt b/acceptance/localenv/job-serverless-check/output.txt index 5c11f8707a9..1abe7d22d77 100644 --- a/acceptance/localenv/job-serverless-check/output.txt +++ b/acceptance/localenv/job-serverless-check/output.txt @@ -4,4 +4,5 @@ Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: tool.uv.constraint-dependencies changed region: databricks-connect + changed region: tool.databricks.environment Check complete. No files were modified. diff --git a/acceptance/localenv/job-task-foreach/output.txt b/acceptance/localenv/job-task-foreach/output.txt index 1b6110d05fd..760ddc579ea 100644 --- a/acceptance/localenv/job-task-foreach/output.txt +++ b/acceptance/localenv/job-task-foreach/output.txt @@ -4,4 +4,5 @@ Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: tool.uv.constraint-dependencies changed region: databricks-connect + changed region: tool.databricks.environment Check complete. No files were modified. diff --git a/acceptance/localenv/merge-warnings-json/output.txt b/acceptance/localenv/merge-warnings-json/output.txt index 6005d4f186a..2f11282d105 100644 --- a/acceptance/localenv/merge-warnings-json/output.txt +++ b/acceptance/localenv/merge-warnings-json/output.txt @@ -21,7 +21,7 @@ "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldBackup": "[TEST_TMP_DIR]/pyproject.toml.bak", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,17 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml.new\n@@ -1,9 +1,20 @@\n [project]\n name = \"demo\"\n-requires-python = \"\u003e=3.10\"\n+requires-python = \"\u003e=3.12\"\n dependencies = [\"pyarrow==21.0.0\"]\n \n [dependency-groups]\n-dev = [\"databricks-connect~=16.0.0\", {include-group = \"spark\"}]\n-spark = [\"databricks-connect==15.0.0\"]\n+dev = [\"databricks-connect~=17.2.0\", {include-group = \"spark\"}]\n+spark = []\n qa = [\"pandas==4.0.0\"]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/merge-warnings/output.txt b/acceptance/localenv/merge-warnings/output.txt index d6ddf728c83..67372978cb8 100644 --- a/acceptance/localenv/merge-warnings/output.txt +++ b/acceptance/localenv/merge-warnings/output.txt @@ -8,5 +8,6 @@ warning: dependency "pandas==4.0.0" conflicts with the environment constraint "p Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: databricks-connect + changed region: tool.databricks.environment changed region: tool.uv.constraint-dependencies Check complete. No files were modified. diff --git a/acceptance/localenv/serverless-check/output.txt b/acceptance/localenv/serverless-check/output.txt index 0dae505cd2f..a62cdbbb2d8 100644 --- a/acceptance/localenv/serverless-check/output.txt +++ b/acceptance/localenv/serverless-check/output.txt @@ -4,4 +4,5 @@ Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python changed region: tool.uv.constraint-dependencies changed region: databricks-connect + changed region: tool.databricks.environment Check complete. No files were modified. diff --git a/acceptance/localenv/serverless-json-major-pin/output.txt b/acceptance/localenv/serverless-json-major-pin/output.txt index 8d04812bfcf..d610b89c3cb 100644 --- a/acceptance/localenv/serverless-json-major-pin/output.txt +++ b/acceptance/localenv/serverless-json-major-pin/output.txt @@ -19,7 +19,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,17 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.0\",\n+]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,20 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.0\",\n+]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/serverless-json/output.txt b/acceptance/localenv/serverless-json/output.txt index 4b46fda397d..040ae13a9b1 100644 --- a/acceptance/localenv/serverless-json/output.txt +++ b/acceptance/localenv/serverless-json/output.txt @@ -20,7 +20,7 @@ "plan": { "wouldWrite": "[TEST_TMP_DIR]/pyproject.toml", "wouldInstallPython": "3.12", - "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,17 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.2.0\",\n+]\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" + "diff": "--- pyproject.toml\n+++ pyproject.toml\n@@ -1 +1,20 @@\n+[project]\n+name = \"001\"\n+version = \"0.0.0\"\n+requires-python = \"\u003e=3.12\"\n+\n+[dependency-groups]\n+dev = [\n+ \"databricks-connect~=17.2.0\",\n+]\n+\n+[tool.databricks.environment]\n+environment_version = \"4\"\n+\n+# managed by databricks environments setup-local — do not edit\n+[tool.uv]\n+constraint-dependencies = [\n+ \"pyarrow\u003c19\",\n+ \"pandas\u003c3\",\n+]\n+# end managed by databricks environments setup-local\n" }, "phases": [ { diff --git a/acceptance/localenv/wildcard-constraint-conflict-check/output.txt b/acceptance/localenv/wildcard-constraint-conflict-check/output.txt index a45244f373c..8fdf2ffeb1c 100644 --- a/acceptance/localenv/wildcard-constraint-conflict-check/output.txt +++ b/acceptance/localenv/wildcard-constraint-conflict-check/output.txt @@ -4,5 +4,6 @@ warning: requires-python ">=3.10" is replaced by the environment's ">=3.12" warning: dependency "pyarrow==18.*" conflicts with the environment constraint "pyarrow~=19.0" Plan: [TEST_TMP_DIR]/pyproject.toml changed region: requires-python + changed region: tool.databricks.environment changed region: tool.uv.constraint-dependencies Check complete. No files were modified. diff --git a/libs/localenv/constraints.go b/libs/localenv/constraints.go index eae55263a6e..bee8ba723e0 100644 --- a/libs/localenv/constraints.go +++ b/libs/localenv/constraints.go @@ -74,6 +74,11 @@ type Constraints struct { DatabricksConnect string // ConstraintDeps is the list of entries from [tool.uv].constraint-dependencies. ConstraintDeps []string + // EnvironmentVersion is the serverless environment version written into + // [tool.databricks.environment].environment_version (e.g. "5"). It is not + // parsed from the artifact but set by the pipeline from the resolved compute + // target; it is empty for cluster targets, where the section is not managed. + EnvironmentVersion string } // cacheFileName maps an env key to a single, collision-free cache filename. diff --git a/libs/localenv/merge.go b/libs/localenv/merge.go index 795492ba876..4c547832e51 100644 --- a/libs/localenv/merge.go +++ b/libs/localenv/merge.go @@ -28,11 +28,20 @@ var ( // Region names reported back to the caller via MergeManaged's regions return value. const ( - regionRequiresPython = "requires-python" - regionDatabricksConnect = "databricks-connect" - regionToolUv = "tool.uv.constraint-dependencies" + regionRequiresPython = "requires-python" + regionDatabricksConnect = "databricks-connect" + regionToolUv = "tool.uv.constraint-dependencies" + regionDatabricksEnvironment = "tool.databricks.environment" ) +// databricksEnvironmentTable is the TOML table header that carries the +// serverless environment version, and environmentVersionRe matches the start of +// its managed environment_version assignment (capturing leading whitespace so it +// is preserved when the value is replaced). +const databricksEnvironmentTable = "[tool.databricks.environment]" + +var environmentVersionRe = regexp.MustCompile(`^(\s*)environment_version\s*=`) + var ( // tableHeaderRe matches a TOML table header line: a standard table like // "[project]" / "[tool.uv]" or an array-of-tables like "[[tool.uv.index]]", @@ -88,7 +97,7 @@ func planDBConnect(target []byte, c Constraints) dbconnectPlan { return dbconnectPlan{replacedDevPin: replaced, removed: removed} } -// MergeManaged applies the three managed transforms to target, preserving every other +// MergeManaged applies the managed transforms to target, preserving every other // byte (comments, ordering, whitespace). It returns the merged bytes and the list of // regions that actually changed. The operation is idempotent: feeding its own output // back in produces identical bytes. @@ -144,6 +153,11 @@ func MergeManaged(target []byte, c Constraints) (merged []byte, regions []string regions = append(regions, regionDatabricksConnect) } + lines, envChanged := mergeDatabricksEnvironment(lines, c.EnvironmentVersion) + if envChanged { + regions = append(regions, regionDatabricksEnvironment) + } + lines, uvChanged := mergeToolUv(lines, c.ConstraintDeps) if uvChanged { regions = append(regions, regionToolUv) @@ -239,6 +253,45 @@ func mergeRequiresPython(lines []string, value string) ([]string, bool) { return inserted, true } +// mergeDatabricksEnvironment pins environment_version to version within the +// env-owned [tool.databricks.environment] table: it replaces an existing value +// (preserving the line's indentation and any inline comment), inserts the key +// when the table exists without it, and appends the table when it is absent. +// An empty version (a cluster target) is a no-op — the section is only written +// for serverless targets, so an existing one is left untouched rather than +// removed. Returns whether the line slice changed. +func mergeDatabricksEnvironment(lines []string, version string) ([]string, bool) { + if version == "" { + return lines, false + } + + header, end, found := tableBounds(lines, databricksEnvironmentTable) + if !found { + return appendManagedBlock(lines, []string{databricksEnvironmentTable, fmt.Sprintf(`environment_version = "%s"`, version)}), true + } + + for i := header + 1; i < end; i++ { + m := environmentVersionRe.FindStringSubmatch(lines[i]) + if m == nil { + continue + } + // Only the value is managed; a trailing inline comment is user content. + replacement := fmt.Sprintf(`%senvironment_version = "%s"%s`, m[1], version, trailingComment(lines[i])) + if lines[i] == replacement { + return lines, false + } + lines[i] = replacement + return lines, true + } + + // Table exists but has no environment_version: insert directly under the header. + inserted := make([]string, 0, len(lines)+1) + inserted = append(inserted, lines[:header+1]...) + inserted = append(inserted, fmt.Sprintf(`environment_version = "%s"`, version)) + inserted = append(inserted, lines[header+1:]...) + return inserted, true +} + // trailingComment returns the inline TOML comment suffix of a line (including the // leading whitespace and "#"), or "" if there is none. It ignores "#" characters // inside a quoted string so a value like requires-python = ">=3.10 # x" is not @@ -1100,9 +1153,10 @@ func equalLines(a, b []string) bool { const freshProjectVersion = "0.0.0" // RenderFreshPyproject produces a complete managed pyproject.toml for a project that has -// none, with [project], [dependency-groups].dev (carrying the databricks-connect pin), and -// the marker-bracketed [tool.uv] constraint block. When c.DatabricksConnect is empty -// (constraints-only mode) the dev group is emitted empty rather than with a blank entry. +// none, with [project], [dependency-groups].dev (carrying the databricks-connect pin), the +// [tool.databricks.environment] section (serverless targets only), and the marker-bracketed +// [tool.uv] constraint block. When c.DatabricksConnect is empty (constraints-only mode) the +// dev group is emitted empty rather than with a blank entry. func RenderFreshPyproject(projectName string, c Constraints) []byte { var b strings.Builder b.WriteString("[project]\n") @@ -1120,6 +1174,13 @@ func RenderFreshPyproject(projectName string, c Constraints) []byte { b.WriteString("dev = []\n") } b.WriteString("\n") + // The serverless environment version is written only for serverless targets; + // a cluster target leaves EnvironmentVersion empty and omits the section. + if c.EnvironmentVersion != "" { + b.WriteString(databricksEnvironmentTable + "\n") + fmt.Fprintf(&b, "environment_version = %q\n", c.EnvironmentVersion) + b.WriteString("\n") + } for _, line := range renderToolUvBlock(c.ConstraintDeps, true) { b.WriteString(line) b.WriteString("\n") diff --git a/libs/localenv/merge_test.go b/libs/localenv/merge_test.go index 9e3ffbdb8f5..96ced5dfc4f 100644 --- a/libs/localenv/merge_test.go +++ b/libs/localenv/merge_test.go @@ -452,12 +452,154 @@ func TestRenderFreshPyproject(t *testing.T) { assert.Contains(t, s, managedMarkerStart) assert.Contains(t, s, managedMarkerEnd) assert.Contains(t, s, "pydantic~=2.10.6") + // A cluster target (no EnvironmentVersion) writes no [tool.databricks.environment]. + assert.NotContains(t, s, "[tool.databricks.environment]") // A fresh render is itself a no-op under MergeManaged (already fully managed). merged, _, err := MergeManaged(out, testConstraints()) require.NoError(t, err) assert.Equal(t, s, string(merged)) } +func TestRenderFreshPyprojectServerlessWritesEnvironment(t *testing.T) { + c := testConstraints() + c.EnvironmentVersion = "5" + out := RenderFreshPyproject("demo", c) + s := string(out) + assert.Contains(t, s, "[tool.databricks.environment]") + assert.Contains(t, s, `environment_version = "5"`) + requireValidTOML(t, out) + // A fresh render is itself a no-op under MergeManaged (already fully managed). + merged, _, err := MergeManaged(out, c) + require.NoError(t, err) + assert.Equal(t, s, string(merged)) +} + +func TestMergeInsertsDatabricksEnvironmentWhenAbsent(t *testing.T) { + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "[tool.databricks.environment]") + assert.Contains(t, s, `environment_version = "5"`) + assert.Contains(t, regions, regionDatabricksEnvironment) + requireValidTOML(t, out) + // Idempotent. + twice, _, err := MergeManaged(out, c) + require.NoError(t, err) + assert.Equal(t, s, string(twice)) +} + +func TestMergeReplacesExistingEnvironmentVersionPreservingComment(t *testing.T) { + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] + +[tool.databricks.environment] +environment_version = "4" # pinned +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, `environment_version = "5" # pinned`) + assert.NotContains(t, s, `environment_version = "4"`) + assert.Contains(t, regions, regionDatabricksEnvironment) + // The table is refreshed in place, not duplicated. + assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) + requireValidTOML(t, out) +} + +func TestMergeInsertsEnvironmentVersionKeyWhenTableExists(t *testing.T) { + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] + +[tool.databricks.environment] +# user note +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, _, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, `environment_version = "5"`) + assert.Contains(t, s, "# user note") + assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) + requireValidTOML(t, out) +} + +func TestMergeEnvironmentNoopForClusterTarget(t *testing.T) { + // A cluster target leaves EnvironmentVersion empty: the section is never + // written, and an existing one is left untouched rather than removed. + in := []byte(`[project] +requires-python = ">=3.10" + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] + +[tool.databricks.environment] +environment_version = "4" +`) + c := testConstraints() // EnvironmentVersion == "": cluster target. + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + assert.Contains(t, string(out), `environment_version = "4"`) + assert.NotContains(t, regions, regionDatabricksEnvironment) +} + +func TestMergeAddsEnvironmentToPreFeatureManagedFile(t *testing.T) { + // The common upgrade path: a file a pre-feature CLI wrote for a serverless + // target already carries the managed [tool.uv] marker block but no + // [tool.databricks.environment] section. Re-running the new CLI must add the + // section, keep exactly one managed block, and stay idempotent. + in := []byte(`[project] +name = "demo" +version = "0.0.0" +requires-python = "==3.12.*" + +[dependency-groups] +dev = [ + "databricks-connect~=17.2.0", +] + +` + managedMarkerStart + ` +[tool.uv] +constraint-dependencies = [ + "pydantic~=2.10.6", + "anyio~=4.6.2", +] +` + managedMarkerEnd + ` +`) + c := testConstraints() + c.EnvironmentVersion = "5" + out, regions, err := MergeManaged(in, c) + require.NoError(t, err) + s := string(out) + assert.Contains(t, s, "[tool.databricks.environment]") + assert.Contains(t, s, `environment_version = "5"`) + assert.Contains(t, regions, regionDatabricksEnvironment) + // The pre-existing managed [tool.uv] block is neither duplicated nor disturbed. + assert.Equal(t, 1, countOccurrences(s, managedMarkerStart)) + assert.Equal(t, 1, countOccurrences(s, "[tool.databricks.environment]")) + requireValidTOML(t, out) + // Idempotent on the upgraded file. + twice, _, err := MergeManaged(out, c) + require.NoError(t, err) + assert.Equal(t, s, string(twice)) +} + func TestMergeStripsMultiLineConstraintDepsWithBracketInFirstElement(t *testing.T) { // The user's stale constraint-dependencies is a multi-line array whose FIRST // element line contains a "]" inside an extras spec. A naive diff --git a/libs/localenv/pipeline.go b/libs/localenv/pipeline.go index 46a5507a9fa..39d13bb1b4b 100644 --- a/libs/localenv/pipeline.go +++ b/libs/localenv/pipeline.go @@ -205,8 +205,10 @@ func (p *Pipeline) run(ctx context.Context) error { } // Phase: merge — compute the merged pyproject.toml (in-memory, no writes yet). + // The serverless environment version (empty for cluster targets) is written + // into [tool.databricks.environment] so the project also runs in serverless Jobs. p.report(ctx, PhaseMerge) - mergedBytes, greenfield, err := p.mergePlan(ctx, pyMinor, c, dbcPin) + mergedBytes, greenfield, err := p.mergePlan(ctx, pyMinor, c, dbcPin, compute.ServerlessEnvironmentVersion()) if err != nil { return err } @@ -277,8 +279,9 @@ func (p *Pipeline) backupPath() string { // mergePlan computes the merged pyproject.toml bytes (without writing to disk), // decides greenfield vs. existing, and builds the Plan (populated only under // --dry-run). dbcPin is the databricks-connect pin to inject, or "" in -// constraints-only mode. -func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, dbcPin string) (merged []byte, greenfield bool, err error) { +// constraints-only mode. envVersion is the serverless environment version to +// write into [tool.databricks.environment], or "" for a cluster target. +func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, dbcPin, envVersion string) (merged []byte, greenfield bool, err error) { pyproject := p.pyprojectPath() backup := p.backupPath() @@ -305,9 +308,17 @@ func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, greenfield = baseBytes == nil // The artifact drives the merge; in constraints-only mode we clear the - // databricks-connect pin so it is neither written nor asserted. + // databricks-connect pin so it is neither written nor asserted. envVersion is + // the resolved serverless version (empty for cluster targets). + // + // envVersion is deliberately NOT cleared in constraints-only mode: unlike the + // databricks-connect pin (a managed *dependency* the mode opts out of), the + // environment version records the resolved compute *target*, which the mode + // still resolves. Recording it keeps the target discoverable for VS Code and + // serverless Jobs even when dependency management is turned off. effective := *c effective.DatabricksConnect = dbcPin + effective.EnvironmentVersion = envVersion var changedRegions []string if greenfield { @@ -318,6 +329,9 @@ func (p *Pipeline) mergePlan(_ context.Context, pyMinor string, c *Constraints, if dbcPin != "" { changedRegions = append(changedRegions, regionDatabricksConnect) } + if envVersion != "" { + changedRegions = append(changedRegions, regionDatabricksEnvironment) + } } else { merged, changedRegions, err = MergeManaged(baseBytes, effective) if err != nil { diff --git a/libs/localenv/pipeline_test.go b/libs/localenv/pipeline_test.go index 5b007d7571d..c4748a31985 100644 --- a/libs/localenv/pipeline_test.go +++ b/libs/localenv/pipeline_test.go @@ -480,6 +480,10 @@ func TestPipelineGreenfieldCreatesNewPyproject(t *testing.T) { data, readErr := os.ReadFile(filepath.Join(dir, "pyproject.toml")) require.NoError(t, readErr) assert.Contains(t, string(data), `"databricks-connect~=17.2.0",`) + // A serverless target records the environment version so the same project + // also runs in serverless Jobs (DECO-27998). + assert.Contains(t, string(data), "[tool.databricks.environment]") + assert.Contains(t, string(data), `environment_version = "4"`) // No backup created when pyproject.toml did not previously exist. assert.NoFileExists(t, filepath.Join(dir, "pyproject.toml.bak")) } diff --git a/libs/localenv/result.go b/libs/localenv/result.go index 1e704bcc55d..99266128abc 100644 --- a/libs/localenv/result.go +++ b/libs/localenv/result.go @@ -178,7 +178,7 @@ func (c *ComputeInfo) Label() string { switch { case c.ServerlessVersion != "": // ServerlessVersion is normalized to "v4"; drop the "v" for display. - return "serverless " + strings.TrimPrefix(c.ServerlessVersion, "v") + return "serverless " + c.ServerlessEnvironmentVersion() case c.ClusterID != "": return "cluster " + c.ClusterID case c.SparkVersion != "": @@ -192,6 +192,15 @@ func (c *ComputeInfo) Label() string { } } +// ServerlessEnvironmentVersion returns the bare serverless environment version +// (e.g. "5") to write into [tool.databricks.environment].environment_version. +// ServerlessVersion is normalized to "vN", so the leading "v" is dropped to +// match the documented bare-number form. It is empty for a cluster target +// (which leaves ServerlessVersion unset), where the section is not managed. +func (c *ComputeInfo) ServerlessEnvironmentVersion() string { + return strings.TrimPrefix(c.ServerlessVersion, "v") +} + // ResolvedInfo is the resolved environment definition (spec §6 "resolved"). // DBConnectVersion is omitted in constraints-only mode. type ResolvedInfo struct { @@ -265,6 +274,13 @@ const ( // groups — since constraint-dependencies applies to the whole resolution. Emitted // only when the ranges are provably disjoint; ambiguous cases are not flagged. WarnUserConstraintConflict = "W_USER_CONSTRAINT_CONFLICT" + // WarnStaleEnvironmentVersion: the target is a cluster, which does not manage the + // serverless environment section, but the file carries a [tool.databricks.environment] + // environment_version left over from an earlier serverless run. The value is not + // updated (cluster targets are a no-op there), so it now describes a target the + // project is no longer set up for — worth surfacing because VS Code and serverless + // Jobs read that section as a source of truth. + WarnStaleEnvironmentVersion = "W_STALE_ENVIRONMENT_VERSION" ) // Result is the full outcome of a sync run and the root of the --json object diff --git a/libs/localenv/warnings.go b/libs/localenv/warnings.go index d6737c31926..5e1c15f2203 100644 --- a/libs/localenv/warnings.go +++ b/libs/localenv/warnings.go @@ -77,6 +77,17 @@ type userPyprojectTOML struct { // Every group is decoded, not just dev: uv locks all declared groups, so a pin in // any of them is subject to constraint-dependencies (see resolutionRequirements). DependencyGroups map[string]any `toml:"dependency-groups"` + // Tool.Databricks.Environment.EnvironmentVersion is the serverless version the + // merge manages; it is read here only to warn when it is left stale on a cluster + // target. A non-string value here is malformed and, like any decode error, yields + // no warnings rather than a crash. + Tool struct { + Databricks struct { + Environment struct { + EnvironmentVersion string `toml:"environment_version"` + } `toml:"environment"` + } `toml:"databricks"` + } `toml:"tool"` } // devGroup is the dependency group whose databricks-connect pin the merge manages. @@ -198,6 +209,19 @@ func detectMergeWarnings(userPyproject []byte, c Constraints, plan dbconnectPlan } warnings = append(warnings, constraintConflicts(survivors, c.ConstraintDeps)...) + + // A cluster target leaves c.EnvironmentVersion empty and does not manage the + // serverless environment section, so an environment_version left over from an + // earlier serverless run is neither refreshed nor removed. Warn that it is now + // stale rather than let it silently misdescribe the target to a downstream reader. + if c.EnvironmentVersion == "" { + if ev := strings.TrimSpace(p.Tool.Databricks.Environment.EnvironmentVersion); ev != "" { + warnings = append(warnings, Warning{ + Code: WarnStaleEnvironmentVersion, + Message: fmt.Sprintf("[tool.databricks.environment] environment_version %q is left from a serverless target but the current target is a cluster; it is not updated", ev), + }) + } + } return warnings } diff --git a/libs/localenv/warnings_test.go b/libs/localenv/warnings_test.go index 3fd6bb1e573..d2f4c07cfb7 100644 --- a/libs/localenv/warnings_test.go +++ b/libs/localenv/warnings_test.go @@ -72,6 +72,52 @@ dev = ["databricks-connect~=16.1.0"] assert.Empty(t, detectWarnings(user, c)) } +func TestDetectMergeWarningsStaleEnvironmentVersionOnClusterTarget(t *testing.T) { + // A cluster target (c.EnvironmentVersion == "") that finds a leftover serverless + // environment_version warns that the value is now stale. + user := []byte(`[project] +requires-python = "==3.12.*" + +[dependency-groups] +dev = ["databricks-connect~=18.0.0"] + +[tool.databricks.environment] +environment_version = "5" +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0", EnvironmentVersion: ""} + got := detectWarnings(user, c) + assert.Equal(t, []string{WarnStaleEnvironmentVersion}, codes(got)) + assert.Contains(t, got[0].Message, `"5"`) +} + +func TestDetectMergeWarningsNoStaleWarningForServerlessTarget(t *testing.T) { + // A serverless target manages the section (refreshes it), so an existing value + // is not stale and must not warn. + user := []byte(`[project] +requires-python = "==3.12.*" + +[dependency-groups] +dev = ["databricks-connect~=18.0.0"] + +[tool.databricks.environment] +environment_version = "4" +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0", EnvironmentVersion: "5"} + assert.NotContains(t, codes(detectWarnings(user, c)), WarnStaleEnvironmentVersion) +} + +func TestDetectMergeWarningsNoStaleWarningWhenSectionAbsent(t *testing.T) { + // A cluster target with no existing section has nothing to go stale. + user := []byte(`[project] +requires-python = "==3.12.*" + +[dependency-groups] +dev = ["databricks-connect~=18.0.0"] +`) + c := Constraints{RequiresPython: "==3.12.*", DatabricksConnect: "databricks-connect~=18.0.0", EnvironmentVersion: ""} + assert.NotContains(t, codes(detectWarnings(user, c)), WarnStaleEnvironmentVersion) +} + func TestDetectMergeWarningsUserConstraintConflict(t *testing.T) { user := []byte(`[project] requires-python = "==3.12.*"