AIR CLI Integration: remove requirements file path for config

acceptance/experimental/air/run-submit-deps/.gitattributes

@@ -1,6 +1,3 @@
-# These YAML files' contents are uploaded verbatim (as training_config.yaml and
-# requirements.yaml), so their line endings must stay \n on every OS — a Windows
-# \r would change the recorded payload.
+# run.yaml's contents are uploaded verbatim (as training_config.yaml), so its line
+# endings must stay \n on every OS — a Windows \r would change the recorded payload.
 run.yaml text eol=lf
-run-file.yaml text eol=lf
-reqs.yaml text eol=lf

acceptance/experimental/air/run-submit-deps/output.txt

@@ -56,59 +56,6 @@ Tip: use --watch to stream logs until the run completes.
   }
 }
 
-=== file-form deps: version comes from the requirements file
+=== a requirements.yaml file path is rejected; deps must be inline
 >>> [CLI] experimental air run -f run-file.yaml
-Submitting experiment: deps-file-smoke
-Submitted workload with Job Run ID: 555
-View job run at: [DATABRICKS_URL]/jobs/runs/555
-
-Tip: use --watch to stream logs until the run completes.
-
-=== file-form deps: the requirements file is not uploaded either
->>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep
-{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"}
-{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-file-smoke\ncommand: python train.py\ncompute:\n  accelerator_type: GPU_1xH100\n  num_accelerators: 1\nenvironment:\n  dependencies: ./reqs.yaml\n"}
-
-=== file-form deps ride on environments[].spec.dependencies
->>> print_requests.py //api/2.2/jobs/runs/submit
-{
-  "method": "POST",
-  "path": "/api/2.2/jobs/runs/submit",
-  "body": {
-    "environments": [
-      {
-        "environment_key": "default",
-        "spec": {
-          "dependencies": [
-            "numpy",
-            "torch==2.3.0"
-          ],
-          "environment_version": "5"
-        }
-      }
-    ],
-    "idempotency_token": "[UUID]",
-    "run_name": "deps-file-smoke",
-    "tasks": [
-      {
-        "ai_runtime_task": {
-          "deployments": [
-            {
-              "command_path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/command.sh",
-              "compute": {
-                "accelerator_count": 1,
-                "accelerator_type": "GPU_1xH100"
-              }
-            }
-          ],
-          "experiment": "deps-file-smoke"
-        },
-        "environment_key": "default",
-        "max_retries": 3,
-        "retry_on_timeout": true,
-        "run_if": "ALL_SUCCESS",
-        "task_key": "deps-file-smoke"
-      }
-    ]
-  }
-}
+Error: invalid config run-file.yaml: environment.dependencies must be a list of packages or reference a requirements.txt (see https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference). A direct file reference is not supported

acceptance/experimental/air/run-submit-deps/script

@@ -7,11 +7,5 @@ trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort -
 title "declared deps ride on environments[].spec.dependencies"
 trace print_requests.py //api/2.2/jobs/runs/submit
 
-title "file-form deps: version comes from the requirements file"
-trace $CLI experimental air run -f run-file.yaml
-
-title "file-form deps: the requirements file is not uploaded either"
-trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep
-
-title "file-form deps ride on environments[].spec.dependencies"
-trace print_requests.py //api/2.2/jobs/runs/submit
+title "a requirements.yaml file path is rejected; deps must be inline"
+musterr trace $CLI experimental air run -f run-file.yaml

acceptance/experimental/air/run-submit-deps/test.toml

@@ -17,7 +17,3 @@ Response.Body = '''
 [[Repls]]
 Old = 'deps-smoke_[0-9a-f]{16}'
 New = 'deps-smoke_[RUN_ID]'
-
-[[Repls]]
-Old = 'deps-file-smoke_[0-9a-f]{16}'
-New = 'deps-file-smoke_[RUN_ID]'

experimental/air/cmd/runconfig.go

@@ -251,40 +251,29 @@ func (e *environmentConfig) validate() error {
 		return e.DockerImage.validate()
 	}
 
-	// version pins the client image version, which is only meaningful for an
-	// inline (list) dependency set — a requirements.yaml file carries its own.
-	if e.Version.set {
-		if e.Dependencies.set && !e.Dependencies.isList {
-			return errors.New("'environment.version' is only valid with inline dependencies (a list); when 'dependencies' points to a requirements.yaml file, set the version inside that file")
-		}
-		if !e.Dependencies.set {
-			return errors.New("'environment.version' requires inline 'dependencies' (a list of packages)")
-		}
+	// version pins the client image version, which is only meaningful alongside an
+	// inline dependency set.
+	if e.Version.set && !e.Dependencies.set {
+		return errors.New("'environment.version' requires inline 'dependencies' (a list of packages)")
 	}
 
 	return nil
 }
 
-// dependencies is environment.dependencies, which is polymorphic: a string is a
-// path to a requirements.yaml file; a list is an inline package list.
+// dependencies is environment.dependencies: an inline list of packages. A scalar
+// (e.g. a path to a requirements file) is rejected — the list may itself reference
+// a requirements.txt, but dependencies must be given as a list.
 type dependencies struct {
-	set    bool
-	isList bool
-	path   string
-	list   []string
+	set  bool
+	list []string
 }
 
 func (d *dependencies) UnmarshalYAML(node *yaml.Node) error {
-	switch node.Kind {
-	case yaml.ScalarNode:
-		d.set, d.isList = true, false
-		return node.Decode(&d.path)
-	case yaml.SequenceNode:
-		d.set, d.isList = true, true
-		return node.Decode(&d.list)
-	default:
-		return errors.New("environment.dependencies must be a string path or a list of packages")
+	if node.Kind != yaml.SequenceNode {
+		return errors.New("environment.dependencies must be a list of packages or reference a requirements.txt (see https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference). A direct file reference is not supported")
 	}
+	d.set = true
+	return node.Decode(&d.list)
 }
 
 // stringOrInt holds a scalar that may be a string or an integer in YAML

experimental/air/cmd/runconfig_launch.go

@@ -36,27 +36,17 @@ func (c *runConfig) dockerImageURL() string {
 	return ""
 }
 
-// requirementsFile returns the path to a requirements file when
-// environment.dependencies is a string, and whether it was set.
-func (c *runConfig) requirementsFile() (string, bool) {
-	if c.Environment == nil || !c.Environment.Dependencies.set || c.Environment.Dependencies.isList {
-		return "", false
-	}
-	return c.Environment.Dependencies.path, true
-}
-
-// inlineDependencies returns the inline package list when
-// environment.dependencies is a list, and whether it was set.
+// inlineDependencies returns the inline package list from
+// environment.dependencies, and whether it was set.
 func (c *runConfig) inlineDependencies() ([]string, bool) {
-	if c.Environment == nil || !c.Environment.Dependencies.set || !c.Environment.Dependencies.isList {
+	if c.Environment == nil || !c.Environment.Dependencies.set {
 		return nil, false
 	}
 	return c.Environment.Dependencies.list, true
 }
 
 // runtimeVersion returns the client image version from environment.version when
-// set. For a requirements-file dependency set, the version lives in that file and
-// is resolved at launch, not here.
+// set.
 func (c *runConfig) runtimeVersion() (string, bool) {
 	if c.Environment == nil || !c.Environment.Version.set {
 		return "", false

experimental/air/cmd/runconfig_launch_test.go

@@ -39,32 +39,17 @@ func TestRunConfigDockerImageURL(t *testing.T) {
 func TestRunConfigDependencies(t *testing.T) {
 	t.Run("unset", func(t *testing.T) {
 		c := &runConfig{}
-		_, ok := c.requirementsFile()
-		assert.False(t, ok)
-		_, ok = c.inlineDependencies()
-		assert.False(t, ok)
-	})
-
-	t.Run("file path", func(t *testing.T) {
-		c := &runConfig{Environment: &environmentConfig{
-			Dependencies: dependencies{set: true, isList: false, path: "req.yaml"},
-		}}
-		path, ok := c.requirementsFile()
-		assert.True(t, ok)
-		assert.Equal(t, "req.yaml", path)
-		_, ok = c.inlineDependencies()
+		_, ok := c.inlineDependencies()
 		assert.False(t, ok)
 	})
 
 	t.Run("inline list", func(t *testing.T) {
 		c := &runConfig{Environment: &environmentConfig{
-			Dependencies: dependencies{set: true, isList: true, list: []string{"torch", "numpy"}},
+			Dependencies: dependencies{set: true, list: []string{"torch", "numpy"}},
 		}}
 		list, ok := c.inlineDependencies()
 		assert.True(t, ok)
 		assert.Equal(t, []string{"torch", "numpy"}, list)
-		_, ok = c.requirementsFile()
-		assert.False(t, ok)
 	})
 }
 

experimental/air/cmd/runconfig_test.go

@@ -82,7 +82,7 @@ permissions:
 	require.NoError(t, err)
 	assert.Equal(t, gpuType8xH100, gpuType(cfg.Compute.AcceleratorType))
 	require.NotNil(t, cfg.Environment)
-	assert.True(t, cfg.Environment.Dependencies.isList)
+	assert.True(t, cfg.Environment.Dependencies.set)
 	assert.Equal(t, []string{"torch==2.3.0", "numpy"}, cfg.Environment.Dependencies.list)
 	assert.True(t, cfg.Environment.Version.set)
 	assert.Equal(t, "5", cfg.Environment.Version.raw)
@@ -94,18 +94,17 @@ permissions:
 	assert.Len(t, cfg.Permissions, 2)
 }
 
-// TestLoadRunConfig_PolymorphicFields exercises the str|list, str|int, and
-// bool|str unions decoded by custom UnmarshalYAML.
+// TestLoadRunConfig_PolymorphicFields exercises the str|int and bool|str unions
+// decoded by custom UnmarshalYAML, plus the rejection of the removed
+// dependencies string form.
 func TestLoadRunConfig_PolymorphicFields(t *testing.T) {
-	t.Run("dependencies as string path", func(t *testing.T) {
-		cfg, err := loadRunConfig(writeConfig(t, minimalConfig+`
+	t.Run("dependencies as string path is rejected", func(t *testing.T) {
+		_, err := loadRunConfig(writeConfig(t, minimalConfig+`
 environment:
   dependencies: requirements.yaml
 `))
-		require.NoError(t, err)
-		assert.True(t, cfg.Environment.Dependencies.set)
-		assert.False(t, cfg.Environment.Dependencies.isList)
-		assert.Equal(t, "requirements.yaml", cfg.Environment.Dependencies.path)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "must be a list of packages")
 	})
 
 	t.Run("git remote as bool true is rejected", func(t *testing.T) {
@@ -278,7 +277,7 @@ func TestEnvironmentConfigValidate(t *testing.T) {
 			"docker image with deps conflicts",
 			environmentConfig{
 				DockerImage:  &dockerImageConfig{URL: "org/repo:tag"},
-				Dependencies: dependencies{set: true, isList: true, list: []string{"torch"}},
+				Dependencies: dependencies{set: true, list: []string{"torch"}},
 			},
 			"not allowed: dependencies",
 		},
@@ -287,14 +286,6 @@ func TestEnvironmentConfigValidate(t *testing.T) {
 			environmentConfig{DockerImage: &dockerImageConfig{URL: "  "}},
 			"docker_image.url cannot be empty",
 		},
-		{
-			"version with file deps",
-			environmentConfig{
-				Version:      stringOrInt{set: true, raw: "5"},
-				Dependencies: dependencies{set: true, isList: false, path: "req.yaml"},
-			},
-			"only valid with inline dependencies",
-		},
 		{
 			"version without deps",
 			environmentConfig{Version: stringOrInt{set: true, raw: "5"}},
@@ -304,7 +295,7 @@ func TestEnvironmentConfigValidate(t *testing.T) {
 			"version with inline deps ok",
 			environmentConfig{
 				Version:      stringOrInt{set: true, raw: "5"},
-				Dependencies: dependencies{set: true, isList: true, list: []string{"torch"}},
+				Dependencies: dependencies{set: true, list: []string{"torch"}},
 			},
 			"",
 		},

experimental/air/cmd/runsubmit.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"path"
-	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -39,26 +38,6 @@ func dlRuntimeImage(ctx context.Context, runtimeVersion string) string {
 	return strings.TrimPrefix(img, "CLIENT-GPU-")
 }
 
-// environmentDependencies resolves the user's declared dependencies as a flat
-// list to carry inline on the serverless environment's spec.dependencies: the
-// inline list directly, or the dependencies read from a requirements file
-// (resolved against the config's directory). For file-form deps it also returns
-// the version declared inside that file, which selects the runtime image since
-// top-level environment.version is not allowed there. Returns nil when none are
-// declared.
-func environmentDependencies(cfg *runConfig, configPath string) (deps []string, fileVersion string, err error) {
-	if deps, ok := cfg.inlineDependencies(); ok {
-		return deps, "", nil
-	}
-	if reqPath, ok := cfg.requirementsFile(); ok {
-		if !filepath.IsAbs(reqPath) {
-			reqPath = filepath.Join(filepath.Dir(configPath), reqPath)
-		}
-		return readRequirementsDependencies(reqPath)
-	}
-	return nil, "", nil
-}
-
 // buildSubmitPayload assembles the runs/submit payload. commandPath is the
 // workspace path of the uploaded command.sh; dlImage is the runtime channel;
 // usagePolicyID is the already-resolved policy id ("" when the run has none);
@@ -182,12 +161,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
 		}
 	}
 
-	// Resolve dependencies before any upload too, so a bad requirements file fails
-	// fast without leaving orphaned artifacts in the workspace.
-	deps, fileVersion, err := environmentDependencies(cfg, configPath)
-	if err != nil {
-		return 0, "", err
-	}
+	deps, _ := cfg.inlineDependencies()
 
 	experimentDir := ""
 	if cfg.MLflowExperimentDirectory != nil {
@@ -237,12 +211,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
 		}
 	}
 
-	// Top-level environment.version wins; for file-form deps it is disallowed, so
-	// fall back to the version declared inside the requirements file.
-	runtimeVersion, ok := cfg.runtimeVersion()
-	if !ok {
-		runtimeVersion = fileVersion
-	}
+	runtimeVersion, _ := cfg.runtimeVersion()
 	payload := buildSubmitPayload(cfg, path.Join(funcDir, commandScriptName), dlRuntimeImage(ctx, runtimeVersion), usagePolicyID, snap, deps)
 	payload.IdempotencyToken = token