feat: add non-cone mode support for sparse checkout

agent/agent_configuration.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/buildkite/agent/v3/env"
+	"github.com/buildkite/agent/v3/internal/job"
 )
 
 // AgentConfiguration is the run-time configuration for an agent that
@@ -29,6 +30,7 @@ type AgentConfiguration struct {
 	GitCommitVerification           string
 	GitFetchFlags                   string
 	GitSparseCheckoutPaths          []string
+	GitSparseCheckoutMode           job.SparseCheckoutMode
 	GitSubmodules                   bool
 	GitSubmoduleCloneConfig         []string
 	SkipCheckout                    bool

agent/integration/job_environment_integration_test.go

@@ -536,6 +536,45 @@ func TestCheckoutScopedJobEnvOverrideHonorsCheckoutOverrideMode(t *testing.T) {
 			},
 			wantEnvValue: "job/path",
 		},
+		// The sparse-checkout mode shares the from-job floor with paths: pipeline/step
+		// env may set it under the default mode; only strict locks it to agent config.
+		{
+			name:    "from_job_allows_job_env_sparse_checkout_mode",
+			varName: "BUILDKITE_GIT_SPARSE_CHECKOUT_MODE",
+			jobEnv: map[string]string{
+				"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE": "no-cone",
+			},
+			agentCfg: agent.AgentConfiguration{
+				GitSparseCheckoutMode: "cone",
+				CheckoutOverrideMode:  env.CheckoutOverrideFromJob,
+			},
+			wantEnvValue: "no-cone",
+		},
+		{
+			name:    "strict_locks_sparse_checkout_mode_to_agent_config",
+			varName: "BUILDKITE_GIT_SPARSE_CHECKOUT_MODE",
+			jobEnv: map[string]string{
+				"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE": "no-cone",
+			},
+			agentCfg: agent.AgentConfiguration{
+				GitSparseCheckoutMode: "cone",
+				CheckoutOverrideMode:  env.CheckoutOverrideStrict,
+			},
+			wantEnvValue:       "cone",
+			wantIgnoredEnvVars: []string{"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE"},
+		},
+		{
+			name:    "none_allows_job_env_to_override_sparse_checkout_mode",
+			varName: "BUILDKITE_GIT_SPARSE_CHECKOUT_MODE",
+			jobEnv: map[string]string{
+				"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE": "no-cone",
+			},
+			agentCfg: agent.AgentConfiguration{
+				GitSparseCheckoutMode: "cone",
+				CheckoutOverrideMode:  env.CheckoutOverrideNone,
+			},
+			wantEnvValue: "no-cone",
+		},
 		// Inverse cases: when the agent config sits on the side that emits no var
 		// by default, the lock must still force the agent value (regression for the
 		// leak where backend job env survived while checkout override was locked).

agent/job_runner.go

@@ -408,10 +408,11 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
 	// Most checkout-scoped vars are locked against the backend job env in every
 	// mode except none; only none lets pipeline/step env override agent config,
 	// matching the rule secrets follow (see IsCheckoutLockedForSecrets). The
-	// exception is sparse-checkout paths, which have a from-job floor: a step's
-	// checkout.sparse config is honored under the default mode too, and only strict
-	// locks it (see IsCheckoutLockedForJobEnv). Within-job sources (hooks, plugins,
-	// the Job API) are governed separately and only strict locks them.
+	// exception is the sparse-checkout paths and mode, which have a from-job floor:
+	// a step's checkout.sparse config is honored under the default mode too, and
+	// only strict locks them (see IsCheckoutLockedForJobEnv). Within-job sources
+	// (hooks, plugins, the Job API) are governed separately and only strict locks
+	// them.
 	checkoutMode := r.conf.AgentConfiguration.CheckoutOverrideMode
 
 	// Create a clone of our jobs environment. We'll then set the
@@ -669,6 +670,7 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	setCheckoutEnv("BUILDKITE_GIT_CLONE_FLAGS", r.conf.AgentConfiguration.GitCloneFlags)
 	setCheckoutEnv("BUILDKITE_GIT_FETCH_FLAGS", r.conf.AgentConfiguration.GitFetchFlags)
 	setCheckoutEnv("BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS", strings.Join(r.conf.AgentConfiguration.GitSparseCheckoutPaths, ","))
+	setCheckoutEnv("BUILDKITE_GIT_SPARSE_CHECKOUT_MODE", r.conf.AgentConfiguration.GitSparseCheckoutMode.String())
 	setEnv("BUILDKITE_GIT_CLONE_MIRROR_FLAGS", r.conf.AgentConfiguration.GitCloneMirrorFlags)
 	setEnv("BUILDKITE_GIT_MIRROR_CHECKOUT_MODE", r.conf.AgentConfiguration.GitMirrorCheckoutMode)
 	setCheckoutEnv("BUILDKITE_GIT_CLEAN_FLAGS", r.conf.AgentConfiguration.GitCleanFlags)

clicommand/agent_start.go

@@ -33,6 +33,7 @@ import (
 	awssigner "github.com/buildkite/agent/v3/internal/cryptosigner/aws"
 	gcpsigner "github.com/buildkite/agent/v3/internal/cryptosigner/gcp"
 	"github.com/buildkite/agent/v3/internal/experiments"
+	"github.com/buildkite/agent/v3/internal/job"
 	"github.com/buildkite/agent/v3/internal/job/hook"
 	"github.com/buildkite/agent/v3/internal/osutil"
 	"github.com/buildkite/agent/v3/internal/process"
@@ -165,6 +166,7 @@ type AgentStartConfig struct {
 	GitCleanFlags               string   `cli:"git-clean-flags"`
 	GitFetchFlags               string   `cli:"git-fetch-flags"`
 	GitSparseCheckoutPaths      []string `cli:"git-sparse-checkout-paths" normalize:"list"`
+	GitSparseCheckoutMode       string   `cli:"git-sparse-checkout-mode"`
 	GitMirrorsPath              string   `cli:"git-mirrors-path" normalize:"filepath"`
 	GitMirrorCheckoutMode       string   `cli:"git-mirror-checkout-mode"`
 	GitMirrorsLockTimeout       int      `cli:"git-mirrors-lock-timeout"`
@@ -550,6 +552,7 @@ var AgentStartCommand = cli.Command{
 		GitCommitVerificationFlag,
 		GitFetchFlagsFlag,
 		GitSparseCheckoutPathsFlag,
+		GitSparseCheckoutModeFlag,
 		GitCloneMirrorFlagsFlag,
 		GitMirrorsPathFlag,
 		GitMirrorCheckoutModeFlag,
@@ -878,6 +881,11 @@ var AgentStartCommand = cli.Command{
 			return fmt.Errorf("invalid git mirror checkout mode %q, must be one of %v", cfg.GitMirrorCheckoutMode, mirrorCheckoutModes)
 		}
 
+		sparseCheckoutMode, err := job.ParseSparseCheckoutMode(cfg.GitSparseCheckoutMode)
+		if err != nil {
+			return err
+		}
+
 		if !slices.Contains(pingModes, cfg.PingMode) {
 			return fmt.Errorf("invalid ping mode %q, must be one of %v", cfg.PingMode, pingModes)
 		}
@@ -1117,6 +1125,7 @@ var AgentStartCommand = cli.Command{
 			GitCommitVerification:           cfg.GitCommitVerification,
 			GitFetchFlags:                   cfg.GitFetchFlags,
 			GitSparseCheckoutPaths:          cfg.GitSparseCheckoutPaths,
+			GitSparseCheckoutMode:           sparseCheckoutMode,
 			GitSubmodules:                   !cfg.NoGitSubmodules,
 			GitSubmoduleCloneConfig:         cfg.GitSubmoduleCloneConfig,
 			SkipCheckout:                    cfg.SkipCheckout,

clicommand/bootstrap.go

@@ -78,6 +78,7 @@ type BootstrapConfig struct {
 	GitCloneFlags                string   `cli:"git-clone-flags"`
 	GitFetchFlags                string   `cli:"git-fetch-flags"`
 	GitSparseCheckoutPaths       []string `cli:"git-sparse-checkout-paths" normalize:"list"`
+	GitSparseCheckoutMode        string   `cli:"git-sparse-checkout-mode"`
 	GitCloneMirrorFlags          string   `cli:"git-clone-mirror-flags"`
 	GitCleanFlags                string   `cli:"git-clean-flags"`
 	GitSSHKey                    string   `cli:"git-ssh-key"`
@@ -258,6 +259,7 @@ var BootstrapCommand = cli.Command{
 		GitCommitVerificationFlag,
 		GitFetchFlagsFlag,
 		GitSparseCheckoutPathsFlag,
+		GitSparseCheckoutModeFlag,
 		cli.StringFlag{
 			Name:   "git-ssh-key",
 			Usage:  "SSH private key to use for git checkout",
@@ -436,6 +438,11 @@ var BootstrapCommand = cli.Command{
 			return fmt.Errorf("invalid git mirror checkout mode %q, must be one of %v", cfg.GitMirrorCheckoutMode, mirrorCheckoutModes)
 		}
 
+		sparseCheckoutMode, err := job.ParseSparseCheckoutMode(cfg.GitSparseCheckoutMode)
+		if err != nil {
+			return err
+		}
+
 		cancelSig, err := process.ParseSignal(cfg.CancelSignal)
 		if err != nil {
 			return fmt.Errorf("failed to parse cancel-signal: %w", err)
@@ -484,6 +491,7 @@ var BootstrapCommand = cli.Command{
 			GitFetchFlags:                cfg.GitFetchFlags,
 			GitLFSEnabled:                cfg.GitLFSEnabled,
 			GitSparseCheckoutPaths:       cfg.GitSparseCheckoutPaths,
+			GitSparseCheckoutMode:        sparseCheckoutMode.String(),
 			GitSSHKey:                    cfg.GitSSHKey,
 			GitMirrorsLockTimeout:        cfg.GitMirrorsLockTimeout,
 			GitMirrorsPath:               cfg.GitMirrorsPath,

clicommand/global.go

@@ -209,7 +209,7 @@ var (
 	CheckoutOverrideModeFlag = cli.StringFlag{
 		Name:   "checkout-override-mode",
 		Value:  "from-job",
-		Usage:  fmt.Sprintf("Controls which sources may override the agent's checkout settings; one of %v. ′strict′ makes the agent authoritative against pipeline/step env, secrets, hooks, plugins, and the Job API. ′from-job′ (default) lets hooks, plugins, and the Job API set checkout vars, blocks secrets, and keeps the agent's checkout flags authoritative over pipeline/step env; pipeline/step env may still set the checkout timeout, submodules, skip-checkout, and skip-fetch-existing-commits toggles that the agent leaves unset, matching earlier agent behaviour, and may set the sparse-checkout paths outright. ′none′ additionally lets pipeline/step env and secrets set them. All mirror configuration and submodule clone config stay agent-authoritative in every mode. The checkout SSH key and Git LFS toggle are not governed by this flag and stay job-settable in every mode. Disabling command-eval forces this to ′strict′.", env.CheckoutOverrideModeNames),
+		Usage:  fmt.Sprintf("Controls which sources may override the agent's checkout settings; one of %v. ′strict′ makes the agent authoritative against pipeline/step env, secrets, hooks, plugins, and the Job API. ′from-job′ (default) lets hooks, plugins, and the Job API set checkout vars, blocks secrets, and keeps the agent's checkout flags authoritative over pipeline/step env; pipeline/step env may still set the checkout timeout, submodules, skip-checkout, and skip-fetch-existing-commits toggles that the agent leaves unset, matching earlier agent behaviour, and may set the sparse-checkout paths and mode outright. ′none′ additionally lets pipeline/step env and secrets set them. All mirror configuration and submodule clone config stay agent-authoritative in every mode. The checkout SSH key and Git LFS toggle are not governed by this flag and stay job-settable in every mode. Disabling command-eval forces this to ′strict′.", env.CheckoutOverrideModeNames),
 		EnvVar: "BUILDKITE_CHECKOUT_OVERRIDE_MODE",
 	}
 
@@ -261,10 +261,17 @@ var (
 	GitSparseCheckoutPathsFlag = cli.StringSliceFlag{
 		Name:   "git-sparse-checkout-paths",
 		Value:  &cli.StringSlice{},
-		Usage:  "Comma-separated list of paths for git sparse checkout (cone mode). When set, only the listed paths are materialized in the working tree.",
+		Usage:  "Comma-separated list of paths for git sparse checkout. When set, only the listed paths are materialized in the working tree. Paths are interpreted according to --git-sparse-checkout-mode.",
 		EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS",
 	}
 
+	GitSparseCheckoutModeFlag = cli.StringFlag{
+		Name:   "git-sparse-checkout-mode",
+		Value:  job.SparseCheckoutModeCone.String(),
+		Usage:  fmt.Sprintf("Changes how the sparse checkout paths are interpreted; available modes are %v. ′cone′ treats each path as a directory to include, which is git's default and enables faster pattern matching, but it accepts only directory names. ′no-cone′ treats each path as a gitignore-style pattern, which allows globs and exclusions such as ′!/docs/′, and requires git >= 2.35. Git LFS objects are not scoped to the sparse paths in ′no-cone′ mode.", job.SparseCheckoutModes),
+		EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT_MODE",
+	}
+
 	GitMirrorsPathFlag = cli.StringFlag{
 		Name:   "git-mirrors-path",
 		Value:  "",

env/protected.go

@@ -110,20 +110,24 @@ var checkoutOverrideScope = map[string]struct{}{
 	"BUILDKITE_GIT_COMMIT_VERIFICATION":         {},
 	"BUILDKITE_GIT_FETCH_FLAGS":                 {},
 	"BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS": {},
+	"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE":        {},
 	"BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS":       {},
 	"BUILDKITE_GIT_SUBMODULES":                  {},
 	"BUILDKITE_SKIP_CHECKOUT":                   {},
 }
 
 // checkoutJobEnvFromJobFloor lists the checkout-scoped vars whose backend job env
 // (pipeline/step checkout config) floor is from-job rather than none: the backend
-// may set them under the default mode, and only strict locks them. Sparse-checkout
-// paths qualify because they are handed to `git sparse-checkout set --cone` as argv,
-// not word-split into a git command line, so a step's checkout.sparse config can't
-// bypass no-command-eval or otherwise escalate the way the flag vars can. The flag
-// vars and commit_verification stay at the none floor (see IsCheckoutLockedForJobEnv).
+// may set them under the default mode, and only strict locks them. The
+// sparse-checkout paths and mode qualify because they are handed to
+// `git sparse-checkout set` as argv after a `--` separator, not word-split into a
+// git command line, and the mode is validated against a fixed set of values, so a
+// step's checkout.sparse config can't bypass no-command-eval or otherwise escalate
+// the way the flag vars can. The flag vars and commit_verification stay at the
+// none floor (see IsCheckoutLockedForJobEnv).
 // Entries must also appear in checkoutOverrideScope.
 var checkoutJobEnvFromJobFloor = map[string]struct{}{
+	"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE":  {},
 	"BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS": {},
 }
 
@@ -249,10 +253,10 @@ func IsCheckoutLocked(name string, mode CheckoutOverrideMode) bool {
 // checkout config. The backend job env is mostly the same (enforced in
 // createEnvironment, agent/job_runner.go), except that under from-job it still
 // lets pipeline/step env set the submodules/skip-checkout/skip-fetch/timeout
-// toggles on their default side to match historical behaviour, and set sparse-
-// checkout paths outright (see IsCheckoutLockedForJobEnv); secrets have no such
-// history, so they stay blocked. Vars that aren't checkout-scoped are governed by
-// IsProtected instead.
+// toggles on their default side to match historical behaviour, and set the
+// sparse-checkout paths and mode outright (see IsCheckoutLockedForJobEnv);
+// secrets have no such history, so they stay blocked. Vars that aren't
+// checkout-scoped are governed by IsProtected instead.
 func IsCheckoutLockedForSecrets(name string, mode CheckoutOverrideMode) bool {
 	if !IsCheckoutOverrideScoped(name) {
 		return false

env/protected_test.go

@@ -84,6 +84,7 @@ func TestCheckoutOverrideScope(t *testing.T) {
 		"BUILDKITE_GIT_FETCH_FLAGS",
 		"BUILDKITE_GIT_SUBMODULES",
 		"BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS",
+		"BUILDKITE_GIT_SPARSE_CHECKOUT_MODE",
 		"BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS",
 		"BUILDKITE_SKIP_CHECKOUT",
 	}
@@ -221,13 +222,15 @@ func TestIsCheckoutLockedForJobEnv(t *testing.T) {
 	const (
 		flagVar   = "BUILDKITE_GIT_CLONE_FLAGS"           // injection vector: none floor
 		sparseVar = "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS" // structured argv: from-job floor
+		modeVar   = "BUILDKITE_GIT_SPARSE_CHECKOUT_MODE"  // validated enum: from-job floor
 		protected = "BUILDKITE_COMMAND_EVAL"              // protected, not scoped
 		unscoped  = "MY_CUSTOM_VAR"                       // in neither map
 	)
 
-	// The flag vars share the secrets rule (locked unless none), but sparse-checkout
-	// paths have a lower floor: the backend job env may set them under from-job, and
-	// only strict locks them. That floor must not leak into the secrets rule.
+	// The flag vars share the secrets rule (locked unless none), but the
+	// sparse-checkout paths and mode have a lower floor: the backend job env may set
+	// them under from-job, and only strict locks them. That floor must not leak into
+	// the secrets rule.
 	cases := []struct {
 		mode       CheckoutOverrideMode
 		wantFlag   bool
@@ -242,14 +245,16 @@ func TestIsCheckoutLockedForJobEnv(t *testing.T) {
 		if got := IsCheckoutLockedForJobEnv(flagVar, tc.mode); got != tc.wantFlag {
 			t.Errorf("IsCheckoutLockedForJobEnv(%q, %v) = %t, want %t", flagVar, tc.mode, got, tc.wantFlag)
 		}
-		if got := IsCheckoutLockedForJobEnv(sparseVar, tc.mode); got != tc.wantSparse {
-			t.Errorf("IsCheckoutLockedForJobEnv(%q, %v) = %t, want %t", sparseVar, tc.mode, got, tc.wantSparse)
-		}
+		for _, sparseScoped := range []string{sparseVar, modeVar} {
+			if got := IsCheckoutLockedForJobEnv(sparseScoped, tc.mode); got != tc.wantSparse {
+				t.Errorf("IsCheckoutLockedForJobEnv(%q, %v) = %t, want %t", sparseScoped, tc.mode, got, tc.wantSparse)
+			}
 
-		// Secrets stay locked unless none even for sparse: the from-job floor is
-		// scoped to the backend job env, not secret-to-env mappings.
-		if wantSecrets := tc.mode != CheckoutOverrideNone; IsCheckoutLockedForSecrets(sparseVar, tc.mode) != wantSecrets {
-			t.Errorf("IsCheckoutLockedForSecrets(%q, %v) = %t, want %t", sparseVar, tc.mode, !wantSecrets, wantSecrets)
+			// Secrets stay locked unless none even for sparse: the from-job floor is
+			// scoped to the backend job env, not secret-to-env mappings.
+			if wantSecrets := tc.mode != CheckoutOverrideNone; IsCheckoutLockedForSecrets(sparseScoped, tc.mode) != wantSecrets {
+				t.Errorf("IsCheckoutLockedForSecrets(%q, %v) = %t, want %t", sparseScoped, tc.mode, !wantSecrets, wantSecrets)
+			}
 		}
 
 		// Vars outside the checkout scope are never governed by this predicate.