Skip to content

Commit 318d9ef

Browse files
committed
feat: add sparse checkout support
1 parent a89d520 commit 318d9ef

11 files changed

Lines changed: 621 additions & 2 deletions

agent/agent_configuration.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ type AgentConfiguration struct {
2424
GitCloneMirrorFlags string
2525
GitCleanFlags string
2626
GitFetchFlags string
27+
SparseCheckoutPaths string
2728
GitSubmodules bool
2829
GitSubmoduleCloneConfig []string
2930
SkipCheckout bool

agent/job_runner.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,7 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
590590
setEnv("BUILDKITE_GIT_CHECKOUT_FLAGS", r.conf.AgentConfiguration.GitCheckoutFlags)
591591
setEnv("BUILDKITE_GIT_CLONE_FLAGS", r.conf.AgentConfiguration.GitCloneFlags)
592592
setEnv("BUILDKITE_GIT_FETCH_FLAGS", r.conf.AgentConfiguration.GitFetchFlags)
593+
setEnv("BUILDKITE_SPARSE_CHECKOUT_PATHS", r.conf.AgentConfiguration.SparseCheckoutPaths)
593594
setEnv("BUILDKITE_GIT_CLONE_MIRROR_FLAGS", r.conf.AgentConfiguration.GitCloneMirrorFlags)
594595
setEnv("BUILDKITE_GIT_MIRROR_CHECKOUT_MODE", r.conf.AgentConfiguration.GitMirrorCheckoutMode)
595596
setEnv("BUILDKITE_GIT_CLEAN_FLAGS", r.conf.AgentConfiguration.GitCleanFlags)

clicommand/agent_start.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ type AgentStartConfig struct {
162162
GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"`
163163
GitCleanFlags string `cli:"git-clean-flags"`
164164
GitFetchFlags string `cli:"git-fetch-flags"`
165+
SparseCheckoutPaths string `cli:"sparse-checkout-paths"`
165166
GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"`
166167
GitMirrorCheckoutMode string `cli:"git-mirror-checkout-mode"`
167168
GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"`
@@ -533,6 +534,7 @@ var AgentStartCommand = cli.Command{
533534
GitCloneFlagsFlag,
534535
GitCleanFlagsFlag,
535536
GitFetchFlagsFlag,
537+
SparseCheckoutPathsFlag,
536538
GitCloneMirrorFlagsFlag,
537539
GitMirrorsPathFlag,
538540
GitMirrorCheckoutModeFlag,
@@ -1066,6 +1068,7 @@ var AgentStartCommand = cli.Command{
10661068
GitCloneMirrorFlags: cfg.GitCloneMirrorFlags,
10671069
GitCleanFlags: cfg.GitCleanFlags,
10681070
GitFetchFlags: cfg.GitFetchFlags,
1071+
SparseCheckoutPaths: cfg.SparseCheckoutPaths,
10691072
GitSubmodules: !cfg.NoGitSubmodules,
10701073
GitSubmoduleCloneConfig: cfg.GitSubmoduleCloneConfig,
10711074
SkipCheckout: cfg.SkipCheckout,

clicommand/bootstrap.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ type BootstrapConfig struct {
7474
GitCheckoutFlags string `cli:"git-checkout-flags"`
7575
GitCloneFlags string `cli:"git-clone-flags"`
7676
GitFetchFlags string `cli:"git-fetch-flags"`
77+
SparseCheckoutPaths string `cli:"sparse-checkout-paths"`
7778
GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"`
7879
GitCleanFlags string `cli:"git-clean-flags"`
7980
GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"`
@@ -241,6 +242,7 @@ var BootstrapCommand = cli.Command{
241242
GitCloneMirrorFlagsFlag,
242243
GitCleanFlagsFlag,
243244
GitFetchFlagsFlag,
245+
SparseCheckoutPathsFlag,
244246
GitMirrorsPathFlag,
245247
GitMirrorCheckoutModeFlag,
246248
GitMirrorsLockTimeoutFlag,
@@ -440,6 +442,7 @@ var BootstrapCommand = cli.Command{
440442
GitCloneFlags: cfg.GitCloneFlags,
441443
GitCloneMirrorFlags: cfg.GitCloneMirrorFlags,
442444
GitFetchFlags: cfg.GitFetchFlags,
445+
SparseCheckoutPaths: cfg.SparseCheckoutPaths,
443446
GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout,
444447
GitMirrorsPath: cfg.GitMirrorsPath,
445448
GitMirrorCheckoutMode: cfg.GitMirrorCheckoutMode,

clicommand/global.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,13 @@ var (
236236
EnvVar: "BUILDKITE_GIT_FETCH_FLAGS",
237237
}
238238

239+
SparseCheckoutPathsFlag = cli.StringFlag{
240+
Name: "sparse-checkout-paths",
241+
Value: "",
242+
Usage: "Comma-separated list of paths for git sparse checkout (cone mode). When set, only the listed paths are materialized in the working tree.",
243+
EnvVar: "BUILDKITE_SPARSE_CHECKOUT_PATHS",
244+
}
245+
239246
GitMirrorsPathFlag = cli.StringFlag{
240247
Name: "git-mirrors-path",
241248
Value: "",

internal/job/checkout.go

Lines changed: 123 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package job
22

33
import (
4+
"bytes"
45
"context"
56
"errors"
67
"fmt"
@@ -343,6 +344,118 @@ func hasGitSubmodules(sh *shell.Shell) bool {
343344
return osutil.FileExists(filepath.Join(sh.Getwd(), ".gitmodules"))
344345
}
345346

347+
func parseSparseCheckoutPaths(paths string) []string {
348+
parts := strings.Split(paths, ",")
349+
parsed := make([]string, 0, len(parts))
350+
for _, part := range parts {
351+
part = strings.TrimSpace(part)
352+
if part != "" {
353+
parsed = append(parsed, part)
354+
}
355+
}
356+
return parsed
357+
}
358+
359+
func parseGitVersion(output string) (major, minor int, ok bool) {
360+
if _, err := fmt.Sscanf(output, "git version %d.%d", &major, &minor); err != nil {
361+
return 0, 0, false
362+
}
363+
return major, minor, true
364+
}
365+
366+
func gitVersionAtLeast(ctx context.Context, sh *shell.Shell, major, minor int) (bool, error) {
367+
output, err := sh.Command("git", "--version").RunAndCaptureStdout(ctx)
368+
if err != nil {
369+
return false, err
370+
}
371+
372+
gitMajor, gitMinor, ok := parseGitVersion(strings.TrimSpace(output))
373+
if !ok {
374+
return false, fmt.Errorf("parsing git version from %q", strings.TrimSpace(output))
375+
}
376+
377+
if gitMajor != major {
378+
return gitMajor > major, nil
379+
}
380+
return gitMinor >= minor, nil
381+
}
382+
383+
// sparseCheckoutMayBeConfigured does a cheap filesystem check for marker files
384+
// that indicate sparse checkout (or the worktree-config extension that
385+
// `sparse-checkout` enables) might already be in effect, so we can avoid
386+
// shelling out to `git config` on every checkout. It resolves the .git dir
387+
// directly to handle the worktree/submodule case where .git is a file
388+
// containing `gitdir: <path>`.
389+
func sparseCheckoutMayBeConfigured(sh *shell.Shell) bool {
390+
gitDir := filepath.Join(sh.Getwd(), ".git")
391+
if data, err := os.ReadFile(gitDir); err == nil && bytes.HasPrefix(data, []byte("gitdir:")) {
392+
gitDirValue := strings.TrimSpace(string(bytes.TrimPrefix(data, []byte("gitdir:"))))
393+
if !filepath.IsAbs(gitDirValue) {
394+
gitDirValue = filepath.Join(sh.Getwd(), gitDirValue)
395+
}
396+
gitDir = gitDirValue
397+
}
398+
399+
return osutil.FileExists(filepath.Join(gitDir, "info", "sparse-checkout")) ||
400+
osutil.FileExists(filepath.Join(gitDir, "config.worktree"))
401+
}
402+
403+
func (e *Executor) disableSparseCheckoutIfConfigured(ctx context.Context) {
404+
if !sparseCheckoutMayBeConfigured(e.shell) {
405+
return
406+
}
407+
408+
sparseOutput, err := e.shell.Command("git", "config", "--get", "core.sparseCheckout").RunAndCaptureStdout(ctx, shell.ShowStderr(false))
409+
if err != nil || strings.TrimSpace(sparseOutput) != "true" {
410+
return
411+
}
412+
413+
e.shell.Commentf("Disabling sparse checkout from previous build")
414+
if err := e.shell.Command("git", "sparse-checkout", "disable").Run(ctx); err != nil {
415+
e.shell.Warningf("Failed to disable sparse checkout: %v", err)
416+
}
417+
418+
// `sparse-checkout disable` leaves extensions.worktreeConfig set, which
419+
// can cause problems for subsequent git operations. Only unset it if no
420+
// other worktree-scoped config remains, to avoid clobbering user config.
421+
worktreeConfig, err := e.shell.Command("git", "config", "--worktree", "--list").RunAndCaptureStdout(ctx, shell.ShowStderr(false))
422+
if err == nil && strings.TrimSpace(worktreeConfig) == "" {
423+
_ = e.shell.Command("git", "config", "--unset", "extensions.worktreeConfig").Run(ctx)
424+
}
425+
}
426+
427+
// setupSparseCheckout configures (or disables) git sparse checkout for the
428+
// current working tree. It returns true if sparse checkout was successfully
429+
// applied for this build, so callers can adjust later behaviour (e.g. skip
430+
// submodule init, which requires the full tree).
431+
func (e *Executor) setupSparseCheckout(ctx context.Context) (bool, error) {
432+
paths := parseSparseCheckoutPaths(e.SparseCheckoutPaths)
433+
if len(paths) == 0 {
434+
e.disableSparseCheckoutIfConfigured(ctx)
435+
return false, nil
436+
}
437+
438+
ok, err := gitVersionAtLeast(ctx, e.shell, 2, 26)
439+
if err != nil {
440+
e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout (%v)", err)
441+
e.disableSparseCheckoutIfConfigured(ctx)
442+
return false, nil
443+
}
444+
if !ok {
445+
e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout")
446+
e.disableSparseCheckoutIfConfigured(ctx)
447+
return false, nil
448+
}
449+
450+
e.shell.Commentf("Setting up sparse checkout for paths: %s", e.SparseCheckoutPaths)
451+
args := append([]string{"sparse-checkout", "set", "--cone"}, paths...)
452+
if err := e.shell.Command("git", args...).Run(ctx); err != nil {
453+
return false, fmt.Errorf("setting sparse checkout paths: %w", err)
454+
}
455+
456+
return true, nil
457+
}
458+
346459
func hasGitCommit(ctx context.Context, sh *shell.Shell, gitDir, commit string) bool {
347460
// Resolve commit to an actual commit object
348461
output, err := sh.Command("git", "--git-dir", gitDir, "rev-parse", commit+"^{commit}").RunAndCaptureStdout(ctx, shell.ShowStderr(false))
@@ -918,6 +1031,11 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
9181031
return err
9191032
}
9201033

1034+
sparseCheckoutActive, err := e.setupSparseCheckout(ctx)
1035+
if err != nil {
1036+
return err
1037+
}
1038+
9211039
gitCheckoutFlags := e.GitCheckoutFlags
9221040

9231041
if e.Commit == "HEAD" {
@@ -932,10 +1050,13 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
9321050

9331051
gitSubmodules := false
9341052
if hasGitSubmodules(e.shell) {
935-
if e.GitSubmodules {
1053+
switch {
1054+
case sparseCheckoutActive:
1055+
e.shell.Commentf("Submodule initialization skipped during sparse checkout")
1056+
case e.GitSubmodules:
9361057
e.shell.Commentf("Git submodules detected")
9371058
gitSubmodules = true
938-
} else {
1059+
default:
9391060
e.shell.OptionalWarningf("submodules-disabled", "This repository has submodules, but submodules are disabled")
9401061
}
9411062
}

0 commit comments

Comments
 (0)