Support Inputs in pipeline config, hashes are per task

.vscode/launch.json

@@ -60,6 +60,15 @@
       "program": "${workspaceRoot}/cli/cmd/turbo",
       "cwd": "${workspaceRoot}",
       "args": ["run", "build", "--force"]
+    },
+    {
+      "name": "Kitchen Sink Dry Run",
+      "type": "go",
+      "request": "launch",
+      "mode": "debug",
+      "program": "${workspaceRoot}/cli/cmd/turbo",
+      "cwd": "${workspaceRoot}/examples/kitchen-sink",
+      "args": ["run", "build", "--dry"]
     }
   ]
 }

cli/internal/context/context.go

@@ -20,11 +20,10 @@ import (
 	"github.com/Masterminds/semver"
 	mapset "github.com/deckarep/golang-set"
 	"github.com/pyr-sh/dag"
-	gitignore "github.com/sabhiram/go-gitignore"
 	"golang.org/x/sync/errgroup"
 )
 
-const GLOBAL_CACHE_KEY = "the hero we needed"
+const GLOBAL_CACHE_KEY = "Ba weep granna weep ninny bong"
 
 // Context of the CLI
 type Context struct {
@@ -35,7 +34,6 @@ type Context struct {
 	RootNode         string
 	GlobalHash       string
 	Lockfile         *fs.YarnLockfile
-	SCC              [][]dag.Vertex
 	Backend          *api.LanguageBackend
 	// Used to arbitrate access to the graph. We parallelise most build operations
 	// and Go maps aren't natively threadsafe so this is needed.
@@ -182,76 +180,22 @@ func WithGraph(rootpath string, config *config.Config) Option {
 		if err := parseJSONWaitGroup.Wait(); err != nil {
 			return err
 		}
-		packageDepsHashGroup := new(errgroup.Group)
 		populateGraphWaitGroup := new(errgroup.Group)
 		for _, pkg := range c.PackageInfos {
 			pkg := pkg
 			populateGraphWaitGroup.Go(func() error {
 				return c.populateTopologicGraphForPackageJson(pkg, rootpath)
 			})
-			packageDepsHashGroup.Go(func() error {
-				return c.loadPackageDepsHash(pkg)
-			})
 		}
 
 		if err := populateGraphWaitGroup.Wait(); err != nil {
 			return err
 		}
-		if err := packageDepsHashGroup.Wait(); err != nil {
-			return err
-		}
 
-		// Only now can we get the SCC (i.e. topological order)
-		c.SCC = dag.StronglyConnected(&c.TopologicalGraph.Graph)
 		return nil
 	}
 }
 
-func (c *Context) loadPackageDepsHash(pkg *fs.PackageJSON) error {
-	pkg.Mu.Lock()
-	defer pkg.Mu.Unlock()
-	hashObject, pkgDepsErr := fs.GetPackageDeps(&fs.PackageDepsOptions{
-		PackagePath: pkg.Dir,
-	})
-	if pkgDepsErr != nil {
-		hashObject = make(map[string]string)
-		// Instead of implementing all gitignore properly, we hack it. We only respect .gitignore in the root and in
-		// the directory of a package.
-		ignore, err := safeCompileIgnoreFile(".gitignore")
-		if err != nil {
-			return err
-		}
-
-		ignorePkg, err := safeCompileIgnoreFile(filepath.Join(pkg.Dir, ".gitignore"))
-		if err != nil {
-			return err
-		}
-
-		fs.Walk(pkg.Dir, func(name string, isDir bool) error {
-			rootMatch := ignore.MatchesPath(name)
-			otherMatch := ignorePkg.MatchesPath(name)
-			if !rootMatch && !otherMatch {
-				if !isDir {
-					hash, err := fs.GitLikeHashFile(name)
-					if err != nil {
-						return fmt.Errorf("could not hash file %v. \n%w", name, err)
-					}
-					hashObject[strings.TrimPrefix(name, pkg.Dir+"/")] = hash
-				}
-			}
-			return nil
-		})
-
-		// ignorefile rules matched files
-	}
-	hashOfFiles, otherErr := fs.HashObject(hashObject)
-	if otherErr != nil {
-		return otherErr
-	}
-	pkg.FilesHash = hashOfFiles
-	return nil
-}
-
 func (c *Context) resolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON) error {
 	seen := mapset.NewSet()
 	var lockfileWg sync.WaitGroup
@@ -432,14 +376,6 @@ func (c *Context) resolveDepGraph(wg *sync.WaitGroup, unresolvedDirectDeps map[s
 	}
 }
 
-func safeCompileIgnoreFile(filepath string) (*gitignore.GitIgnore, error) {
-	if fs.FileExists(filepath) {
-		return gitignore.CompileIgnoreFile(filepath)
-	}
-	// no op
-	return gitignore.CompileIgnoreLines([]string{}...), nil
-}
-
 func getWorkspaceIgnores() []string {
 	return []string{
 		"**/node_modules/",

cli/internal/fs/package_deps_hash.go

@@ -29,17 +29,32 @@ type PackageDepsOptions struct {
 	ExcludedPaths []string
 	// GitPath is an optional alternative path to the git installation
 	GitPath string
+
+	InputPatterns []string
 }
 
 // GetPackageDeps Builds an object containing git hashes for the files under the specified `packagePath` folder.
 func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) {
-	gitLsOutput, err := gitLsTree(p.PackagePath, p.GitPath)
-	if err != nil {
-		return nil, fmt.Errorf("could not get git hashes for files in package %s: %w", p.PackagePath, err)
-	}
 	// Add all the checked in hashes.
 	// TODO(gsoltis): are these platform-dependent paths?
-	result := parseGitLsTree(gitLsOutput)
+	var result map[string]string
+	if len(p.InputPatterns) == 0 {
+		gitLsOutput, err := gitLsTree(p.PackagePath, p.GitPath)
+		if err != nil {
+			return nil, fmt.Errorf("could not get git hashes for files in package %s: %w", p.PackagePath, err)
+		}
+		result = parseGitLsTree(gitLsOutput)
+	} else {
+		gitLsOutput, err := gitLsFiles(p.PackagePath, p.GitPath, p.InputPatterns)
+		if err != nil {
+			return nil, fmt.Errorf("could not get git hashes for file patterns %v in package %s: %w", p.InputPatterns, p.PackagePath, err)
+		}
+		parsedLines, err := parseGitLsFiles(gitLsOutput)
+		if err != nil {
+			return nil, err
+		}
+		result = parsedLines
+	}
 
 	if len(p.ExcludedPaths) > 0 {
 		for _, p := range p.ExcludedPaths {
@@ -149,6 +164,19 @@ func gitLsTree(path string, gitPath string) (string, error) {
 	return strings.TrimSpace(string(out)), nil
 }
 
+func gitLsFiles(path string, gitPath string, patterns []string) (string, error) {
+	cmd := exec.Command("git", "ls-files", "-s", "--")
+	for _, pattern := range patterns {
+		cmd.Args = append(cmd.Args, pattern)
+	}
+	cmd.Dir = path
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to read `git ls-tree`: %w", err)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
 func parseGitLsTree(output string) map[string]string {
 	changes := make(map[string]string)
 	if len(output) > 0 {
@@ -178,6 +206,35 @@ func parseGitLsTree(output string) map[string]string {
 	return changes
 }
 
+func parseGitLsFiles(output string) (map[string]string, error) {
+	changes := make(map[string]string)
+	if len(output) > 0 {
+		// A line is expected to look like:
+		// 100644 3451bccdc831cb43d7a70ed8e628dcf9c7f888c8 0   src/typings/tsd.d.ts
+		// 160000 c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac 0   rushstack
+		gitRex := regexp.MustCompile(`[0-9]{6}\s([a-f0-9]{40})\s[0-3]\s*(.+)`)
+		outputLines := strings.Split(output, "\n")
+
+		for _, line := range outputLines {
+			if len(line) > 0 {
+				match := gitRex.FindStringSubmatch(line)
+				// we found matches, and the slice has three parts:
+				// 0 - the whole string
+				// 1 - the hash
+				// 2 - the filename
+				if match != nil && len(match) == 3 {
+					hash := match[1]
+					filename := parseGitFilename(match[2])
+					changes[filename] = hash
+				} else {
+					return nil, fmt.Errorf("failed to parse git ls-files output line %v", line)
+				}
+			}
+		}
+	}
+	return changes, nil
+}
+
 // Couldn't figure out how to deal with special characters. Skipping for now.
 // @todo see https://github.com/microsoft/rushstack/blob/925ad8c9e22997c1edf5fe38c53fa618e8180f70/libraries/package-deps-hash/src/getPackageDeps.ts#L19
 func parseGitFilename(filename string) string {

cli/internal/fs/package_json.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"sync"
 
+	"github.com/vercel/turborepo/cli/internal/util"
 	"github.com/yosuke-furukawa/json5/encoding/json5"
 )
 
@@ -17,7 +18,7 @@ type TurboConfigJSON struct {
 	GlobalDependencies []string `json:"globalDependencies,omitempty"`
 	// Pipeline is a map of Turbo pipeline entries which define the task graph
 	// and cache behavior on a per task or per package-task basis.
-	Pipeline map[string]Pipeline
+	Pipeline PipelineConfig
 	// Configuration options when interfacing with the remote cache
 	RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"`
 }
@@ -47,12 +48,25 @@ type PPipeline struct {
 	Outputs   *[]string `json:"outputs"`
 	Cache     *bool     `json:"cache,omitempty"`
 	DependsOn []string  `json:"dependsOn,omitempty"`
+	Inputs    []string  `json:"inputs,omitempty"`
+}
+
+type PipelineConfig map[string]Pipeline
+
+func (pc PipelineConfig) GetPipeline(taskID string) (Pipeline, bool) {
+	if entry, ok := pc[taskID]; ok {
+		return entry, true
+	}
+	_, task := util.GetPackageTaskFromId(taskID)
+	entry, ok := pc[task]
+	return entry, ok
 }
 
 type Pipeline struct {
 	Outputs   []string `json:"-"`
 	Cache     *bool    `json:"cache,omitempty"`
 	DependsOn []string `json:"dependsOn,omitempty"`
+	Inputs    []string `json:"inputs,omitempty"`
 	PPipeline
 }
 
@@ -69,6 +83,7 @@ func (c *Pipeline) UnmarshalJSON(data []byte) error {
 	}
 	c.Cache = c.PPipeline.Cache
 	c.DependsOn = c.PPipeline.DependsOn
+	c.Inputs = c.PPipeline.Inputs
 	return nil
 }
 
@@ -86,15 +101,13 @@ type PackageJSON struct {
 	Workspaces             Workspaces        `json:"workspaces,omitempty"`
 	Private                bool              `json:"private,omitempty"`
 	PackageJSONPath        string
-	Hash                   string
-	Dir                    string
+	Dir                    string // relative path from repo root to the package
 	InternalDeps           []string
 	UnresolvedExternalDeps map[string]string
 	ExternalDeps           []string
 	SubLockfile            YarnLockfile
 	LegacyTurboConfig      *TurboConfigJSON `json:"turbo"`
 	Mu                     sync.Mutex
-	FilesHash              string
 	ExternalDepsHash       string
 }
 

cli/internal/fs/package_json_test.go

@@ -20,9 +20,24 @@ func Test_ParseTurboConfigJson(t *testing.T) {
 	}
 	BoolFalse := false
 
-	build := Pipeline{[]string{"dist/**", ".next/**"}, nil, []string{"^build"}, PPipeline{&[]string{"dist/**", ".next/**"}, nil, []string{"^build"}}}
-	lint := Pipeline{[]string{}, nil, nil, PPipeline{&[]string{}, nil, nil}}
-	dev := Pipeline{nil, &BoolFalse, nil, PPipeline{nil, &BoolFalse, nil}}
+	build := Pipeline{
+		Outputs:   []string{"dist/**", ".next/**"},
+		DependsOn: []string{"^build"},
+		PPipeline: PPipeline{
+			Outputs:   &[]string{"dist/**", ".next/**"},
+			DependsOn: []string{"^build"},
+		},
+	}
+	lint := Pipeline{
+		Outputs:   []string{},
+		PPipeline: PPipeline{Outputs: &[]string{}},
+	}
+	dev := Pipeline{
+		Cache: &BoolFalse,
+		PPipeline: PPipeline{
+			Cache: &BoolFalse,
+		},
+	}
 	pipelineExpected := map[string]Pipeline{"build": build, "lint": lint, "dev": dev}
 
 	remoteCacheOptionsExpected := RemoteCacheOptions{"team_id", true}