From f423b91cbf060ea33b1224001eeea4d151d3c7eb Mon Sep 17 00:00:00 2001 From: Christian Gregg Date: Fri, 20 Aug 2021 12:52:06 +0100 Subject: [PATCH 1/3] Add support for .slugignore --- README.md | 3 +- go.mod | 1 + go.sum | 2 + main_test.go | 52 +++++++++++++++++ prepare.go | 15 ++++- slugignore/slugignore.go | 117 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 slugignore/slugignore.go diff --git a/README.md b/README.md index e7cdb56d..6bee71c8 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,7 @@ available attached to tagged [releases]. #### `prepare [APPLICATION] --build-dir [BUILD-DIR] --source-dir [SOURCE-DIR]` In the prepare step, `slugcmplr` will fetch the metadata required to compile -your application. It will copy your project `SOURCE-DIR` into `BUILD-DIR/app` -(it currently do not respect your `.slugcleanup` file, this is TODO). +your application. It will copy your project `SOURCE-DIR` into `BUILD-DIR/app`. It will fetch the buildpacks as defined by your Heroku application download and decompress them into `BUILD-DIR/buildpacks`. If using official buildpacks (e.g. diff --git a/go.mod b/go.mod index ac960e22..d21f44df 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d + github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/go-git/go-git/v5 v5.4.2 github.com/heroku/heroku-go/v5 v5.3.0 github.com/otiai10/copy v1.6.0 diff --git a/go.sum b/go.sum index ae4a7550..c15d6f86 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= +github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/main_test.go b/main_test.go index 8b247da4..6f5c00aa 100644 --- a/main_test.go +++ b/main_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" "testing" @@ -22,6 +23,7 @@ func Test_Suite(t *testing.T) { // nolint: paralleltest t.Run("End to end tests", func(t *testing.T) { t.Run("TestDetectFail", testDetectFail) + t.Run("TestSlugIgnore", testSlugIgnore) t.Run("TestPrepare", testPrepare) t.Run("TestGo", testGo) t.Run("TestRails", testRails) @@ -163,6 +165,56 @@ func testDetectFail(t *testing.T) { }) } +func testSlugIgnore(t *testing.T) { + t.Parallel() + + buildpacks := []*BuildpackDescription{ + {URL: "https://github.com/CGA1123/heroku-buildpack-bar", Name: "CGA1123/heroku-buildpack-bar"}, + {URL: "https://github.com/CGA1123/heroku-buildpack-foo", Name: "CGA1123/heroku-buildpack-foo"}, + } + + configVars := map[string]string{"FOO": "BAR", "BAR": "FOO"} + + withStubPrepare(t, "CGA1123/slugcmplr-fixture-slugignore", buildpacks, configVars, func(t *testing.T, app, buildDir string) { + foundPaths := []string{} + expectedPaths := []string{ + "/README.md", + "/.slugignore", + "/keep-me/hello.txt", + "/vendor/keep-this-dir/file-1.txt", + "/vendor/keep-this-dir/file-2.txt", + } + + filepath.Walk(filepath.Join(buildDir, buildpack.AppDir), func(path string, info os.FileInfo, err error) error { + if err != nil { + t.Fatalf("error while walking directory: %v", err) + } + + if info.Mode().IsRegular() { + foundPaths = append(foundPaths, strings.TrimPrefix(path, filepath.Join(buildDir, buildpack.AppDir))) + } + + return nil + }) + + sort.Sort(sort.StringSlice(foundPaths)) + sort.Sort(sort.StringSlice(expectedPaths)) + + if !SliceEqual(foundPaths, expectedPaths, func(i int) bool { + if foundPaths[i] != expectedPaths[i] { + return false + } + + return true + }) { + expected := strings.Join(expectedPaths, "\n") + actual := strings.Join(foundPaths, "\n") + + t.Fatalf("\nexpected:\n%v\n---\nactual:\n%v\n", expected, actual) + } + }) +} + func testBinary(t *testing.T) { t.Parallel() diff --git a/prepare.go b/prepare.go index 667d8dbd..698e115c 100644 --- a/prepare.go +++ b/prepare.go @@ -7,8 +7,10 @@ import ( "os" "path/filepath" "sort" + "strings" "github.com/cga1123/slugcmplr/buildpack" + "github.com/cga1123/slugcmplr/slugignore" "github.com/otiai10/copy" "github.com/spf13/cobra" ) @@ -100,8 +102,19 @@ func prepare(ctx context.Context, cmd Outputter, p *Prepare) error { log(cmd, "From: %v", p.SourceDir) log(cmd, "To: %v", appDir) + ignore, err := slugignore.ForDirectory(p.SourceDir) + if err != nil { + return fmt.Errorf("failed to read .slugignore") + } + // copy source - if err := copy.Copy(p.SourceDir, appDir); err != nil { + if err := copy.Copy(p.SourceDir, appDir, copy.Options{ + Skip: func(path string) (bool, error) { + return ignore.IsIgnored( + strings.TrimPrefix(path, p.SourceDir), + ), nil + }, + }); err != nil { return fmt.Errorf("failed to copy source: %w", err) } diff --git a/slugignore/slugignore.go b/slugignore/slugignore.go new file mode 100644 index 00000000..740b1ce1 --- /dev/null +++ b/slugignore/slugignore.go @@ -0,0 +1,117 @@ +// package slugignore implements Heroku-like .slugignore functionality for a +// given directory. +// +// Heroku's .slugignore format treats all non-empty and non comment lines +// (comment lines are those begining with a # characted) as Ruby Dir globs, +package slugignore + +import ( + "bufio" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" +) + +// SlugIgnore is the interface for a parsed .slugignore file +// +// A given SlugIgnore is only applicable to the directory which contains the +// .slugignore file, at the time at which it was parsed. +type SlugIgnore interface { + IsIgnored(path string) bool +} + +// ForDirectory parses the .slugignore for a given directory. +// +// If there is no .slugignore file found, it returns a SlugIgnore which always +// returns false when IsIgnored is called. +func ForDirectory(dir string) (SlugIgnore, error) { + f, err := os.Open(filepath.Join(dir, ".slugignore")) + if err != nil { + if err == os.ErrNotExist { + return &nullSlugIgnore{}, nil + } + } + defer f.Close() + + s := bufio.NewScanner(bufio.NewReader(f)) + globs := []string{} + + for s.Scan() { + line := s.Text() + if strings.HasPrefix(line, "#") { + continue + } + + if strings.TrimSpace(line) == "" { + continue + } + + trimmed := strings.TrimPrefix(line, "/") + if strings.HasPrefix(line, "/") { + globs = append(globs, trimmed) + } else { + globs = append( + globs, + trimmed, + filepath.Join("**", trimmed), + ) + } + } + if err := s.Err(); err != nil { + return nil, fmt.Errorf("error parsing .slugignore: %w", err) + } + + ignored := map[string]struct{}{} + for _, glob := range globs { + if !doublestar.ValidatePattern(glob) { + return nil, fmt.Errorf("slugignore pattern is malformed: %v", glob) + } + + matches, err := doublestar.Glob(os.DirFS(dir), glob) + if err != nil { + return nil, fmt.Errorf("error expanding glob %v: %w", glob, err) + } + + for _, match := range matches { + ignored[match] = struct{}{} + } + } + + for k := range ignored { + log.Printf(k) + } + + return cache(ignored), nil +} + +type nullSlugIgnore struct{} + +func (*nullSlugIgnore) IsIgnored(path string) bool { + return false +} + +type cache map[string]struct{} + +func (c cache) IsIgnored(path string) bool { + if strings.HasPrefix(path, "/") { + path = strings.TrimPrefix(path, "/") + } + + for { + if _, ok := c[path]; ok { + return true + } + + path = filepath.Join(path, "..") + + if path == "." { + break + } + } + + return false +} From f8cd6bd9bfc44d3ba570f516ea8080414f2bef5f Mon Sep 17 00:00:00 2001 From: Christian Gregg Date: Fri, 20 Aug 2021 13:05:36 +0100 Subject: [PATCH 2/3] lint --- main_test.go | 15 +++++++-------- slugignore/slugignore.go | 11 ++--------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/main_test.go b/main_test.go index 6f5c00aa..928257a9 100644 --- a/main_test.go +++ b/main_test.go @@ -185,7 +185,7 @@ func testSlugIgnore(t *testing.T) { "/vendor/keep-this-dir/file-2.txt", } - filepath.Walk(filepath.Join(buildDir, buildpack.AppDir), func(path string, info os.FileInfo, err error) error { + err := filepath.Walk(filepath.Join(buildDir, buildpack.AppDir), func(path string, info os.FileInfo, err error) error { if err != nil { t.Fatalf("error while walking directory: %v", err) } @@ -196,16 +196,15 @@ func testSlugIgnore(t *testing.T) { return nil }) + if err != nil { + t.Fatalf("filepath.Walk error: %v", err) + } - sort.Sort(sort.StringSlice(foundPaths)) - sort.Sort(sort.StringSlice(expectedPaths)) + sort.Strings(foundPaths) + sort.Strings(expectedPaths) if !SliceEqual(foundPaths, expectedPaths, func(i int) bool { - if foundPaths[i] != expectedPaths[i] { - return false - } - - return true + return foundPaths[i] == expectedPaths[i] }) { expected := strings.Join(expectedPaths, "\n") actual := strings.Join(foundPaths, "\n") diff --git a/slugignore/slugignore.go b/slugignore/slugignore.go index 740b1ce1..fd22adf2 100644 --- a/slugignore/slugignore.go +++ b/slugignore/slugignore.go @@ -2,13 +2,12 @@ // given directory. // // Heroku's .slugignore format treats all non-empty and non comment lines -// (comment lines are those begining with a # characted) as Ruby Dir globs, +// (comment lines are those beginning with a # characted) as Ruby Dir globs, package slugignore import ( "bufio" "fmt" - "log" "os" "path/filepath" "strings" @@ -81,10 +80,6 @@ func ForDirectory(dir string) (SlugIgnore, error) { } } - for k := range ignored { - log.Printf(k) - } - return cache(ignored), nil } @@ -97,9 +92,7 @@ func (*nullSlugIgnore) IsIgnored(path string) bool { type cache map[string]struct{} func (c cache) IsIgnored(path string) bool { - if strings.HasPrefix(path, "/") { - path = strings.TrimPrefix(path, "/") - } + path = strings.TrimPrefix(path, "/") for { if _, ok := c[path]; ok { From 04e9bb37dc5f3928438161139e2230fc6a909d99 Mon Sep 17 00:00:00 2001 From: Christian Gregg Date: Fri, 20 Aug 2021 13:12:01 +0100 Subject: [PATCH 3/3] Check errors better --- prepare.go | 2 +- slugignore/slugignore.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/prepare.go b/prepare.go index 698e115c..57fba7a7 100644 --- a/prepare.go +++ b/prepare.go @@ -104,7 +104,7 @@ func prepare(ctx context.Context, cmd Outputter, p *Prepare) error { ignore, err := slugignore.ForDirectory(p.SourceDir) if err != nil { - return fmt.Errorf("failed to read .slugignore") + return fmt.Errorf("failed to read .slugignore: %v", err) } // copy source diff --git a/slugignore/slugignore.go b/slugignore/slugignore.go index fd22adf2..f175d632 100644 --- a/slugignore/slugignore.go +++ b/slugignore/slugignore.go @@ -7,6 +7,7 @@ package slugignore import ( "bufio" + "errors" "fmt" "os" "path/filepath" @@ -30,9 +31,11 @@ type SlugIgnore interface { func ForDirectory(dir string) (SlugIgnore, error) { f, err := os.Open(filepath.Join(dir, ".slugignore")) if err != nil { - if err == os.ErrNotExist { + if errors.Is(err, os.ErrNotExist) { return &nullSlugIgnore{}, nil } + + return nil, err } defer f.Close()