Support loading Starlark modules from Git (#85)

* Move loader to a separate package

* Make context available to the loader

* Support loading Starlark modules from Git

See #53.

* Use bounded filesystem wrapper when working with Git

* git.Retrieve(): return more contextualized errors

* Provide a hint when loading of the module that ends with .start fails

Author: edigaryev

go.mod

@@ -13,7 +13,8 @@ require (
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0
+	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0
 	github.com/go-test/deep v1.0.7
 	github.com/gogo/protobuf v1.3.1 // indirect

pkg/larker/larker.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/loader"
 	"go.starlark.net/resolve"
 	"go.starlark.net/starlark"
 	"gopkg.in/yaml.v2"
@@ -18,8 +19,7 @@ var (
 )
 
 type Larker struct {
-	fs     fs.FileSystem
-	loader *Loader
+	fs fs.FileSystem
 }
 
 func New(opts ...Option) *Larker {
@@ -36,17 +36,14 @@ func New(opts ...Option) *Larker {
 		opt(lrk)
 	}
 
-	// Some fields can only be set after we apply the options
-	lrk.loader = NewLoader(lrk.fs)
-
 	return lrk
 }
 
 func (larker *Larker) Main(ctx context.Context, source string) (string, error) {
 	discard := func(thread *starlark.Thread, msg string) {}
 
 	thread := &starlark.Thread{
-		Load:  larker.loader.LoadFunc(),
+		Load:  loader.NewLoader(ctx, larker.fs).LoadFunc(),
 		Print: discard,
 	}
 

pkg/larker/larker_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/cirruslabs/cirrus-cli/pkg/larker"
 	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -102,3 +103,55 @@ func TestCycle(t *testing.T) {
 	assert.Error(t, err)
 	assert.True(t, errors.Is(err, larker.ErrLoadFailed))
 }
+
+// TestLoadGitHelpers ensures that we can use https://github.com/cirrus-templates/helpers
+// as demonstrated in it's README.md.
+//
+// Note that we lock the revision in the .cirrus.star's load statement to prevent failures in the future.
+func TestLoadGitHelpers(t *testing.T) {
+	dir := testutil.TempDirPopulatedWith(t, "testdata/load-git-helpers")
+
+	// Read the source code
+	source, err := ioutil.ReadFile(filepath.Join(dir, ".cirrus.star"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Run the source code
+	lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
+	result, err := lrk.Main(context.Background(), string(source))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Compare the output
+	expected, err := ioutil.ReadFile(filepath.Join(dir, "expected.yml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.YAMLEq(t, string(expected), result)
+}
+
+// TestLoadTypoStarVsStart ensures that we return a user-friendly hint when loading of the module
+// that ends with ".start" fails.
+func TestLoadTypoStarVsStart(t *testing.T) {
+	dir := testutil.TempDir(t)
+
+	lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
+
+	// No hint
+	_, err := lrk.Main(context.Background(), "load(\"some/lib.star\", \"symbol\")\n")
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), "perhaps you've meant")
+
+	// Hint when loading from Git
+	_, err = lrk.Main(context.Background(),
+		"load(\"github.com/cirrus-templates/helpers/dir/lib.start@master\", \"symbol\")\n")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "instead of the .start?")
+
+	// Hint when loading from FS
+	_, err = lrk.Main(context.Background(), "load(\"dir/lib.start\", \"symbol\")\n")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "instead of the .start?")
+}

pkg/larker/loader/git/bounded/bounded.go

@@ -0,0 +1,107 @@
+package bounded
+
+import (
+	"errors"
+	"fmt"
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-billy/v5/memfs"
+	"os"
+)
+
+var ErrExhausted = errors.New("filesystem limit exhausted")
+
+type FileSystem struct {
+	maxFiles int
+	maxBytes int
+
+	filesCreated int
+	bytesWritten int
+
+	billy.Filesystem
+}
+
+type File struct {
+	bfs *FileSystem
+
+	billy.File
+}
+
+func NewFilesystem(maxBytes, maxFiles int) billy.Filesystem {
+	return &FileSystem{
+		maxFiles:   maxFiles,
+		maxBytes:   maxBytes,
+		Filesystem: memfs.New(),
+	}
+}
+
+func (bfs *FileSystem) Create(filename string) (billy.File, error) {
+	return bfs.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+}
+
+func (bfs *FileSystem) Open(filename string) (billy.File, error) {
+	return bfs.OpenFile(filename, os.O_RDONLY, 0)
+}
+
+func (bfs *FileSystem) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) {
+	if err := bfs.moreFiles(1); err != nil {
+		return nil, err
+	}
+
+	file, err := bfs.Filesystem.OpenFile(filename, flag, perm)
+
+	// Accounting
+	if err == nil && (flag&os.O_CREATE) != 0 {
+		bfs.filesCreated++
+	}
+
+	return &File{
+		bfs:  bfs,
+		File: file,
+	}, err
+}
+
+func (bfs *FileSystem) MkdirAll(filename string, perm os.FileMode) error {
+	if err := bfs.moreFiles(1); err != nil {
+		return err
+	}
+
+	err := bfs.Filesystem.MkdirAll(filename, perm)
+
+	// Accounting
+	if err == nil {
+		bfs.filesCreated++
+	}
+
+	return err
+}
+
+func (bfs *FileSystem) moreFiles(n int) error {
+	if (bfs.filesCreated + n) > bfs.maxFiles {
+		return fmt.Errorf("%w: attempted to create more than %d files", ErrExhausted, bfs.maxFiles)
+	}
+
+	return nil
+}
+
+func (bfs *FileSystem) moreBytes(n int) error {
+	if (bfs.bytesWritten + n) > bfs.maxBytes {
+		return fmt.Errorf("%w: attempted to write more than %d bytes", ErrExhausted, bfs.maxBytes)
+	}
+
+	return nil
+}
+
+func (bf *File) Write(p []byte) (int, error) {
+	if err := bf.bfs.moreBytes(len(p)); err != nil {
+		return -1, err
+	}
+
+	n, err := bf.File.Write(p)
+
+	// Accounting
+	if err == nil {
+		bf.bfs.bytesWritten += n
+	}
+
+	return n, err
+}

pkg/larker/loader/git/bounded/bounded_test.go

@@ -0,0 +1,49 @@
+package bounded_test
+
+import (
+	"errors"
+	"fmt"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/loader/git/bounded"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestByteBounds(t *testing.T) {
+	const maxBytes = 65536
+
+	fs := bounded.NewFilesystem(maxBytes, 1)
+
+	file, err := fs.Create("some-file.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < maxBytes; i++ {
+		_, err := file.Write([]byte("A"))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	_, err = file.Write([]byte("A"))
+	require.Error(t, err)
+	assert.True(t, errors.Is(err, bounded.ErrExhausted))
+}
+
+func TestFileBounds(t *testing.T) {
+	const maxFiles = 128
+
+	fs := bounded.NewFilesystem(1, maxFiles)
+
+	for i := 0; i < maxFiles; i++ {
+		_, err := fs.Create(fmt.Sprintf("%d.txt", i))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	_, err := fs.Create("some-file.txt")
+	require.Error(t, err)
+	assert.True(t, errors.Is(err, bounded.ErrExhausted))
+}