Skip to content

Commit 04bbcae

Browse files
edigaryevedigaryev
committed
Support loading Starlark modules from Git
See #53.
1 parent 9fef72f commit 04bbcae

File tree

7 files changed

+232
-1
lines changed

7 files changed

+232
-1
lines changed

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ require (
1414
github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible
1515
github.com/docker/go-connections v0.4.0 // indirect
1616
github.com/docker/go-units v0.4.0 // indirect
17+
github.com/go-git/go-billy/v5 v5.0.0
1718
github.com/go-git/go-git/v5 v5.1.0
1819
github.com/go-test/deep v1.0.7
1920
github.com/gogo/protobuf v1.3.1 // indirect

‎pkg/larker/larker_test.go

+28
Original file line numberDiff line numberDiff line change
@@ -102,3 +102,31 @@ func TestCycle(t *testing.T) {
102102
assert.Error(t, err)
103103
assert.True(t, errors.Is(err, larker.ErrLoadFailed))
104104
}
105+
106+
// TestLoadGitHelpers ensures that we can use https://github.com/cirrus-templates/helpers
107+
// as demonstrated in it's README.md.
108+
//
109+
// Note that we lock the revision in the .cirrus.star's load statement to prevent failures in the future.
110+
func TestLoadGitHelpers(t *testing.T) {
111+
dir := testutil.TempDirPopulatedWith(t, "testdata/load-git-helpers")
112+
113+
// Read the source code
114+
source, err := ioutil.ReadFile(filepath.Join(dir, ".cirrus.star"))
115+
if err != nil {
116+
t.Fatal(err)
117+
}
118+
119+
// Run the source code
120+
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
121+
result, err := lrk.Main(context.Background(), string(source))
122+
if err != nil {
123+
t.Fatal(err)
124+
}
125+
126+
// Compare the output
127+
expected, err := ioutil.ReadFile(filepath.Join(dir, "expected.yml"))
128+
if err != nil {
129+
t.Fatal(err)
130+
}
131+
assert.YAMLEq(t, string(expected), result)
132+
}

‎pkg/larker/loader/git/git.go

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package git
2+
3+
import (
4+
"context"
5+
"github.com/go-git/go-billy/v5/memfs"
6+
"github.com/go-git/go-git/v5"
7+
"github.com/go-git/go-git/v5/plumbing"
8+
"github.com/go-git/go-git/v5/storage/memory"
9+
"io/ioutil"
10+
"regexp"
11+
)
12+
13+
type Locator struct {
14+
URL string
15+
Path string
16+
Revision string
17+
}
18+
19+
const (
20+
// Captures the path after / in non-greedy manner.
21+
optionalPath = `(?:/(?P<path>.*?))?`
22+
23+
// Captures the revision after @ in non-greedy manner.
24+
optionalRevision = `(?:@(?P<revision>.*))?`
25+
)
26+
27+
var regexVariants = []*regexp.Regexp{
28+
// GitHub
29+
regexp.MustCompile(`^(?P<root>github\.com/.*?/.*?)` + optionalPath + optionalRevision + `$`),
30+
// Other Git hosting services
31+
regexp.MustCompile(`^(?P<root>.*?)\.git` + optionalPath + optionalRevision + `$`),
32+
}
33+
34+
func Parse(module string) *Locator {
35+
result := &Locator{
36+
Path: "lib.star",
37+
Revision: "master",
38+
}
39+
40+
for _, regex := range regexVariants {
41+
matches := regex.FindStringSubmatch(module)
42+
if matches == nil {
43+
continue
44+
}
45+
46+
result.URL = "https://" + matches[regex.SubexpIndex("root")] + ".git"
47+
48+
path := matches[regex.SubexpIndex("path")]
49+
if path != "" {
50+
result.Path = path
51+
}
52+
53+
revision := matches[regex.SubexpIndex("revision")]
54+
if revision != "" {
55+
result.Revision = revision
56+
}
57+
58+
return result
59+
}
60+
61+
return nil
62+
}
63+
64+
func Retrieve(ctx context.Context, locator *Locator) ([]byte, error) {
65+
storer := memory.NewStorage()
66+
fs := memfs.New()
67+
68+
// Clone the repository
69+
repo, err := git.CloneContext(ctx, storer, fs, &git.CloneOptions{
70+
URL: locator.URL,
71+
Depth: 1,
72+
})
73+
if err != nil {
74+
return nil, err
75+
}
76+
77+
// Checkout the working tree to the specified revision
78+
worktree, err := repo.Worktree()
79+
if err != nil {
80+
return nil, err
81+
}
82+
83+
hash, err := repo.ResolveRevision(plumbing.Revision(locator.Revision))
84+
if err != nil {
85+
return nil, err
86+
}
87+
88+
if err := worktree.Checkout(&git.CheckoutOptions{
89+
Hash: *hash,
90+
}); err != nil {
91+
return nil, err
92+
}
93+
94+
// Read the file from the working tree
95+
file, err := worktree.Filesystem.Open(locator.Path)
96+
if err != nil {
97+
return nil, err
98+
}
99+
100+
fileBytes, err := ioutil.ReadAll(file)
101+
if err != nil {
102+
return nil, err
103+
}
104+
105+
return fileBytes, nil
106+
}

‎pkg/larker/loader/git/git_test.go

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package git_test
2+
3+
import (
4+
"github.com/cirruslabs/cirrus-cli/pkg/larker/loader/git"
5+
"github.com/stretchr/testify/assert"
6+
"testing"
7+
)
8+
9+
func TestParse(t *testing.T) {
10+
testCases := []struct {
11+
Name string
12+
Module string
13+
ExpectedGitLocator *git.Locator
14+
}{
15+
// GitHub
16+
{"defaults to lib.star", "github.com/some-org/some-repo", &git.Locator{
17+
URL: "https://github.com/some-org/some-repo.git",
18+
Path: "lib.star",
19+
Revision: "master",
20+
}},
21+
{"parses path", "github.com/some-org/some-repo/dir/some.star", &git.Locator{
22+
URL: "https://github.com/some-org/some-repo.git",
23+
Path: "dir/some.star",
24+
Revision: "master",
25+
}},
26+
{"parses revision", "github.com/some-org/some-repo@da39a3ee5e6b4b0d3255bfef95601890afd80709", &git.Locator{
27+
URL: "https://github.com/some-org/some-repo.git",
28+
Path: "lib.star",
29+
Revision: "da39a3ee5e6b4b0d3255bfef95601890afd80709",
30+
}},
31+
{"parses path and revision", "github.com/some-org/some-repo/dir/some.star@da39a3ee", &git.Locator{
32+
URL: "https://github.com/some-org/some-repo.git",
33+
Path: "dir/some.star",
34+
Revision: "da39a3ee",
35+
}},
36+
// Other Git hosting services (with the ".git" hint)
37+
{"parses .git hint", "gitlab.com/some-org/some-repo.git/some.star", &git.Locator{
38+
URL: "https://gitlab.com/some-org/some-repo.git",
39+
Path: "some.star",
40+
Revision: "master",
41+
}},
42+
// Other Git hosting services (without the ".git" hint)
43+
{"fails without .git hint", "gitlab.com/some-org/some-repo", nil},
44+
}
45+
46+
for _, testCase := range testCases {
47+
testCase := testCase
48+
t.Run(testCase.Name, func(t *testing.T) {
49+
assert.Equal(t, testCase.ExpectedGitLocator, git.Parse(testCase.Module))
50+
})
51+
}
52+
}

‎pkg/larker/loader/loader.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"errors"
66
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
7+
"github.com/cirruslabs/cirrus-cli/pkg/larker/loader/git"
78
"go.starlark.net/starlark"
89
"path/filepath"
910
)
@@ -29,6 +30,15 @@ func NewLoader(ctx context.Context, fs fs.FileSystem) *Loader {
2930
}
3031
}
3132

33+
func (loader *Loader) Retrieve(module string) ([]byte, error) {
34+
gitLocator := git.Parse(module)
35+
if gitLocator != nil {
36+
return git.Retrieve(loader.ctx, gitLocator)
37+
}
38+
39+
return loader.fs.Get(module)
40+
}
41+
3242
func (loader *Loader) LoadFunc() func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
3343
return func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
3444
// Lookup cache
@@ -46,7 +56,7 @@ func (loader *Loader) LoadFunc() func(thread *starlark.Thread, module string) (s
4656
}
4757

4858
// Retrieve module source code
49-
source, err := loader.fs.Get(module)
59+
source, err := loader.Retrieve(module)
5060
if err != nil {
5161
return nil, err
5262
}

‎pkg/larker/testdata/load-git-helpers/.cirrus.star

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
load("github.com/cirrus-templates/helpers@a5e5d1649c05c40bab6c82f084b69a8d82977d96", "task", "container", "script", "always", "artifacts")
2+
3+
def main(ctx):
4+
return [
5+
task(
6+
name="Lint",
7+
instance=container("golangci/golangci-lint:latest", cpu=1.0, memory=512),
8+
env={
9+
"STARLARK": True
10+
},
11+
instructions=[
12+
script("lint", "echo $STARLARK", "golangci-lint run -v --out-format json > golangci.json"),
13+
always(
14+
artifacts("report", "golangci.json", type="text/json", format="golangci")
15+
)
16+
]
17+
)
18+
]

‎pkg/larker/testdata/load-git-helpers/expected.yml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
task:
2+
name: Lint
3+
env:
4+
STARLARK: true
5+
container:
6+
image: golangci/golangci-lint:latest
7+
cpu: 1
8+
memory: 512
9+
lint_script:
10+
- echo $STARLARK
11+
- golangci-lint run -v --out-format json > golangci.json
12+
always:
13+
report_artifacts:
14+
path: golangci.json
15+
type: text/json
16+
format: golangci

0 commit comments

Comments
 (0)