Skip to content

Commit ad87b7e

Browse files
edigaryevedigaryev
authored
Basic GitHub filesystem support (#89)
* Require ctx.Context when calling FileSystem methods * Move FS implementations to separate packages * Basic GitHub filesystem support A gateway to proper filesystem builtins needed for #53. * Use token if provided * Handle GitHub API 404 error and return os.ErrNotExist * Implement TestGetNonExistentFile for local FS
1 parent bc37373 commit ad87b7e

File tree

13 files changed

+205
-30
lines changed

13 files changed

+205
-30
lines changed

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ require (
1919
github.com/go-test/deep v1.0.7
2020
github.com/gogo/protobuf v1.3.1 // indirect
2121
github.com/golang/protobuf v1.4.2
22+
github.com/google/go-github/v32 v32.1.0
2223
github.com/google/uuid v1.1.1
2324
github.com/gorilla/mux v1.7.4 // indirect
2425
github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
@@ -40,6 +41,7 @@ require (
4041
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
4142
go.starlark.net v0.0.0-20200821142938-949cc6f4b097
4243
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect
44+
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
4345
golang.org/x/sys v0.0.0-20200819141100-7c7a22168250 // indirect
4446
google.golang.org/genproto v0.0.0-20200815001618-f69a88009b70 // indirect
4547
google.golang.org/grpc v1.32.0

‎go.sum

+5
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,10 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
118118
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
119119
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
120120
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
121+
github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
122+
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
123+
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
124+
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
121125
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
122126
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
123127
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
@@ -299,6 +303,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgN
299303
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
300304
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4=
301305
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
306+
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
302307
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
303308
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
304309
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

‎internal/commands/run.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"github.com/cirruslabs/cirrus-cli/internal/executor/options"
1010
"github.com/cirruslabs/cirrus-cli/internal/executor/taskfilter"
1111
"github.com/cirruslabs/cirrus-cli/pkg/larker"
12-
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
12+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/local"
1313
"github.com/cirruslabs/cirrus-cli/pkg/parser"
1414
"github.com/cirruslabs/cirrus-cli/pkg/rpcparser"
1515
"github.com/cirruslabs/echelon"
@@ -95,7 +95,7 @@ func readStarlarkConfig(ctx context.Context) (string, error) {
9595
return "", err
9696
}
9797

98-
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(".")))
98+
lrk := larker.New(larker.WithFileSystem(local.New(".")))
9999
return lrk.Main(ctx, string(starlarkSource))
100100
}
101101

‎pkg/larker/fs/dummy.go

-15
This file was deleted.

‎pkg/larker/fs/dummy/dummy.go

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package dummy
2+
3+
import (
4+
"context"
5+
"os"
6+
)
7+
8+
type Dummy struct{}
9+
10+
func New() *Dummy {
11+
return &Dummy{}
12+
}
13+
14+
func (dfs *Dummy) Get(ctx context.Context, path string) ([]byte, error) {
15+
return nil, os.ErrNotExist
16+
}

‎pkg/larker/fs/fs.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
package fs
22

3+
import (
4+
"context"
5+
)
6+
37
type FileSystem interface {
4-
Get(path string) ([]byte, error)
8+
Get(ctx context.Context, path string) ([]byte, error)
59
}

‎pkg/larker/fs/github/github.go

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package github
2+
3+
import (
4+
"context"
5+
"encoding/base64"
6+
"errors"
7+
"fmt"
8+
"github.com/google/go-github/v32/github"
9+
"golang.org/x/oauth2"
10+
"net/http"
11+
"os"
12+
"syscall"
13+
)
14+
15+
var ErrAPI = errors.New("failed to communicate with the GitHub API")
16+
17+
type GitHub struct {
18+
token string
19+
owner string
20+
repo string
21+
reference string
22+
}
23+
24+
func New(owner, repo, reference, token string) *GitHub {
25+
return &GitHub{
26+
token: token,
27+
owner: owner,
28+
repo: repo,
29+
reference: reference,
30+
}
31+
}
32+
33+
func (gh *GitHub) Get(ctx context.Context, path string) ([]byte, error) {
34+
fileContent, _, resp, err := gh.client(ctx).Repositories.GetContents(ctx, gh.owner, gh.repo, path,
35+
&github.RepositoryContentGetOptions{
36+
Ref: gh.reference,
37+
},
38+
)
39+
if err != nil {
40+
if resp != nil && resp.StatusCode == http.StatusNotFound {
41+
return nil, os.ErrNotExist
42+
}
43+
44+
return nil, fmt.Errorf("%w: %v", ErrAPI, err)
45+
}
46+
47+
// Simulate os.Read() behavior in case the supplied path points to a directory
48+
if fileContent == nil {
49+
return nil, syscall.EISDIR
50+
}
51+
52+
fileBytes, err := base64.StdEncoding.DecodeString(*fileContent.Content)
53+
if err != nil {
54+
return nil, fmt.Errorf("%w: %v", ErrAPI, err)
55+
}
56+
57+
return fileBytes, nil
58+
}
59+
60+
func (gh *GitHub) client(ctx context.Context) *github.Client {
61+
var client *http.Client
62+
63+
if gh.token != "" {
64+
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{
65+
AccessToken: gh.token,
66+
})
67+
client = oauth2.NewClient(ctx, tokenSource)
68+
}
69+
70+
return github.NewClient(client)
71+
}

‎pkg/larker/fs/github/github_test.go

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package github_test
2+
3+
import (
4+
"context"
5+
"errors"
6+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
7+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/github"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
"os"
11+
"syscall"
12+
"testing"
13+
)
14+
15+
func selfFS() fs.FileSystem {
16+
return github.New("cirruslabs", "cirrus-cli", "master", "")
17+
}
18+
19+
func TestGetFile(t *testing.T) {
20+
fileBytes, err := selfFS().Get(context.Background(), "go.mod")
21+
if err != nil {
22+
t.Fatal(err)
23+
}
24+
25+
assert.Contains(t, string(fileBytes), "module github.com/cirruslabs/cirrus-cli")
26+
}
27+
28+
func TestGetDirectory(t *testing.T) {
29+
_, err := selfFS().Get(context.Background(), ".")
30+
31+
require.Error(t, err)
32+
assert.True(t, errors.Is(err, syscall.EISDIR))
33+
}
34+
35+
func TestGetNonExistentFile(t *testing.T) {
36+
_, err := selfFS().Get(context.Background(), "the-file-that-should-not-exist.txt")
37+
38+
require.Error(t, err)
39+
assert.True(t, errors.Is(err, os.ErrNotExist))
40+
}

‎pkg/larker/fs/local.go ‎pkg/larker/fs/local/local.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
package fs
1+
package local
22

33
import (
4+
"context"
45
"io/ioutil"
56
"path/filepath"
67
)
@@ -9,13 +10,13 @@ type Local struct {
910
root string
1011
}
1112

12-
func NewLocalFileSystem(root string) *Local {
13+
func New(root string) *Local {
1314
return &Local{
1415
root: root,
1516
}
1617
}
1718

18-
func (lfs *Local) Get(path string) ([]byte, error) {
19+
func (lfs *Local) Get(ctx context.Context, path string) ([]byte, error) {
1920
// To make Starlark scripts cross-platform, load statements are expected to always use slashes,
2021
// but to actually make this work on non-Unix platforms we need to adapt the path
2122
// to the current platform

‎pkg/larker/fs/local/local_test.go

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package local_test
2+
3+
import (
4+
"context"
5+
"errors"
6+
"github.com/cirruslabs/cirrus-cli/internal/testutil"
7+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/local"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
"io/ioutil"
11+
"os"
12+
"path/filepath"
13+
"syscall"
14+
"testing"
15+
)
16+
17+
func TestGetFile(t *testing.T) {
18+
// Prepare temporary directory
19+
dir := testutil.TempDir(t)
20+
if err := ioutil.WriteFile(filepath.Join(dir, "some-file.txt"), []byte("some-contents"), 0600); err != nil {
21+
t.Fatal(err)
22+
}
23+
24+
fileBytes, err := local.New(dir).Get(context.Background(), "some-file.txt")
25+
if err != nil {
26+
t.Fatal(err)
27+
}
28+
29+
assert.Contains(t, string(fileBytes), "some-contents")
30+
}
31+
32+
func TestGetDirectory(t *testing.T) {
33+
// Prepare temporary directory
34+
dir := testutil.TempDir(t)
35+
36+
_, err := local.New(dir).Get(context.Background(), ".")
37+
38+
require.Error(t, err)
39+
assert.True(t, errors.Is(err, syscall.EISDIR))
40+
}
41+
42+
func TestGetNonExistentFile(t *testing.T) {
43+
// Prepare temporary directory
44+
dir := testutil.TempDir(t)
45+
46+
_, err := local.New(dir).Get(context.Background(), "the-file-that-should-not-exist.txt")
47+
48+
require.Error(t, err)
49+
assert.True(t, errors.Is(err, os.ErrNotExist))
50+
}

‎pkg/larker/larker.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"errors"
66
"fmt"
77
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
8+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/dummy"
89
"github.com/cirruslabs/cirrus-cli/pkg/larker/loader"
910
"go.starlark.net/resolve"
1011
"go.starlark.net/starlark"
@@ -24,7 +25,7 @@ type Larker struct {
2425

2526
func New(opts ...Option) *Larker {
2627
lrk := &Larker{
27-
fs: fs.NewDummyFileSystem(),
28+
fs: dummy.New(),
2829
}
2930

3031
// weird global init by Starlark

‎pkg/larker/larker_test.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"errors"
66
"github.com/cirruslabs/cirrus-cli/internal/testutil"
77
"github.com/cirruslabs/cirrus-cli/pkg/larker"
8-
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
8+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/local"
99
"github.com/stretchr/testify/assert"
1010
"github.com/stretchr/testify/require"
1111
"io/ioutil"
@@ -34,7 +34,7 @@ func validateExpected(t *testing.T, testDir string) {
3434
}
3535

3636
// Run the source code to produce a YAML configuration
37-
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
37+
lrk := larker.New(larker.WithFileSystem(local.New(dir)))
3838
configuration, err := lrk.Main(context.Background(), string(source))
3939
if err != nil {
4040
t.Fatal(err)
@@ -59,7 +59,7 @@ func TestLoadFileSystemLocal(t *testing.T) {
5959
}
6060

6161
// Run the source code
62-
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
62+
lrk := larker.New(larker.WithFileSystem(local.New(dir)))
6363
_, err = lrk.Main(context.Background(), string(source))
6464
if err != nil {
6565
t.Fatal(err)
@@ -81,7 +81,7 @@ func TestTimeout(t *testing.T) {
8181
defer cancel()
8282

8383
// Run the source code
84-
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
84+
lrk := larker.New(larker.WithFileSystem(local.New(dir)))
8585
_, err = lrk.Main(ctx, string(source))
8686
assert.Error(t, err)
8787
assert.True(t, errors.Is(err, context.DeadlineExceeded))
@@ -98,7 +98,7 @@ func TestCycle(t *testing.T) {
9898
}
9999

100100
// Run the source code
101-
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
101+
lrk := larker.New(larker.WithFileSystem(local.New(dir)))
102102
_, err = lrk.Main(context.Background(), string(source))
103103
assert.Error(t, err)
104104
assert.True(t, errors.Is(err, larker.ErrLoadFailed))
@@ -118,7 +118,7 @@ func TestLoadGitHelpers(t *testing.T) {
118118
}
119119

120120
// Run the source code
121-
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
121+
lrk := larker.New(larker.WithFileSystem(local.New(dir)))
122122
result, err := lrk.Main(context.Background(), string(source))
123123
if err != nil {
124124
t.Fatal(err)
@@ -137,7 +137,7 @@ func TestLoadGitHelpers(t *testing.T) {
137137
func TestLoadTypoStarVsStart(t *testing.T) {
138138
dir := testutil.TempDir(t)
139139

140-
lrk := larker.New(larker.WithFileSystem(fs.NewLocalFileSystem(dir)))
140+
lrk := larker.New(larker.WithFileSystem(local.New(dir)))
141141

142142
// No hint
143143
_, err := lrk.Main(context.Background(), "load(\"some/lib.star\", \"symbol\")\n")

‎pkg/larker/loader/loader.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func (loader *Loader) Retrieve(module string) ([]byte, error) {
4242
return git.Retrieve(loader.ctx, gitLocator)
4343
}
4444

45-
return loader.fs.Get(module)
45+
return loader.fs.Get(loader.ctx, module)
4646
}
4747

4848
func (loader *Loader) LoadFunc() func(thread *starlark.Thread, module string) (starlark.StringDict, error) {

0 commit comments

Comments
 (0)