Skip to content

Commit 682a928

Browse files
edigaryevedigaryev
committed
Starlark filesystem builtins
See #53.
1 parent 373586c commit 682a928

File tree

12 files changed

+322
-11
lines changed

12 files changed

+322
-11
lines changed

‎pkg/larker/builtin/fs.go

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package builtin
2+
3+
import (
4+
"context"
5+
"errors"
6+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
7+
"go.starlark.net/starlark"
8+
"os"
9+
)
10+
11+
func FS(ctx context.Context, fs fs.FileSystem) starlark.StringDict {
12+
return starlark.StringDict{
13+
"exists": exists(ctx, fs),
14+
"read": read(ctx, fs),
15+
"readdir": readdir(ctx, fs),
16+
}
17+
}
18+
19+
func exists(ctx context.Context, fs fs.FileSystem) starlark.Value {
20+
const funcName = "exists"
21+
22+
return starlark.NewBuiltin(funcName, func(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
23+
var path string
24+
if err := starlark.UnpackPositionalArgs(funcName, args, kwargs, 1, &path); err != nil {
25+
return nil, err
26+
}
27+
28+
_, err := fs.Stat(ctx, path)
29+
if err != nil {
30+
if errors.Is(err, os.ErrNotExist) {
31+
return starlark.Bool(false), nil
32+
}
33+
34+
return nil, err
35+
}
36+
37+
return starlark.Bool(true), nil
38+
})
39+
}
40+
41+
func read(ctx context.Context, fs fs.FileSystem) starlark.Value {
42+
const funcName = "read"
43+
44+
return starlark.NewBuiltin(funcName, func(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
45+
var path string
46+
if err := starlark.UnpackPositionalArgs(funcName, args, kwargs, 1, &path); err != nil {
47+
return nil, err
48+
}
49+
50+
fileBytes, err := fs.Get(ctx, path)
51+
if err != nil {
52+
return nil, err
53+
}
54+
55+
return starlark.String(fileBytes), nil
56+
})
57+
}
58+
59+
func readdir(ctx context.Context, fs fs.FileSystem) starlark.Value {
60+
const funcName = "readdir"
61+
62+
return starlark.NewBuiltin(funcName, func(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
63+
var path string
64+
if err := starlark.UnpackPositionalArgs(funcName, args, kwargs, 1, &path); err != nil {
65+
return nil, err
66+
}
67+
68+
entries, err := fs.ReadDir(ctx, path)
69+
if err != nil {
70+
return nil, err
71+
}
72+
73+
var starlarkEntries []starlark.Value
74+
for _, entry := range entries {
75+
starlarkEntries = append(starlarkEntries, starlark.String(entry))
76+
}
77+
78+
return starlark.NewList(starlarkEntries), nil
79+
})
80+
}

‎pkg/larker/fs/dummy/dummy.go

+9
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package dummy
22

33
import (
44
"context"
5+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
56
"os"
67
)
78

@@ -11,6 +12,14 @@ func New() *Dummy {
1112
return &Dummy{}
1213
}
1314

15+
func (dfs *Dummy) Stat(ctx context.Context, path string) (fs.FileInfo, error) {
16+
return nil, os.ErrNotExist
17+
}
18+
1419
func (dfs *Dummy) Get(ctx context.Context, path string) ([]byte, error) {
1520
return nil, os.ErrNotExist
1621
}
22+
23+
func (dfs *Dummy) ReadDir(ctx context.Context, path string) ([]string, error) {
24+
return nil, os.ErrNotExist
25+
}

‎pkg/larker/fs/fs.go

+6
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,11 @@ import (
55
)
66

77
type FileSystem interface {
8+
Stat(ctx context.Context, path string) (FileInfo, error)
89
Get(ctx context.Context, path string) ([]byte, error)
10+
ReadDir(ctx context.Context, path string) ([]string, error)
11+
}
12+
13+
type FileInfo interface {
14+
IsDir() bool
915
}

‎pkg/larker/fs/github/github.go

+61-10
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/base64"
66
"errors"
77
"fmt"
8+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
89
"github.com/google/go-github/v32/github"
910
"golang.org/x/oauth2"
1011
"net/http"
@@ -21,6 +22,12 @@ type GitHub struct {
2122
reference string
2223
}
2324

25+
type IsDir bool
26+
27+
func (isDir IsDir) IsDir() bool {
28+
return bool(isDir)
29+
}
30+
2431
func New(owner, repo, reference, token string) *GitHub {
2532
return &GitHub{
2633
token: token,
@@ -30,18 +37,23 @@ func New(owner, repo, reference, token string) *GitHub {
3037
}
3138
}
3239

33-
func (gh *GitHub) Get(ctx context.Context, path string) ([]byte, error) {
34-
fileContent, _, resp, err := gh.client(ctx).Repositories.GetContents(ctx, gh.owner, gh.repo, path,
35-
&github.RepositoryContentGetOptions{
36-
Ref: gh.reference,
37-
},
38-
)
40+
func (gh *GitHub) Stat(ctx context.Context, path string) (fs.FileInfo, error) {
41+
_, directoryContent, err := gh.getContentsWrapper(ctx, path)
3942
if err != nil {
40-
if resp != nil && resp.StatusCode == http.StatusNotFound {
41-
return nil, os.ErrNotExist
42-
}
43+
return nil, err
44+
}
4345

44-
return nil, fmt.Errorf("%w: %v", ErrAPI, err)
46+
if directoryContent != nil {
47+
return IsDir(true), nil
48+
}
49+
50+
return IsDir(false), nil
51+
}
52+
53+
func (gh *GitHub) Get(ctx context.Context, path string) ([]byte, error) {
54+
fileContent, _, err := gh.getContentsWrapper(ctx, path)
55+
if err != nil {
56+
return nil, err
4557
}
4658

4759
// Simulate os.Read() behavior in case the supplied path points to a directory
@@ -57,6 +69,25 @@ func (gh *GitHub) Get(ctx context.Context, path string) ([]byte, error) {
5769
return fileBytes, nil
5870
}
5971

72+
func (gh *GitHub) ReadDir(ctx context.Context, path string) ([]string, error) {
73+
_, directoryContent, err := gh.getContentsWrapper(ctx, path)
74+
if err != nil {
75+
return nil, err
76+
}
77+
78+
// Simulate ioutil.ReadDir() behavior in case the supplied path points to a file
79+
if directoryContent == nil {
80+
return nil, syscall.ENOTDIR
81+
}
82+
83+
var entries []string
84+
for _, fileContent := range directoryContent {
85+
entries = append(entries, *fileContent.Name)
86+
}
87+
88+
return entries, nil
89+
}
90+
6091
func (gh *GitHub) client(ctx context.Context) *github.Client {
6192
var client *http.Client
6293

@@ -69,3 +100,23 @@ func (gh *GitHub) client(ctx context.Context) *github.Client {
69100

70101
return github.NewClient(client)
71102
}
103+
104+
func (gh *GitHub) getContentsWrapper(
105+
ctx context.Context,
106+
path string,
107+
) (*github.RepositoryContent, []*github.RepositoryContent, error) {
108+
fileContent, directoryContent, resp, err := gh.client(ctx).Repositories.GetContents(ctx, gh.owner, gh.repo, path,
109+
&github.RepositoryContentGetOptions{
110+
Ref: gh.reference,
111+
},
112+
)
113+
if err != nil {
114+
if resp != nil && resp.StatusCode == http.StatusNotFound {
115+
return nil, nil, os.ErrNotExist
116+
}
117+
118+
return nil, nil, fmt.Errorf("%w: %v", ErrAPI, err)
119+
}
120+
121+
return fileContent, directoryContent, nil
122+
}

‎pkg/larker/fs/github/github_test.go

+41
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,24 @@ func selfFS() fs.FileSystem {
1616
return github.New("cirruslabs", "cirrus-cli", "master", "")
1717
}
1818

19+
func TestStatFile(t *testing.T) {
20+
stat, err := selfFS().Stat(context.Background(), "go.mod")
21+
if err != nil {
22+
t.Fatal(err)
23+
}
24+
25+
assert.False(t, stat.IsDir())
26+
}
27+
28+
func TestStatDirectory(t *testing.T) {
29+
stat, err := selfFS().Stat(context.Background(), ".")
30+
if err != nil {
31+
t.Fatal(err)
32+
}
33+
34+
assert.True(t, stat.IsDir())
35+
}
36+
1937
func TestGetFile(t *testing.T) {
2038
fileBytes, err := selfFS().Get(context.Background(), "go.mod")
2139
if err != nil {
@@ -38,3 +56,26 @@ func TestGetNonExistentFile(t *testing.T) {
3856
require.Error(t, err)
3957
assert.True(t, errors.Is(err, os.ErrNotExist))
4058
}
59+
60+
func TestReadDirFile(t *testing.T) {
61+
_, err := selfFS().ReadDir(context.Background(), "go.mod")
62+
63+
require.Error(t, err)
64+
assert.True(t, errors.Is(err, syscall.ENOTDIR))
65+
}
66+
67+
func TestReadDirDirectory(t *testing.T) {
68+
entries, err := selfFS().ReadDir(context.Background(), ".")
69+
if err != nil {
70+
t.Fatal(err)
71+
}
72+
73+
assert.Contains(t, entries, "go.mod", "go.sum")
74+
}
75+
76+
func TestReadDirNonExistentDirectory(t *testing.T) {
77+
_, err := selfFS().ReadDir(context.Background(), "the-directory-that-should-not-exist")
78+
79+
require.Error(t, err)
80+
assert.True(t, errors.Is(err, os.ErrNotExist))
81+
}

‎pkg/larker/fs/local/local.go

+25-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package local
22

33
import (
44
"context"
5+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
56
"io/ioutil"
7+
"os"
68
"path/filepath"
79
)
810

@@ -16,11 +18,33 @@ func New(root string) *Local {
1618
}
1719
}
1820

21+
func (lfs *Local) Stat(ctx context.Context, path string) (fs.FileInfo, error) {
22+
return os.Stat(lfs.pivot(path))
23+
}
24+
1925
func (lfs *Local) Get(ctx context.Context, path string) ([]byte, error) {
2026
// To make Starlark scripts cross-platform, load statements are expected to always use slashes,
2127
// but to actually make this work on non-Unix platforms we need to adapt the path
2228
// to the current platform
2329
adaptedPath := filepath.FromSlash(path)
2430

25-
return ioutil.ReadFile(filepath.Join(lfs.root, adaptedPath))
31+
return ioutil.ReadFile(lfs.pivot(adaptedPath))
32+
}
33+
34+
func (lfs *Local) ReadDir(ctx context.Context, path string) ([]string, error) {
35+
fileInfos, err := ioutil.ReadDir(lfs.pivot(path))
36+
if err != nil {
37+
return nil, err
38+
}
39+
40+
var result []string
41+
for _, fileInfo := range fileInfos {
42+
result = append(result, fileInfo.Name())
43+
}
44+
45+
return result, nil
46+
}
47+
48+
func (lfs *Local) pivot(path string) string {
49+
return filepath.Join(lfs.root, path)
2650
}

‎pkg/larker/fs/local/local_test.go

+29
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,38 @@ import (
1414
"testing"
1515
)
1616

17+
func TestStatFile(t *testing.T) {
18+
// Prepare temporary directory
19+
dir := testutil.TempDir(t)
20+
21+
if err := ioutil.WriteFile(filepath.Join(dir, "some-file.txt"), []byte("some-contents"), 0600); err != nil {
22+
t.Fatal(err)
23+
}
24+
25+
stat, err := local.New(dir).Stat(context.Background(), "some-file.txt")
26+
if err != nil {
27+
t.Fatal(err)
28+
}
29+
30+
assert.False(t, stat.IsDir())
31+
}
32+
33+
func TestStatDirectory(t *testing.T) {
34+
// Prepare temporary directory
35+
dir := testutil.TempDir(t)
36+
37+
stat, err := local.New(dir).Stat(context.Background(), ".")
38+
if err != nil {
39+
t.Fatal(err)
40+
}
41+
42+
assert.True(t, stat.IsDir())
43+
}
44+
1745
func TestGetFile(t *testing.T) {
1846
// Prepare temporary directory
1947
dir := testutil.TempDir(t)
48+
2049
if err := ioutil.WriteFile(filepath.Join(dir, "some-file.txt"), []byte("some-contents"), 0600); err != nil {
2150
t.Fatal(err)
2251
}

‎pkg/larker/larker_test.go

+18
Original file line numberDiff line numberDiff line change
@@ -155,3 +155,21 @@ func TestLoadTypoStarVsStart(t *testing.T) {
155155
require.Error(t, err)
156156
assert.Contains(t, err.Error(), "instead of the .start?")
157157
}
158+
159+
// TestBuiltinFS ensures that filesystem-related builtins provided by the cirrus.fs module work correctly.
160+
func TestBuiltinFS(t *testing.T) {
161+
dir := testutil.TempDirPopulatedWith(t, "testdata/builtin-fs")
162+
163+
// Read the source code
164+
source, err := ioutil.ReadFile(filepath.Join(dir, ".cirrus.star"))
165+
if err != nil {
166+
t.Fatal(err)
167+
}
168+
169+
// Run the source code
170+
lrk := larker.New(larker.WithFileSystem(local.New(dir)))
171+
_, err = lrk.Main(context.Background(), string(source))
172+
if err != nil {
173+
t.Fatal(err)
174+
}
175+
}

0 commit comments

Comments
 (0)