Skip to content

Commit 520f64c

Browse files
edigaryevedigaryev
authored
Introduce serve command (#114)
* Introduce serve command See #53. * Pick up GitHub-related environment variables and instantiate FS * Work around unparam linter error * parser: reference FS instead of files contents map
1 parent 65368b9 commit 520f64c

File tree

9 files changed

+411
-59
lines changed

9 files changed

+411
-59
lines changed

‎internal/commands/root.go

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ func NewRootCmd() *cobra.Command {
3737
cmd.AddCommand(
3838
newValidateCmd(),
3939
newRunCmd(),
40+
newServeCmd(),
4041
)
4142

4243
return cmd

‎internal/commands/run.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ func run(cmd *cobra.Command, args []string) error {
149149
var result *parser.Result
150150
if experimentalParser {
151151
p := parser.New(parser.WithEnvironment(userSpecifiedEnvironment))
152-
result, err = p.Parse(mergedYAML)
152+
result, err = p.Parse(cmd.Context(), mergedYAML)
153153
if err != nil {
154154
return err
155155
}

‎internal/commands/serve.go

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package commands
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"github.com/cirruslabs/cirrus-cli/internal/evaluator"
7+
"github.com/spf13/cobra"
8+
"net"
9+
)
10+
11+
var ErrServe = errors.New("serve failed")
12+
13+
var address string
14+
15+
func serve(cmd *cobra.Command, args []string) error {
16+
// https://github.com/spf13/cobra/issues/340#issuecomment-374617413
17+
cmd.SilenceUsage = true
18+
19+
lis, err := net.Listen("tcp", address)
20+
if err != nil {
21+
return fmt.Errorf("%w: %v", ErrServe, err)
22+
}
23+
24+
fmt.Printf("listening on %s\n", lis.Addr().String())
25+
26+
if err := evaluator.Serve(cmd.Context(), lis); err != nil {
27+
return fmt.Errorf("%w: %v", ErrServe, err)
28+
}
29+
30+
return nil
31+
}
32+
33+
func newServeCmd() *cobra.Command {
34+
cmd := &cobra.Command{
35+
Use: "serve [flags]",
36+
Short: "Run RPC server that evaluates YAML and Starlark configurations",
37+
RunE: serve,
38+
Hidden: true,
39+
}
40+
41+
cmd.PersistentFlags().StringVarP(&address, "listen", "l", ":8080", "address to listen on")
42+
43+
return cmd
44+
}

‎internal/evaluator/evaluator.go

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
package evaluator
2+
3+
import (
4+
"context"
5+
"github.com/cirruslabs/cirrus-ci-agent/api"
6+
"github.com/cirruslabs/cirrus-cli/pkg/larker"
7+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
8+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/dummy"
9+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/github"
10+
"github.com/cirruslabs/cirrus-cli/pkg/parser"
11+
"google.golang.org/grpc"
12+
"google.golang.org/grpc/codes"
13+
"google.golang.org/grpc/status"
14+
"net"
15+
"strings"
16+
)
17+
18+
func Serve(ctx context.Context, lis net.Listener) error {
19+
server := grpc.NewServer()
20+
21+
api.RegisterCirrusConfigurationEvaluatorServiceService(server, &api.CirrusConfigurationEvaluatorServiceService{
22+
EvaluateConfig: evaluateConfig,
23+
})
24+
25+
errChan := make(chan error)
26+
27+
go func() {
28+
errChan <- server.Serve(lis)
29+
}()
30+
31+
select {
32+
case <-ctx.Done():
33+
server.GracefulStop()
34+
35+
if err := <-errChan; err != nil {
36+
return err
37+
}
38+
39+
return ctx.Err()
40+
case err := <-errChan:
41+
return err
42+
}
43+
}
44+
45+
func fsFromEnvironment(env map[string]string) (fs fs.FileSystem) {
46+
// Fallback to dummy filesystem implementation
47+
fs = dummy.New()
48+
49+
// Use GitHub filesystem if all the required variables are present
50+
owner, ok := env["CIRRUS_REPO_OWNER"]
51+
if !ok {
52+
return
53+
}
54+
repo, ok := env["CIRRUS_REPO_NAME"]
55+
if !ok {
56+
return
57+
}
58+
reference, ok := env["CIRRUS_CHANGE_IN_REPO"]
59+
if !ok {
60+
return
61+
}
62+
token, ok := env["CIRRUS_REPO_CLONE_TOKEN"]
63+
if !ok {
64+
return
65+
}
66+
67+
return github.New(owner, repo, reference, token)
68+
}
69+
70+
func evaluateConfig(ctx context.Context, request *api.EvaluateConfigRequest) (*api.EvaluateConfigResponse, error) {
71+
var yamlConfigs []string
72+
73+
// Register YAML configuration (if any)
74+
if request.YamlConfig != "" {
75+
yamlConfigs = append(yamlConfigs, request.YamlConfig)
76+
}
77+
78+
fs := fsFromEnvironment(request.Environment)
79+
80+
// Run Starlark script and register generated YAML configuration (if any)
81+
if request.StarlarkConfig != "" {
82+
lrk := larker.New(
83+
larker.WithFileSystem(fs),
84+
larker.WithEnvironment(request.Environment),
85+
)
86+
87+
generatedYamlConfig, err := lrk.Main(ctx, request.StarlarkConfig)
88+
if err != nil {
89+
return nil, status.Error(codes.InvalidArgument, err.Error())
90+
}
91+
92+
yamlConfigs = append(yamlConfigs, generatedYamlConfig)
93+
}
94+
95+
// Parse combined YAML
96+
p := parser.New(
97+
parser.WithEnvironment(request.Environment),
98+
parser.WithFileSystem(fs),
99+
)
100+
101+
result, err := p.Parse(ctx, strings.Join(yamlConfigs, "\n"))
102+
if err != nil {
103+
return nil, status.Error(codes.InvalidArgument, err.Error())
104+
}
105+
106+
if len(result.Errors) != 0 {
107+
return nil, status.Error(codes.InvalidArgument, result.Errors[0])
108+
}
109+
110+
return &api.EvaluateConfigResponse{Tasks: result.Tasks}, nil
111+
}

‎internal/evaluator/evaluator_test.go

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package evaluator_test
2+
3+
import (
4+
"context"
5+
"errors"
6+
"github.com/cirruslabs/cirrus-ci-agent/api"
7+
"github.com/cirruslabs/cirrus-cli/internal/evaluator"
8+
"github.com/stretchr/testify/require"
9+
"google.golang.org/grpc"
10+
"net"
11+
"testing"
12+
)
13+
14+
func evaluateHelper(t *testing.T, request *api.EvaluateConfigRequest) (*api.EvaluateConfigResponse, error) {
15+
ctx, cancel := context.WithCancel(context.Background())
16+
17+
lis, err := net.Listen("tcp", "localhost:0")
18+
if err != nil {
19+
t.Fatal(err)
20+
}
21+
22+
errChan := make(chan error)
23+
24+
go func() {
25+
errChan <- evaluator.Serve(ctx, lis)
26+
}()
27+
28+
defer func() {
29+
cancel()
30+
31+
if err := <-errChan; err != nil && !errors.Is(err, context.Canceled) {
32+
t.Fatal(err)
33+
}
34+
}()
35+
36+
conn, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure())
37+
if err != nil {
38+
t.Fatal(err)
39+
}
40+
41+
client := api.NewCirrusConfigurationEvaluatorServiceClient(conn)
42+
43+
response, err := client.EvaluateConfig(context.Background(), request)
44+
if err != nil {
45+
return nil, err
46+
}
47+
48+
return response, nil
49+
}
50+
51+
// TestCrossDependencies ensures that tasks declared in YAML and generated from Starlark can reference each other.
52+
func TestCrossDependencies(t *testing.T) {
53+
yamlConfig := `
54+
container:
55+
image: debian:latest
56+
57+
task:
58+
name: Black
59+
depends_on:
60+
- White
61+
script: true
62+
63+
task:
64+
name: Green
65+
script: true
66+
`
67+
68+
starlarkConfig := `
69+
def main(ctx):
70+
return [
71+
{
72+
"name": "White",
73+
"script": "true",
74+
},
75+
{
76+
"name": "Yellow",
77+
"depends_on": [
78+
"Green",
79+
],
80+
"script": "true",
81+
}
82+
]
83+
`
84+
85+
response, err := evaluateHelper(t, &api.EvaluateConfigRequest{
86+
YamlConfig: yamlConfig,
87+
StarlarkConfig: starlarkConfig,
88+
})
89+
require.NoError(t, err)
90+
require.Len(t, response.Tasks, 4)
91+
}
92+
93+
// TestGitHubFS ensures that evaluator picks up GitHub-related environment variables if present
94+
// and instantiates GitHub filesystem for Starlark execution.
95+
func TestGitHubFS(t *testing.T) {
96+
starlarkConfig := `
97+
load("cirrus", "fs")
98+
99+
def main(ctx):
100+
go_mod = fs.read("go.mod")
101+
102+
if go_mod == None:
103+
fail("go.mod does not exists")
104+
105+
canary = "module github.com/cirruslabs/cirrus-cli"
106+
107+
if canary not in go_mod:
108+
fail("go.mod does not contain '%s'" % canary)
109+
110+
return [
111+
{
112+
"container": {
113+
"image": "debian:latest",
114+
},
115+
"script": "true",
116+
},
117+
]
118+
`
119+
120+
env := map[string]string{
121+
"CIRRUS_REPO_CLONE_TOKEN": "",
122+
"CIRRUS_REPO_OWNER": "cirruslabs",
123+
"CIRRUS_REPO_NAME": "cirrus-cli",
124+
}
125+
126+
// Try specifying a branch
127+
env["CIRRUS_CHANGE_IN_REPO"] = "master"
128+
_, err := evaluateHelper(t, &api.EvaluateConfigRequest{
129+
StarlarkConfig: starlarkConfig,
130+
Environment: env,
131+
})
132+
require.NoError(t, err)
133+
134+
// Try specifying a commit currently pointed to by the master branch
135+
env["CIRRUS_CHANGE_IN_REPO"] = "65368b9c"
136+
_, err = evaluateHelper(t, &api.EvaluateConfigRequest{
137+
StarlarkConfig: starlarkConfig,
138+
Environment: env,
139+
})
140+
require.NoError(t, err)
141+
}

‎pkg/larker/fs/memory/memory.go

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package memory
2+
3+
import (
4+
"context"
5+
"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
6+
"github.com/go-git/go-billy/v5"
7+
"github.com/go-git/go-billy/v5/memfs"
8+
"github.com/go-git/go-billy/v5/util"
9+
"io/ioutil"
10+
)
11+
12+
type Memory struct {
13+
fs billy.Filesystem
14+
}
15+
16+
func New(fileContents map[string][]byte) (*Memory, error) {
17+
memory := &Memory{
18+
fs: memfs.New(),
19+
}
20+
21+
for path, contents := range fileContents {
22+
if err := util.WriteFile(memory.fs, path, contents, 0600); err != nil {
23+
return nil, err
24+
}
25+
}
26+
27+
return memory, nil
28+
}
29+
30+
func (memory *Memory) Stat(ctx context.Context, path string) (*fs.FileInfo, error) {
31+
fileInfo, err := memory.fs.Stat(path)
32+
if err != nil {
33+
return nil, err
34+
}
35+
36+
return &fs.FileInfo{IsDir: fileInfo.IsDir()}, nil
37+
}
38+
39+
func (memory *Memory) Get(ctx context.Context, path string) ([]byte, error) {
40+
file, err := memory.fs.Open(path)
41+
if err != nil {
42+
return nil, err
43+
}
44+
45+
fileBytes, err := ioutil.ReadAll(file)
46+
if err != nil {
47+
return nil, err
48+
}
49+
50+
return fileBytes, nil
51+
}
52+
53+
func (memory *Memory) ReadDir(ctx context.Context, path string) ([]string, error) {
54+
fileInfos, err := memory.fs.ReadDir(path)
55+
if err != nil {
56+
return nil, err
57+
}
58+
59+
var result []string
60+
for _, fileInfo := range fileInfos {
61+
result = append(result, fileInfo.Name())
62+
}
63+
64+
return result, nil
65+
}

0 commit comments

Comments
 (0)