Skip to content

Commit dd6f7f7

Browse files
gregfurmangregfurman
authored
Refactor AWS tests to use new LocalStack fixture (#195)
1 parent 113b1ae commit dd6f7f7

10 files changed

+255
-206
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ release_notes.md
1111
.op
1212
.opp
1313
go.work*
14+
*.env

‎go.mod

+5-4
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,14 @@ require (
2727
github.com/PaesslerAG/jsonpath v0.1.1
2828
github.com/apache/pulsar-client-go v0.12.1
2929
github.com/aws/aws-lambda-go v1.46.0
30-
github.com/aws/aws-sdk-go-v2 v1.32.4
30+
github.com/aws/aws-sdk-go-v2 v1.32.6
3131
github.com/aws/aws-sdk-go-v2/config v1.27.43
3232
github.com/aws/aws-sdk-go-v2/credentials v1.17.41
3333
github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.6.17
3434
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
3535
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.32.2
3636
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1
37+
github.com/aws/aws-sdk-go-v2/service/elasticsearchservice v1.32.7
3738
github.com/aws/aws-sdk-go-v2/service/firehose v1.24.0
3839
github.com/aws/aws-sdk-go-v2/service/kinesis v1.24.7
3940
github.com/aws/aws-sdk-go-v2/service/lambda v1.50.0
@@ -207,8 +208,8 @@ require (
207208
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
208209
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.17 // indirect
209210
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
210-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect
211-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect
211+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
212+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
212213
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
213214
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
214215
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7 // indirect
@@ -220,7 +221,7 @@ require (
220221
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.5
221222
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
222223
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
223-
github.com/aws/smithy-go v1.22.0 // indirect
224+
github.com/aws/smithy-go v1.22.1 // indirect
224225
github.com/aymerick/douceur v0.2.0 // indirect
225226
github.com/beorn7/perks v1.0.1 // indirect
226227
github.com/bits-and-blooms/bitset v1.4.0 // indirect

‎go.sum

+10-8
Original file line numberDiff line numberDiff line change
@@ -821,8 +821,8 @@ github.com/aws/aws-sdk-go v1.42.37/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRj
821821
github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
822822
github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
823823
github.com/aws/aws-sdk-go-v2 v1.23.0/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA=
824-
github.com/aws/aws-sdk-go-v2 v1.32.4 h1:S13INUiTxgrPueTmrm5DZ+MiAo99zYzHEFh1UNkOxNE=
825-
github.com/aws/aws-sdk-go-v2 v1.32.4/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
824+
github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4=
825+
github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
826826
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
827827
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ=
828828
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
@@ -849,12 +849,12 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15 h1:2MUXyGW6dVaQz6aqycpb
849849
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15/go.mod h1:aHbhbR6WEQgHAiRj41EQ2W47yOYwNtIkWTXmcAtYqj8=
850850
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM=
851851
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3/go.mod h1:7sGSz1JCKHWWBHq98m6sMtWQikmYPpxjqOydDemiVoM=
852-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4=
853-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0=
852+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU=
853+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU=
854854
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM=
855855
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3/go.mod h1:ify42Rb7nKeDDPkFjKn7q1bPscVPu/+gmHH8d2c+anU=
856-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU=
857-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23/go.mod h1:c48kLgzO19wAu3CPkDWC28JbaJ+hfQlsdl7I2+oqIbk=
856+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA=
857+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE=
858858
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10/go.mod h1:8DcYQcz0+ZJaSxANlHIsbbi6S+zMwjwdDqwW3r9AzaE=
859859
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
860860
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
@@ -868,6 +868,8 @@ github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1 h1:plNo3WtooT2fYnhdyuzzsIJ
868868
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1/go.mod h1:N5tqZcYMM0N1PN7UQYJNWuGyO886OfnMhf/3MAbqMcI=
869869
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7 h1:srShyROqxzC7p18Ws8mqM2sqxJO/8L3Kpiqf+NboJLg=
870870
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7/go.mod h1:9efZgg4nJCGRp91MuHhkwd2kvyp7PWLRYYk5WjEQ5ts=
871+
github.com/aws/aws-sdk-go-v2/service/elasticsearchservice v1.32.7 h1:LEo0xkW2yM3qVp8awRVG1lp1o/eR2o0CSR4h9CpB3us=
872+
github.com/aws/aws-sdk-go-v2/service/elasticsearchservice v1.32.7/go.mod h1:cMruo7iPDprwYjl8CruF4+ht8gZnMJ8nmbVtgEEBu8M=
871873
github.com/aws/aws-sdk-go-v2/service/firehose v1.24.0 h1:U3F5oeq3Lp1jv9ebLHNr1OSBjCP7qwIOuj+tNqJOuzw=
872874
github.com/aws/aws-sdk-go-v2/service/firehose v1.24.0/go.mod h1:vHumFD15AwENJSM3SsWzcPpMK24s/7vGN1Xp5rLguz0=
873875
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1/go.mod h1:GeUru+8VzrTXV/83XyMJ80KpH8xO89VPoUileyNQ+tc=
@@ -920,8 +922,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ
920922
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo=
921923
github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
922924
github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
923-
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
924-
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
925+
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
926+
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
925927
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
926928
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
927929
github.com/beanstalkd/go-beanstalk v0.2.0 h1:6UOJugnu47uNB2jJO/lxyDgeD1Yds7owYi1USELqexA=

‎internal/impl/aws/integration_test.go

+1-51
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,18 @@
11
package aws
22

33
import (
4-
"context"
5-
"fmt"
6-
"strconv"
74
"testing"
8-
"time"
9-
10-
"github.com/ory/dockertest/v3"
11-
"github.com/ory/dockertest/v3/docker"
12-
"github.com/stretchr/testify/assert"
13-
"github.com/stretchr/testify/require"
145

156
"github.com/warpstreamlabs/bento/public/service/integration"
167

178
_ "github.com/warpstreamlabs/bento/internal/impl/pure"
189
)
1910

20-
func getLocalStack(t testing.TB) (port string) {
21-
portInt, err := integration.GetFreePort()
22-
require.NoError(t, err)
23-
24-
port = strconv.Itoa(portInt)
25-
26-
pool, err := dockertest.NewPool("")
27-
require.NoError(t, err)
28-
29-
pool.MaxWait = time.Minute
30-
31-
resource, err := pool.RunWithOptions(&dockertest.RunOptions{
32-
Repository: "localstack/localstack",
33-
ExposedPorts: []string{port + "/tcp"},
34-
PortBindings: map[docker.Port][]docker.PortBinding{
35-
docker.Port(port + "/tcp"): {
36-
docker.PortBinding{HostIP: "", HostPort: port},
37-
},
38-
},
39-
Env: []string{
40-
fmt.Sprintf("GATEWAY_LISTEN=0.0.0.0:%v", port),
41-
},
42-
})
43-
require.NoError(t, err)
44-
t.Cleanup(func() {
45-
assert.NoError(t, pool.Purge(resource))
46-
})
47-
48-
_ = resource.Expire(900)
49-
50-
require.NoError(t, pool.Retry(func() (err error) {
51-
defer func() {
52-
if err != nil {
53-
t.Logf("localstack probe error: %v", err)
54-
}
55-
}()
56-
return createBucket(context.Background(), port, "test-bucket")
57-
}))
58-
return
59-
}
60-
6111
func TestIntegration(t *testing.T) {
6212
integration.CheckSkip(t)
6313
t.Parallel()
6414

65-
servicePort := getLocalStack(t)
15+
servicePort := GetLocalStack(t, nil)
6616

6717
t.Run("kinesis", func(t *testing.T) {
6818
kinesisIntegrationSuite(t, servicePort)

‎internal/impl/aws/integration_test_helpers.go

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package aws
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"net/http"
7+
"os"
8+
"strconv"
9+
"testing"
10+
"time"
11+
12+
"github.com/ory/dockertest/v3"
13+
"github.com/ory/dockertest/v3/docker"
14+
"github.com/stretchr/testify/assert"
15+
"github.com/stretchr/testify/require"
16+
"github.com/warpstreamlabs/bento/public/service/integration"
17+
)
18+
19+
// TODO: Add config + options pattern or use an already existing library like https://github.com/elgohr/go-localstack
20+
func GetLocalStack(t testing.TB, envVars []string, readyFns ...func(port string) error) (port string) {
21+
portInt, err := integration.GetFreePort()
22+
require.NoError(t, err)
23+
24+
port = strconv.Itoa(portInt)
25+
26+
pool, err := dockertest.NewPool("")
27+
require.NoError(t, err)
28+
29+
lsImageName := "localstack/localstack"
30+
var env []string
31+
env = append(env, envVars...)
32+
33+
// If an auth token is provided, use the pro-image
34+
if authToken, isPro := os.LookupEnv("LOCALSTACK_AUTH_TOKEN"); isPro && authToken != "" {
35+
env = append(env, "LOCALSTACK_AUTH_TOKEN="+authToken)
36+
lsImageName = lsImageName + "-pro"
37+
}
38+
env = append(env, "LS_LOG=debug")
39+
40+
pool.MaxWait = time.Minute * 300
41+
42+
resource, err := pool.RunWithOptions(&dockertest.RunOptions{
43+
Repository: lsImageName,
44+
ExposedPorts: []string{"4566/tcp"},
45+
PortBindings: map[docker.Port][]docker.PortBinding{
46+
docker.Port(port + "/tcp"): {
47+
docker.PortBinding{HostIP: "", HostPort: port},
48+
},
49+
},
50+
Env: env,
51+
})
52+
port = resource.GetPort("4566/tcp")
53+
require.NoError(t, err)
54+
t.Cleanup(func() {
55+
assert.NoError(t, pool.Purge(resource))
56+
})
57+
58+
_ = resource.Expire(900)
59+
60+
require.NoError(t, pool.Retry(func() error {
61+
var err error
62+
defer func() {
63+
if err != nil {
64+
t.Logf("localstack probe error: %v", err)
65+
}
66+
}()
67+
resp, err := http.Get(fmt.Sprintf("http://localhost:%s/_localstack/health", port))
68+
if err != nil {
69+
return err
70+
}
71+
defer resp.Body.Close()
72+
73+
if resp.StatusCode != http.StatusOK {
74+
return errors.New("cannot connect to LocalStack")
75+
}
76+
77+
return nil
78+
}))
79+
80+
for _, readyFn := range readyFns {
81+
require.NoError(t, pool.Retry(func() error {
82+
return readyFn(port)
83+
}))
84+
}
85+
86+
return
87+
}

‎internal/impl/aws/output_kinesis_integration_test.go

+6-38
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,13 @@ import (
44
"bytes"
55
"context"
66
"fmt"
7-
"strconv"
87
"testing"
9-
"time"
108

119
"github.com/aws/aws-sdk-go-v2/aws"
1210
"github.com/aws/aws-sdk-go-v2/config"
1311
"github.com/aws/aws-sdk-go-v2/credentials"
1412
"github.com/aws/aws-sdk-go-v2/service/kinesis"
1513
"github.com/aws/aws-sdk-go-v2/service/kinesis/types"
16-
"github.com/ory/dockertest/v3"
1714
"github.com/stretchr/testify/assert"
1815
"github.com/stretchr/testify/require"
1916

@@ -22,41 +19,14 @@ import (
2219
)
2320

2421
func TestKinesisIntegration(t *testing.T) {
25-
t.Skip("The docker image we're using here is old and deprecated")
2622
integration.CheckSkip(t)
2723

2824
if testing.Short() {
2925
t.Skip("Skipping integration test in short mode")
3026
}
3127

32-
pool, err := dockertest.NewPool("")
33-
if err != nil {
34-
t.Skipf("Could not connect to docker: %s", err)
35-
}
36-
pool.MaxWait = time.Second * 30
37-
38-
// start mysql container with binlog enabled
39-
resource, err := pool.RunWithOptions(&dockertest.RunOptions{
40-
Repository: "vsouza/kinesis-local",
41-
Cmd: []string{
42-
"--createStreamMs=5",
43-
},
44-
})
45-
if err != nil {
46-
t.Fatalf("Could not start resource: %v", err)
47-
}
48-
defer func() {
49-
if err := pool.Purge(resource); err != nil {
50-
t.Logf("Failed to clean up docker resource: %v", err)
51-
}
52-
}()
53-
54-
port, err := strconv.ParseInt(resource.GetPort("4567/tcp"), 10, 64)
55-
if err != nil {
56-
t.Fatal(err)
57-
}
58-
59-
endpoint := fmt.Sprintf("http://localhost:%d", port)
28+
port := GetLocalStack(t, nil)
29+
endpoint := fmt.Sprintf("http://localhost:%s", port)
6030

6131
pConf, err := koOutputSpec().ParseYAML(fmt.Sprintf(`
6232
stream: foo
@@ -79,12 +49,10 @@ credentials:
7949

8050
// bootstrap kinesis
8151
client := kinesis.NewFromConfig(conf)
82-
if err := pool.Retry(func() error {
83-
_, err := client.CreateStream(context.TODO(), &kinesis.CreateStreamInput{
84-
ShardCount: aws.Int32(1),
85-
StreamName: aws.String("foo"),
86-
})
87-
return err
52+
53+
if _, err := client.CreateStream(context.TODO(), &kinesis.CreateStreamInput{
54+
ShardCount: aws.Int32(1),
55+
StreamName: aws.String("foo"),
8856
}); err != nil {
8957
t.Fatalf("Could not connect to docker resource: %s", err)
9058
}

‎internal/impl/aws/processor_s3_integration_test.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,14 @@ import (
2424

2525
func TestIntegrationS3Processor(t *testing.T) {
2626
integration.CheckSkip(t)
27-
servicePort := getLocalStack(t)
27+
servicePort := GetLocalStack(t, nil)
2828
bucketName := "test-bucket"
2929
objectKey := "example.txt"
3030
objectData := "hello world"
3131

32+
err := createBucket(context.TODO(), servicePort, bucketName)
33+
require.NoError(t, err)
34+
3235
client, err := uploadFile(servicePort, bucketName, objectKey, objectData)
3336
require.NoError(t, err)
3437

0 commit comments

Comments
 (0)