Skip to content

Commit 98d138e

Browse files
BEvgeniySBEvgeniyS
authored
feat: cross account caching with role (estahn#336)
This allows to have a single account for caching in multi-account environments. Changes: - Added support to assume role - Added the ability to specify access policy - Added the ability to specify lifecycle policy Initially, I had two approaches to make this work: 1) let other accounts create repos and fill them with images 2) use the role in the target account (this PR) While 1) would be preferable, unfortunately, it's doesn't look like it's possible: you can allow other accounts to create repos, but not put any policies.
1 parent 730fcaa commit 98d138e

File tree

7 files changed

+228
-31
lines changed

7 files changed

+228
-31
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,5 @@
1515
# vendor/
1616

1717
.idea/
18+
coverage.txt
19+
k8s-image-swapper

‎.k8s-image-swapper.yml

+44
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ target:
4141
aws:
4242
accountId: 123456789
4343
region: ap-southeast-2
44+
role: arn:aws:iam::123456789012:role/roleName
4445
ecrOptions:
4546
tags:
4647
- key: CreatedBy
@@ -51,5 +52,48 @@ target:
5152
encryptionConfiguration:
5253
encryptionType: AES256
5354
kmsKey: string
55+
accessPolicy: |
56+
{
57+
"Version": "2008-10-17",
58+
"Statement": [
59+
{
60+
"Sid": "AllowCrossAccountPull",
61+
"Effect": "Allow",
62+
"Principal": {
63+
"AWS": "*"
64+
},
65+
"Action": [
66+
"ecr:GetDownloadUrlForLayer",
67+
"ecr:BatchGetImage",
68+
"ecr:BatchCheckLayerAvailability"
69+
],
70+
"Condition": {
71+
"StringEquals": {
72+
"aws:PrincipalOrgID": [
73+
"o-xxxxxxxx"
74+
]
75+
}
76+
}
77+
}
78+
]
79+
}
80+
81+
lifecyclePolicy: |
82+
{
83+
"rules": [
84+
{
85+
"rulePriority": 1,
86+
"description": "Rule 1",
87+
"selection": {
88+
"tagStatus": "any",
89+
"countType": "imageCountMoreThan",
90+
"countNumber": 1
91+
},
92+
"action": {
93+
"type": "expire"
94+
}
95+
}
96+
]
97+
}
5498
# dockerio:
5599
# quayio:

‎cmd/root.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
6464
//metricsRec := metrics.NewPrometheus(promReg)
6565
log.Trace().Interface("config", cfg).Msg("config")
6666

67-
rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain())
67+
rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain(), cfg.Target.AWS.AccountID, cfg.Target.AWS.Role, cfg.Target.AWS.AccessPolicy, cfg.Target.AWS.LifecyclePolicy)
6868
if err != nil {
6969
log.Err(err).Msg("error connecting to registry client")
7070
os.Exit(1)

‎docs/getting-started.md

+85
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,91 @@ Choose from one of the strategies below or an alternative if needed.
2929
--from-literal=aws_secret_access_key=<...>
3030
```
3131

32+
#### Using ECR registries cross-account
33+
34+
Although ECR allows creating registry policy that allows reposistories creation from different account, there's no way to push anything to these repositories.
35+
ECR resource-level policy can not be applied during creation, and to apply it afterwards we need ecr:SetRepositoryPolicy permission, which foreign account doesn't have.
36+
37+
One way out of this conundrum is to assume the role in target account
38+
39+
```yaml
40+
target:
41+
type: aws
42+
aws:
43+
accountId: 123456789
44+
region: ap-southeast-2
45+
role: arn:aws:iam::123456789012:role/roleName
46+
```
47+
!!! note
48+
Make sure that target role has proper trust permissions that allow to assume it cross-account
49+
50+
!!! note
51+
In order te be able to pull images from outside accounts, you will have to apply proper access policy
52+
53+
54+
#### Access policy
55+
56+
You can specify the access policy that will be applied to the created repos in config. Policy should be raw json string.
57+
For example:
58+
```yaml
59+
target:
60+
aws:
61+
accountId: 123456789
62+
region: ap-southeast-2
63+
role: arn:aws:iam::123456789012:role/roleName
64+
accessPolicy: '{
65+
"Statement": [
66+
{
67+
"Sid": "AllowCrossAccountPull",
68+
"Effect": "Allow",
69+
"Principal": {
70+
"AWS": "*"
71+
},
72+
"Action": [
73+
"ecr:GetDownloadUrlForLayer",
74+
"ecr:BatchGetImage",
75+
"ecr:BatchCheckLayerAvailability"
76+
],
77+
"Condition": {
78+
"StringEquals": {
79+
"aws:PrincipalOrgID": "o-xxxxxxxxxx"
80+
}
81+
}
82+
}
83+
],
84+
"Version": "2008-10-17"
85+
}'
86+
```
87+
88+
#### Lifecycle policy
89+
90+
Similarly to access policy, lifecycle policy can be specified, for example:
91+
92+
```yaml
93+
target:
94+
aws:
95+
accountId: 123456789
96+
region: ap-southeast-2
97+
role: arn:aws:iam::123456789012:role/roleName
98+
accessPolicy: '{
99+
"rules": [
100+
{
101+
"rulePriority": 1,
102+
"description": "Rule 1",
103+
"selection": {
104+
"tagStatus": "any",
105+
"countType": "imageCountMoreThan",
106+
"countNumber": 1000
107+
},
108+
"action": {
109+
"type": "expire"
110+
}
111+
}
112+
]
113+
}
114+
'
115+
```
116+
32117
#### Service Account
33118

34119
1. Create an Webidentity IAM role (e.g. `k8s-image-swapper`) with the following trust policy, e.g

‎pkg/config/config.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,11 @@ type Target struct {
5454
}
5555

5656
type AWS struct {
57-
AccountID string `yaml:"accountId"`
58-
Region string `yaml:"region"`
57+
AccountID string `yaml:"accountId"`
58+
Region string `yaml:"region"`
59+
Role string `yaml:"role"`
60+
AccessPolicy string `yaml:"accessPolicy"`
61+
LifecyclePolicy string `yaml:"lifecyclePolicy"`
5962
}
6063

6164
func (a *AWS) EcrDomain() string {

‎pkg/registry/ecr.go

+85-26
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/aws/aws-sdk-go/aws"
1010
"github.com/aws/aws-sdk-go/aws/awserr"
11+
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
1112
"github.com/aws/aws-sdk-go/aws/session"
1213
"github.com/aws/aws-sdk-go/service/ecr"
1314
"github.com/aws/aws-sdk-go/service/ecr/ecriface"
@@ -19,11 +20,14 @@ import (
1920
var execCommand = exec.Command
2021

2122
type ECRClient struct {
22-
client ecriface.ECRAPI
23-
ecrDomain string
24-
authToken []byte
25-
cache *ristretto.Cache
26-
scheduler *gocron.Scheduler
23+
client ecriface.ECRAPI
24+
ecrDomain string
25+
authToken []byte
26+
cache *ristretto.Cache
27+
scheduler *gocron.Scheduler
28+
targetAccount string
29+
accessPolicy string
30+
lifecyclePolicy string
2731
}
2832

2933
func (e *ECRClient) Credentials() string {
@@ -41,13 +45,15 @@ func (e *ECRClient) CreateRepository(name string) error {
4145
ScanOnPush: aws.Bool(true),
4246
},
4347
ImageTagMutability: aws.String(ecr.ImageTagMutabilityMutable),
48+
RegistryId: &e.targetAccount,
4449
Tags: []*ecr.Tag{
4550
{
4651
Key: aws.String("CreatedBy"),
4752
Value: aws.String("k8s-image-swapper"),
4853
},
4954
},
5055
})
56+
5157
if err != nil {
5258
if aerr, ok := err.(awserr.Error); ok {
5359
switch aerr.Code() {
@@ -63,6 +69,37 @@ func (e *ECRClient) CreateRepository(name string) error {
6369
}
6470
}
6571

72+
if len(e.accessPolicy) > 0 {
73+
log.Info().Msg("Setting access policy on" + name)
74+
log.Debug().Msg("Access policy: \n" + e.accessPolicy)
75+
_, err := e.client.SetRepositoryPolicy(&ecr.SetRepositoryPolicyInput{
76+
PolicyText: &e.accessPolicy,
77+
RegistryId: &e.targetAccount,
78+
RepositoryName: aws.String(name),
79+
})
80+
81+
if err != nil {
82+
log.Err(err).Msg(err.Error())
83+
return err
84+
}
85+
}
86+
87+
if len(e.lifecyclePolicy) > 0 {
88+
log.Info().Msg("Setting lifecycle policy on" + name)
89+
log.Debug().Msg("Lifecycle policy: \n" + e.lifecyclePolicy)
90+
91+
_, err := e.client.PutLifecyclePolicy(&ecr.PutLifecyclePolicyInput{
92+
LifecyclePolicyText: &e.lifecyclePolicy,
93+
RegistryId: &e.targetAccount,
94+
RepositoryName: aws.String(name),
95+
})
96+
97+
if err != nil {
98+
log.Err(err).Msg(err.Error())
99+
return err
100+
}
101+
}
102+
66103
e.cache.Set(name, "", 1)
67104

68105
return nil
@@ -115,7 +152,10 @@ func (e *ECRClient) Endpoint() string {
115152

116153
// requestAuthToken requests and returns an authentication token from ECR with its expiration date
117154
func (e *ECRClient) requestAuthToken() ([]byte, time.Time, error) {
118-
getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
155+
getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{
156+
RegistryIds: []*string{&e.targetAccount},
157+
})
158+
119159
if err != nil {
120160
return []byte(""), time.Time{}, err
121161
}
@@ -146,18 +186,33 @@ func (e *ECRClient) scheduleTokenRenewal() error {
146186
return nil
147187
}
148188

149-
func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
150-
sess := session.Must(session.NewSessionWithOptions(session.Options{
189+
func NewECRClient(region string, ecrDomain string, targetAccount string, role string, accessPolicy string, lifecyclePolicy string) (*ECRClient, error) {
190+
var sess *session.Session
191+
var config *aws.Config
192+
if role != "" {
193+
log.Debug().Msg("Role is specified. Assuming " + role)
194+
stsSession, _ := session.NewSession(config)
195+
creds := stscreds.NewCredentials(stsSession, role)
196+
config = aws.NewConfig().
197+
WithRegion(region).
198+
WithCredentialsChainVerboseErrors(true).
199+
WithHTTPClient(&http.Client{
200+
Timeout: 3 * time.Second,
201+
}).
202+
WithCredentials(creds)
203+
} else {
204+
config = aws.NewConfig().
205+
WithRegion(region).
206+
WithCredentialsChainVerboseErrors(true).
207+
WithHTTPClient(&http.Client{
208+
Timeout: 3 * time.Second,
209+
})
210+
}
211+
212+
sess = session.Must(session.NewSessionWithOptions(session.Options{
151213
SharedConfigState: session.SharedConfigEnable,
214+
Config: (*config),
152215
}))
153-
154-
config := aws.NewConfig().
155-
WithRegion(region).
156-
WithCredentialsChainVerboseErrors(true).
157-
WithHTTPClient(&http.Client{
158-
Timeout: 3 * time.Second,
159-
})
160-
161216
ecrClient := ecr.New(sess, config)
162217

163218
cache, err := ristretto.NewCache(&ristretto.Config{
@@ -173,10 +228,13 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
173228
scheduler.StartAsync()
174229

175230
client := &ECRClient{
176-
client: ecrClient,
177-
ecrDomain: ecrDomain,
178-
cache: cache,
179-
scheduler: scheduler,
231+
client: ecrClient,
232+
ecrDomain: ecrDomain,
233+
cache: cache,
234+
scheduler: scheduler,
235+
targetAccount: targetAccount,
236+
accessPolicy: accessPolicy,
237+
lifecyclePolicy: lifecyclePolicy,
180238
}
181239

182240
if err := client.scheduleTokenRenewal(); err != nil {
@@ -186,13 +244,14 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
186244
return client, nil
187245
}
188246

189-
func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string) (*ECRClient, error) {
247+
func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string, targetAccount, role string) (*ECRClient, error) {
190248
client := &ECRClient{
191-
client: ecrClient,
192-
ecrDomain: ecrDomain,
193-
cache: nil,
194-
scheduler: nil,
195-
authToken: []byte("mock-ecr-client-fake-auth-token"),
249+
client: ecrClient,
250+
ecrDomain: ecrDomain,
251+
cache: nil,
252+
scheduler: nil,
253+
targetAccount: targetAccount,
254+
authToken: []byte("mock-ecr-client-fake-auth-token"),
196255
}
197256

198257
return client, nil

0 commit comments

Comments
 (0)