Skip to content
This repository was archived by the owner on Oct 2, 2024. It is now read-only.

Commit 4e54cd5

Browse files
gbevangbevan
authored
Merge pull request #92 from gbevan/D20190423_simpler_api_consumption
document simpler api consumption where TLS is trusted.
2 parents 6828b33 + 0eb5aad commit 4e54cd5

6 files changed

+149
-26
lines changed

‎README.md

+8-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# gostint - A Shallow RESTful api for Ansible, Terraform ...
22
... and basically anything you would like to run as jobs in docker containers.
3-
Authenticated and end-to-end encrypted with Hashicorp Vault with Secret Injection
3+
Authenticated and (optionally) end-to-end encrypted with Hashicorp Vault with
4+
Secret Injection
45
* https://goethite.github.io/gostint/
56

67
> gostint:
@@ -28,8 +29,12 @@ JSON jobs used in these tests are in the respective [tests](tests/) files.
2829
* [Brainstorming job sequence diagrams](docs/jobsequence.md)
2930

3031
## Features
31-
* Integrated with Hashicorp Vault's AppRole Authentication, Transit end-to-end
32-
encryption, Cubbyhole, Token Wrapping and KV Secrets.
32+
* Integrated with Hashicorp Vault's AppRole Authentication.
33+
* Optionally consume Hashicorp Vault's Transit end-to-end
34+
encryption, Cubbyhole and Token Wrapping if routing requests through
35+
untrusted networks (e.g. where TLS end-to-end encryption is not available).
36+
* If TLS encryption is unbroken a much simpler way to consume the API is
37+
available.
3338
* Secrets in Vault can be referenced in a job request, which are then injected
3439
into the job's running container.
3540
* Additional content can be flexibly injected into the job container from the

‎docs/jobsequence.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
# Job Sequence Diagrams Brainstorming the Protocol Using Hashicorp Vault
1+
# Job Sequence Diagrams Brainstorming ...
2+
... the Protocol for Traversing Untrusted Networks with Broken TLS Encryption Using Hashicorp Vault. Note that a simpler way of submitting jobs to gostint can be used if
3+
you have unbroken TLS (either direct or via SNI Routing).
24

35
![First thoughts on job submission protocol](job_diag1.mermaid.png)
46
* Requestor and poster here are the same enitity.
@@ -86,7 +88,7 @@ poster / routing.
8688
![job via intermediary](job_via_intermediary.mermaid.png)
8789
---
8890

89-
Copyright 2018 Graham Lee Bevan <graham.bevan@ntlworld.com>
91+
Copyright 2018-2019 Graham Lee Bevan <graham.bevan@ntlworld.com>
9092

9193
<a rel="license" href="http://creativecommons.org/licenses/by/4.0/"><img alt="Creative Commons Licence" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.
9294

‎jobqueues/jobqueues.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1046,8 +1046,8 @@ func (job *Job) runContainer(ctx *context.Context, cli *client.Client, container
10461046
if status != 0 {
10471047
finalStatus = "failed"
10481048
}
1049-
logmsg.Warn("output:%v", buf.String())
1050-
logmsg.Warn("stderr:%v", buferr.String())
1049+
// logmsg.Warn("output:%v", buf.String())
1050+
// logmsg.Warn("stderr:%v", buferr.String())
10511051

10521052
job.UpdateJob(bson.M{
10531053
"status": finalStatus,

‎tests/bats/0200_job10_ansible_play_hashivault_vars.sh ‎tests/bats/0200_job11_ansible_play_hashivault_vars.sh

File renamed without changes.

‎tests/bats/0101_job1_busybox_nocby.sh ‎tests/bats/1100_simple_job1_busybox.sh

+37-19
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,49 @@
11
#!/usr/bin/env bats
22

3-
@test "Submitting job1 busybox without cubbyhole should return json" {
3+
@test "Simple api - Submitting job1 busybox should return json" {
44
# Get a default token for the api post authentication
5-
TOKEN=$(vault write -f auth/token/create policies=default -format=json | jq .auth.client_token -r)
5+
TOKEN=$(
6+
vault write -f \
7+
auth/token/create \
8+
policies=default \
9+
-format=json \
10+
| jq .auth.client_token -r
11+
)
12+
# TOKEN=$(vault write -f auth/token/create policies=gostint-client -format=json | jq .auth.client_token -r)
613
echo "TOKEN: $TOKEN" >&2
714
echo "$TOKEN" > $BATS_TMPDIR/token
815

916
# Get secretId for the approle
10-
WRAPSECRETID=$(vault write -wrap-ttl=144h -f auth/approle/role/$GOSTINT_ROLENAME/secret-id -format=json | jq .wrap_info.token -r)
17+
WRAPSECRETID=$(
18+
vault write -wrap-ttl=144h \
19+
-f auth/approle/role/$GOSTINT_ROLENAME/secret-id \
20+
-format=json \
21+
| jq .wrap_info.token -r
22+
)
1123
echo "WRAPSECRETID: $WRAPSECRETID" >&2
24+
# echo "$WRAPSECRETID" > $BATS_TMPDIR/wrapsecretid
1225

13-
QNAME=$(cat ../job1.json | jq .qname -r)
14-
15-
# Create new job request with the wrapped secret id
26+
# Create new job request with payload
1627
jq --arg wrap_secret_id "$WRAPSECRETID" \
17-
'. | .wrap_secret_id=$wrap_secret_id' \
18-
< ../job1.json >$BATS_TMPDIR/job_ncby.json
19-
cat $BATS_TMPDIR/job_ncby.json >&2
20-
21-
J="$(curl -k -s https://127.0.0.1:3232/v1/api/job --header "X-Auth-Token: $TOKEN" -X POST -d @$BATS_TMPDIR/job_ncby.json | tee $BATS_TMPDIR/job1_ncby.json)"
28+
'. | .wrap_secret_id=$wrap_secret_id' \
29+
< ../job1.json >$BATS_TMPDIR/job.json
30+
cat $BATS_TMPDIR/job.json >&2
31+
32+
J="$(
33+
curl -k -s https://127.0.0.1:3232/v1/api/job \
34+
--header "X-Auth-Token: $TOKEN" \
35+
-X POST \
36+
-d @$BATS_TMPDIR/job.json \
37+
| tee $BATS_TMPDIR/job1.json
38+
)"
2239
echo "J: $J" >&2
2340
[ "$J" != "" ]
41+
# /bin/false
2442
}
2543

2644
@test "job1 should be queued in the play job1 queue" {
2745

28-
J="$(cat $BATS_TMPDIR/job1_ncby.json)"
46+
J="$(cat $BATS_TMPDIR/job1.json)"
2947
echo "J: $J" >&2
3048

3149
id=$(echo $J | jq ._id -r)
@@ -38,7 +56,7 @@
3856
@test "Be able to retrieve the current status" {
3957
TOKEN="$(cat $BATS_TMPDIR/token)"
4058
echo "TOKEN: $TOKEN" >&2
41-
J="$(cat $BATS_TMPDIR/job1_ncby.json)"
59+
J="$(cat $BATS_TMPDIR/job1.json)"
4260

4361
ID=$(echo $J | jq ._id -r)
4462

@@ -52,7 +70,7 @@
5270
@test "Status should eventually be success" {
5371
TOKEN="$(cat $BATS_TMPDIR/token)"
5472
echo "TOKEN: $TOKEN" >&2
55-
J="$(cat $BATS_TMPDIR/job1_ncby.json)"
73+
J="$(cat $BATS_TMPDIR/job1.json)"
5674

5775
ID=$(echo $J | jq ._id -r)
5876
echo "ID:$ID" >&2
@@ -70,12 +88,12 @@
7088
fi
7189
done
7290
echo "status after:$status" >&2
73-
echo "$R" > $BATS_TMPDIR/job1_ncby.final.json
91+
echo "$R" > $BATS_TMPDIR/job1.final.json
7492
[ "$status" == "success" ]
7593
}
7694

7795
@test "Should have final output in json" {
78-
R="$(cat $BATS_TMPDIR/job1_ncby.final.json)"
96+
R="$(cat $BATS_TMPDIR/job1.final.json)"
7997

8098
echo "R:$R" >&2
8199
output="$(echo $R | jq .output -r)"
@@ -86,7 +104,7 @@
86104
@test "Should delete the job id" {
87105
TOKEN="$(cat $BATS_TMPDIR/token)"
88106
echo "TOKEN: $TOKEN" >&2
89-
J="$(cat $BATS_TMPDIR/job1_ncby.json)"
107+
J="$(cat $BATS_TMPDIR/job1.json)"
90108

91109
ID=$(echo $J | jq ._id -r)
92110
echo "ID:$ID" >&2
@@ -102,7 +120,7 @@
102120
@test "Lookup for deleted id should return Not Found error" {
103121
TOKEN="$(cat $BATS_TMPDIR/token)"
104122
echo "TOKEN: $TOKEN" >&2
105-
J="$(cat $BATS_TMPDIR/job1_ncby.json)"
123+
J="$(cat $BATS_TMPDIR/job1.json)"
106124

107125
ID=$(echo $J | jq ._id -r)
108126
echo "ID:$ID" >&2
@@ -116,7 +134,7 @@
116134
@test "Lookup for garbage id should return Invalid job ID error" {
117135
TOKEN="$(cat $BATS_TMPDIR/token)"
118136
echo "TOKEN: $TOKEN" >&2
119-
J="$(cat $BATS_TMPDIR/job1_ncby.json)"
137+
J="$(cat $BATS_TMPDIR/job1.json)"
120138

121139
R="$(curl -k -s https://127.0.0.1:3232/v1/api/job/DOESNOTEXIST --header "X-Auth-Token: $TOKEN")"
122140
echo "R:$R" >&2

‎tests/bats/1110_simple_job2_ansible_ping.sh

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/usr/bin/env bats
2+
3+
@test "Simple API - Submitting job2 ansible ping should return json" {
4+
# Get a default token for the api post authentication
5+
TOKEN=$(
6+
vault write -f \
7+
auth/token/create \
8+
policies=default \
9+
-format=json \
10+
| jq .auth.client_token -r
11+
)
12+
echo "$TOKEN"
13+
echo "$TOKEN" > $BATS_TMPDIR/token
14+
15+
# Get secretId for the approle
16+
WRAPSECRETID=$(
17+
vault write -wrap-ttl=144h \
18+
-f auth/approle/role/$GOSTINT_ROLENAME/secret-id \
19+
-format=json \
20+
| jq .wrap_info.token -r
21+
)
22+
echo "WRAPSECRETID: $WRAPSECRETID" >&2
23+
# echo "$WRAPSECRETID" > $BATS_TMPDIR/wrapsecretid
24+
25+
# Create new job request with payload
26+
jq --arg wrap_secret_id "$WRAPSECRETID" \
27+
'. | .wrap_secret_id=$wrap_secret_id' \
28+
< ../job2_ansible.json >$BATS_TMPDIR/job.json
29+
cat $BATS_TMPDIR/job.json >&2
30+
31+
J="$(
32+
curl -k -s https://127.0.0.1:3232/v1/api/job \
33+
--header "X-Auth-Token: $TOKEN" \
34+
-X POST \
35+
-d @$BATS_TMPDIR/job.json \
36+
| tee $BATS_TMPDIR/job2.json
37+
)"
38+
[ "$J" != "" ]
39+
}
40+
41+
@test "job2 should be queued in the play job2 queue" {
42+
43+
J="$(cat $BATS_TMPDIR/job2.json)"
44+
45+
id=$(echo $J | jq ._id -r)
46+
status=$(echo $J | jq .status -r)
47+
qname=$(echo $J | jq .qname -r)
48+
49+
[ "$id" != "" ] && [ "$status" == "queued" ] && [ "$qname" == "play job2" ]
50+
}
51+
52+
@test "Be able to retrieve the current status" {
53+
TOKEN="$(cat $BATS_TMPDIR/token)"
54+
echo "TOKEN: $TOKEN" >&2
55+
J="$(cat $BATS_TMPDIR/job2.json)"
56+
57+
ID=$(echo $J | jq ._id -r)
58+
59+
R="$(curl -k -s https://127.0.0.1:3232/v1/api/job/$ID --header "X-Auth-Token: $TOKEN")"
60+
echo "R:$R" >&2
61+
status=$(echo $R | jq .status -r)
62+
63+
[ "$status" == "queued" -o "$status" == "running" ]
64+
}
65+
66+
@test "Status should eventually be success" {
67+
TOKEN="$(cat $BATS_TMPDIR/token)"
68+
echo "TOKEN: $TOKEN" >&2
69+
J="$(cat $BATS_TMPDIR/job2.json)"
70+
71+
ID=$(echo $J | jq ._id -r)
72+
echo "ID:$ID" >&2
73+
74+
status="queued"
75+
for i in {1..40}
76+
do
77+
sleep 5
78+
R="$(curl -k -s https://127.0.0.1:3232/v1/api/job/$ID --header "X-Auth-Token: $TOKEN")"
79+
echo "R:$R" >&2
80+
status=$(echo $R | jq .status -r)
81+
if [ "$status" != "queued" -a "$status" != "running" ]
82+
then
83+
break
84+
fi
85+
done
86+
echo "status after:$status" >&2
87+
echo "$R" > $BATS_TMPDIR/job2.final.json
88+
[ "$status" == "success" ]
89+
}
90+
91+
@test "Should have final output in json" {
92+
R="$(cat $BATS_TMPDIR/job2.final.json)"
93+
94+
echo "R:$R" >&2
95+
output="$(echo $R | jq .output -r)"
96+
97+
echo "$output" | grep "pong"
98+
}

0 commit comments

Comments
 (0)