Skip to content

Commit 0de672d

Browse files
nilekhcnilekhc
authored
Metrics (#104)
* feat: adds otel metrics Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com> * chore: gofmt * fix: updates decrypt metrics * fix: updates tests * chore: update metric name * chore: makes comments style consistent * chore: updates klog to v2 * chore: formatting * chore: updates modules * fix: updates metric types * chore: removes dead code * fix: passes raw errors to metrics reporter * docs: adds metrics doc * chore: updates docs * chore: simplifies metrics reporting * feat: implements metrics for grpc server * chore: updates error key * test: adds test for grpc method name * test: adds test for exporter * chore: updates tests * docs: updates metrics docs * test: adds test * fix: checks error * refactor: updates report function * test: updates test * refactor: updates UnaryServerInterceptor * chore: updates comment * docs: updates metrics description * test: adds test for metrics endpoint
1 parent a319efc commit 0de672d

16 files changed

+791
-98
lines changed

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,10 @@ authors:
6969
$Q rm -f GITAUTHORS
7070

7171
integration-test:
72-
$Q sudo GOPATH=$(GOPATH) go test -v -count=1 github.com/Azure/kubernetes-kms/tests/client
72+
$Q sudo GOPATH=$(GOPATH) go test -v -count=1 -failfast github.com/Azure/kubernetes-kms/tests/client
7373

7474
unit-test:
75-
go test -race -v -count=1 `go list ./... | grep -v client`
75+
go test -race -v -count=1 -failfast `go list ./... | grep -v client`
7676

7777
.PHONY: mod
7878
mod:

‎cmd/server/main.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"syscall"
1717
"time"
1818

19+
"github.com/Azure/kubernetes-kms/pkg/metrics"
1920
"github.com/Azure/kubernetes-kms/pkg/plugin"
2021
"github.com/Azure/kubernetes-kms/pkg/utils"
2122
"github.com/Azure/kubernetes-kms/pkg/version"
@@ -40,6 +41,8 @@ var (
4041
healthzPort = flag.Int("healthz-port", 8787, "port for health check")
4142
healthzPath = flag.String("healthz-path", "/healthz", "path for health check")
4243
healthzTimeout = flag.Duration("healthz-timeout", 20*time.Second, "RPC timeout for health check")
44+
metricsBackend = flag.String("metrics-backend", "prometheus", "Backend used for metrics")
45+
metricsAddress = flag.String("metrics-addr", "8095", "The address the metric endpoint binds to")
4346
)
4447

4548
func main() {
@@ -58,6 +61,12 @@ func main() {
5861

5962
ctx := withShutdownSignal(context.Background())
6063

64+
// initialize metrics exporter
65+
err := metrics.InitMetricsExporter(*metricsBackend, *metricsAddress)
66+
if err != nil {
67+
klog.Fatalf("failed to initialize metrics exporter, error: %+v", err)
68+
}
69+
6170
klog.InfoS("Starting KeyManagementServiceServer service", "version", version.BuildVersion, "buildDate", version.BuildDate)
6271
kmsServer, err := plugin.New(ctx, *configFilePath, *keyvaultName, *keyName, *keyVersion)
6372
if err != nil {
@@ -78,7 +87,7 @@ func main() {
7887
klog.Fatalf("failed to listen: %v", err)
7988
}
8089
opts := []grpc.ServerOption{
81-
grpc.UnaryInterceptor(utils.LogGRPC),
90+
grpc.UnaryInterceptor(utils.UnaryServerInterceptor),
8291
}
8392

8493
s := grpc.NewServer(opts...)

‎docs/metrics.md

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Metrics provided by KMS plugin for Key Vault
2+
3+
This project uses [opentelemetry](https://opentelemetry.io/) for reporting metrics. Please refer to it's status [here](https://github.com/open-telemetry/opentelemetry-go#project-status). Prometheus is the only exporter that's currently supported.
4+
5+
## List of metrics provided by the kms plugin
6+
7+
| Metric | Description | Tags |
8+
| ------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------- |
9+
| kms_request | Distribution of how long it took for an operation | `status=success OR error`<br><br>`operation=encrypt OR decrypt OR grpc_encrypt OR grpc_decrypt`<br><br>`error_message` |
10+
11+
12+
### Sample Metrics output
13+
14+
```shell
15+
# HELP kms_request Distribution of how long it took for an operation
16+
# TYPE kms_request histogram
17+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.1"} 39
18+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.2"} 77
19+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.3"} 156
20+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.4"} 170
21+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.5"} 180
22+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1"} 198
23+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1.5"} 200
24+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2"} 200
25+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2.5"} 200
26+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="3"} 200
27+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="5"} 200
28+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="10"} 200
29+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="15"} 200
30+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="30"} 200
31+
kms_request_bucket{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="+Inf"} 200
32+
kms_request_sum{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 49.982473866999996
33+
kms_request_count{operation="decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 200
34+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.1"} 0
35+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.2"} 2
36+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.3"} 12
37+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.4"} 36
38+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.5"} 65
39+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1"} 100
40+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1.5"} 100
41+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2"} 137
42+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2.5"} 168
43+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="3"} 176
44+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="5"} 200
45+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="10"} 200
46+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="15"} 200
47+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="30"} 200
48+
kms_request_bucket{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="+Inf"} 200
49+
kms_request_sum{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 271.335309324
50+
kms_request_count{operation="encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 200
51+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.1"} 39
52+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.2"} 77
53+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.3"} 156
54+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.4"} 170
55+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.5"} 180
56+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1"} 198
57+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1.5"} 200
58+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2"} 200
59+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2.5"} 200
60+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="3"} 200
61+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="5"} 200
62+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="10"} 200
63+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="15"} 200
64+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="30"} 200
65+
kms_request_bucket{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="+Inf"} 200
66+
kms_request_sum{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 49.993816699999996
67+
kms_request_count{operation="grpc_decrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 200
68+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.1"} 0
69+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.2"} 2
70+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.3"} 12
71+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.4"} 36
72+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="0.5"} 65
73+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1"} 100
74+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="1.5"} 100
75+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2"} 137
76+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="2.5"} 168
77+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="3"} 176
78+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="5"} 200
79+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="10"} 200
80+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="15"} 200
81+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="30"} 200
82+
kms_request_bucket{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0",le="+Inf"} 200
83+
kms_request_sum{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 271.3962141690001
84+
kms_request_count{operation="grpc_encrypt",service_name="unknown_service:__debug_bin",status="success",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="0.20.0"} 200
85+
```

‎go.mod

+14-11
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,19 @@ module github.com/Azure/kubernetes-kms
33
go 1.16
44

55
require (
6-
github.com/Azure/azure-sdk-for-go v48.2.0+incompatible
7-
github.com/Azure/go-autorest/autorest v0.9.6
8-
github.com/Azure/go-autorest/autorest/adal v0.8.2
6+
github.com/Azure/azure-sdk-for-go v55.0.0+incompatible
7+
github.com/Azure/go-autorest/autorest v0.11.18
8+
github.com/Azure/go-autorest/autorest/adal v0.9.13
99
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
10-
github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect
11-
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
12-
golang.org/x/net v0.0.0-20200707034311-ab3426394381
13-
google.golang.org/grpc v1.27.0
14-
gopkg.in/yaml.v2 v2.3.0
15-
k8s.io/apiserver v0.19.4
16-
k8s.io/component-base v0.19.4
17-
k8s.io/klog/v2 v2.2.0
10+
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
11+
go.opentelemetry.io/otel v0.20.0
12+
go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0
13+
go.opentelemetry.io/otel/metric v0.20.0
14+
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
15+
golang.org/x/net v0.0.0-20210525063256-abc453219eb5
16+
google.golang.org/grpc v1.38.0
17+
gopkg.in/yaml.v2 v2.4.0
18+
k8s.io/apiserver v0.21.1
19+
k8s.io/component-base v0.21.1
20+
k8s.io/klog/v2 v2.9.0
1821
)

0 commit comments

Comments
 (0)