Skip to content

Commit 247a36f

Browse files
zhihonlzhihonl
authored
Add workload fallback logics for entity service name (fluent#18)
1 parent 85866b1 commit 247a36f

9 files changed

+269
-0
lines changed

‎plugins/filter_kubernetes/kube_meta.c

+24
Original file line numberDiff line numberDiff line change
@@ -737,12 +737,24 @@ static void cb_results_workload(const char *name, const char *value,
737737
}
738738
}
739739

740+
/*
741+
* Search workload based on the following priority
742+
* where the top is highest priority
743+
* 1. Deployment name
744+
* 2. StatefulSet name
745+
* 3. DaemonSet name
746+
* 4. Job name
747+
* 5. CronJob name
748+
* 6. Pod name
749+
* 7. Container name
750+
*/
740751
static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgpack_object map)
741752
{
742753
int i,j,ownerIndex;
743754
int regex_found;
744755
int replicaset_match;
745756
int podname_match = FLB_FALSE;
757+
int workload_found = FLB_FALSE;
746758
msgpack_object k, v;
747759
msgpack_object_map ownerMap;
748760
struct flb_regex_search result;
@@ -788,6 +800,7 @@ static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgp
788800
if (strncmp(key.via.str.ptr, "name", key.via.str.size) == 0) {
789801
/* Store the value of 'name' in workload_val so it can be reused by set_workload */
790802
workload_val = val;
803+
workload_found = FLB_TRUE;
791804
if (replicaset_match) {
792805
regex_found = flb_regex_do(ctx->deploymentRegex, val.via.str.ptr, val.via.str.size, &result);
793806
if (regex_found > 0) {
@@ -808,6 +821,17 @@ static void search_workload(struct flb_kube_meta *meta,struct flb_kube *ctx,msgp
808821
}
809822
}
810823
}
824+
if(!workload_found) {
825+
if(meta->podname != NULL) {
826+
meta->workload = flb_strndup(meta->podname, meta->podname_len);
827+
meta->workload_len = meta->podname_len;
828+
meta->fields++;
829+
} else if (meta->container_name != NULL) {
830+
meta->workload = flb_strndup(meta->container_name, meta->container_name_len);
831+
meta->workload_len = meta->container_name_len;
832+
meta->fields++;
833+
}
834+
}
811835

812836
return;
813837

‎plugins/out_cloudwatch_logs/cloudwatch_api.c

+4
Original file line numberDiff line numberDiff line change
@@ -969,6 +969,10 @@ void parse_entity(struct flb_cloudwatch *ctx, entity *entity, msgpack_object map
969969
}
970970
}
971971
}
972+
if(entity->key_attributes->name == NULL && entity->attributes->name_source == NULL &&entity->attributes->workload != NULL) {
973+
entity->key_attributes->name = flb_strndup(entity->attributes->workload, strlen(entity->attributes->workload));
974+
entity->attributes->name_source = flb_strndup("K8sWorkload", 11);
975+
}
972976
}
973977

974978
void update_or_create_entity(struct flb_cloudwatch *ctx, struct log_stream *stream, const msgpack_object map) {

‎tests/runtime/data/kubernetes/log/options/options_use-kubelet-disabled-pod_fluent-bit.log

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"}

‎tests/runtime/data/kubernetes/log/options/options_use-kubelet-enabled-pod_fluent-bit.log

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"log":"Fluent Bit is logging\n","stream":"stdout","time":"2019-04-01T17:58:33.598656444Z"}

‎tests/runtime/data/kubernetes/meta/options_use-kubelet-disabled-pod.meta

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"apiVersion": "v1",
3+
"kind": "Pod",
4+
"metadata": {
5+
"annotations": {
6+
"prometheus.io/path": "/api/v1/metrics/prometheus",
7+
"prometheus.io/port": "2020",
8+
"prometheus.io/scrape": "true"
9+
},
10+
"creationTimestamp": "2019-04-03T09:29:00Z",
11+
"labels": {
12+
"app.kubernetes.io/name": "fluent-bit"
13+
},
14+
"name": "use-kubelet-disabled-pod",
15+
"namespace": "options",
16+
"resourceVersion": "74466568",
17+
"selfLink": "/api/v1/namespaces/core/pods/base",
18+
"uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a"
19+
},
20+
"spec": {
21+
"containers": [
22+
{
23+
"image": "fluent/fluent-bit",
24+
"imagePullPolicy": "Always",
25+
"name": "fluent-bit",
26+
"resources": {},
27+
"stdin": true,
28+
"stdinOnce": true,
29+
"terminationMessagePath": "/dev/termination-log",
30+
"terminationMessagePolicy": "File",
31+
"tty": true,
32+
"volumeMounts": [
33+
{
34+
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
35+
"name": "default-token-9ffht",
36+
"readOnly": true
37+
}
38+
]
39+
}
40+
],
41+
"dnsPolicy": "ClusterFirst",
42+
"nodeName": "ip-10-49-18-80.eu-west-1.compute.internal",
43+
"restartPolicy": "Never",
44+
"schedulerName": "default-scheduler",
45+
"securityContext": {},
46+
"serviceAccount": "default",
47+
"serviceAccountName": "default",
48+
"terminationGracePeriodSeconds": 30,
49+
"tolerations": [
50+
{
51+
"effect": "NoExecute",
52+
"key": "node.kubernetes.io/not-ready",
53+
"operator": "Exists",
54+
"tolerationSeconds": 300
55+
},
56+
{
57+
"effect": "NoExecute",
58+
"key": "node.kubernetes.io/unreachable",
59+
"operator": "Exists",
60+
"tolerationSeconds": 300
61+
}
62+
],
63+
"volumes": [
64+
{
65+
"name": "default-token-9ffht",
66+
"secret": {
67+
"defaultMode": 420,
68+
"secretName": "default-token-9ffht"
69+
}
70+
}
71+
]
72+
},
73+
"status": {
74+
"conditions": [
75+
{
76+
"lastProbeTime": null,
77+
"lastTransitionTime": "2019-04-03T09:29:00Z",
78+
"status": "True",
79+
"type": "Initialized"
80+
},
81+
{
82+
"lastProbeTime": null,
83+
"lastTransitionTime": "2019-04-03T09:29:06Z",
84+
"status": "True",
85+
"type": "Ready"
86+
},
87+
{
88+
"lastProbeTime": null,
89+
"lastTransitionTime": "2019-04-03T09:29:00Z",
90+
"status": "True",
91+
"type": "PodScheduled"
92+
}
93+
],
94+
"containerStatuses": [
95+
{
96+
"containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16",
97+
"image": "fluent/fluent-bit:latest",
98+
"imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f",
99+
"lastState": {},
100+
"name": "fluent-bit",
101+
"ready": true,
102+
"restartCount": 0,
103+
"state": {
104+
"running": {
105+
"startedAt": "2019-04-03T09:29:05Z"
106+
}
107+
}
108+
}
109+
],
110+
"hostIP": "10.49.18.80",
111+
"phase": "Running",
112+
"podIP": "100.116.192.42",
113+
"qosClass": "BestEffort",
114+
"startTime": "2019-04-03T09:29:00Z"
115+
}
116+
}

‎tests/runtime/data/kubernetes/meta/options_use-kubelet-enabled-pod.meta

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
{
2+
"apiVersion": "v1",
3+
"kind": "Pod",
4+
"metadata": {},
5+
"items": [{
6+
"metadata": {
7+
"annotations": {
8+
"prometheus.io/path": "/api/v1/metrics/prometheus",
9+
"prometheus.io/port": "2020",
10+
"prometheus.io/scrape": "true"
11+
},
12+
"creationTimestamp": "2019-04-03T09:29:00Z",
13+
"labels": {
14+
"app.kubernetes.io/name": "fluent-bit"
15+
},
16+
"name": "use-kubelet-enabled-pod",
17+
"namespace": "options",
18+
"resourceVersion": "74466568",
19+
"selfLink": "/api/v1/namespaces/core/pods/base",
20+
"uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a"
21+
},
22+
"spec": {
23+
"containers": [{
24+
"image": "fluent/fluent-bit",
25+
"imagePullPolicy": "Always",
26+
"name": "fluent-bit",
27+
"resources": {},
28+
"stdin": true,
29+
"stdinOnce": true,
30+
"terminationMessagePath": "/dev/termination-log",
31+
"terminationMessagePolicy": "File",
32+
"tty": true,
33+
"volumeMounts": [{
34+
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
35+
"name": "default-token-9ffht",
36+
"readOnly": true
37+
}]
38+
}],
39+
"dnsPolicy": "ClusterFirst",
40+
"nodeName": "ip-10-49-18-80.eu-west-1.compute.internal",
41+
"restartPolicy": "Never",
42+
"schedulerName": "default-scheduler",
43+
"securityContext": {},
44+
"serviceAccount": "default",
45+
"serviceAccountName": "default",
46+
"terminationGracePeriodSeconds": 30,
47+
"tolerations": [{
48+
"effect": "NoExecute",
49+
"key": "node.kubernetes.io/not-ready",
50+
"operator": "Exists",
51+
"tolerationSeconds": 300
52+
},
53+
{
54+
"effect": "NoExecute",
55+
"key": "node.kubernetes.io/unreachable",
56+
"operator": "Exists",
57+
"tolerationSeconds": 300
58+
}
59+
],
60+
"volumes": [{
61+
"name": "default-token-9ffht",
62+
"secret": {
63+
"defaultMode": 420,
64+
"secretName": "default-token-9ffht"
65+
}
66+
}]
67+
},
68+
"status": {
69+
"conditions": [{
70+
"lastProbeTime": null,
71+
"lastTransitionTime": "2019-04-03T09:29:00Z",
72+
"status": "True",
73+
"type": "Initialized"
74+
},
75+
{
76+
"lastProbeTime": null,
77+
"lastTransitionTime": "2019-04-03T09:29:06Z",
78+
"status": "True",
79+
"type": "Ready"
80+
},
81+
{
82+
"lastProbeTime": null,
83+
"lastTransitionTime": "2019-04-03T09:29:00Z",
84+
"status": "True",
85+
"type": "PodScheduled"
86+
}
87+
],
88+
"containerStatuses": [{
89+
"containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16",
90+
"image": "fluent/fluent-bit:latest",
91+
"imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f",
92+
"lastState": {},
93+
"name": "fluent-bit",
94+
"ready": true,
95+
"restartCount": 0,
96+
"state": {
97+
"running": {
98+
"startedAt": "2019-04-03T09:29:05Z"
99+
}
100+
}
101+
}],
102+
"hostIP": "10.49.18.80",
103+
"phase": "Running",
104+
"podIP": "100.116.192.42",
105+
"qosClass": "BestEffort",
106+
"startTime": "2019-04-03T09:29:00Z"
107+
}
108+
}]
109+
}

‎tests/runtime/data/kubernetes/out/options/options_use-kubelet-disabled-pod_fluent-bit.out

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-disabled-pod","namespace_name":"options","workload":"use-kubelet-disabled-pod","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}]

‎tests/runtime/data/kubernetes/out/options/options_use-kubelet-enabled-pod_fluent-bit.out

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"use-kubelet-enabled-pod","namespace_name":"options","workload":"use-kubelet-enabled-pod","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","labels":{"app.kubernetes.io/name":"fluent-bit"},"annotations":{"prometheus.io/path":"/api/v1/metrics/prometheus","prometheus.io/port":"2020","prometheus.io/scrape":"true"},"host":"ip-10-49-18-80.eu-west-1.compute.internal","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}]

‎tests/runtime/filter_kubernetes.c

+12
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,11 @@ static void flb_test_options_use_kubelet_enabled_daemonset_json()
463463
flb_test_options_use_kubelet_enabled("options_use-kubelet-enabled-daemonset_fluent-bit", NULL, 1);
464464
}
465465

466+
static void flb_test_options_use_kubelet_enabled_pod_json()
467+
{
468+
flb_test_options_use_kubelet_enabled("options_use-kubelet-enabled-pod_fluent-bit", NULL, 1);
469+
}
470+
466471
static void flb_test_options_use_kubelet_disabled_json()
467472
{
468473
flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled_fluent-bit", NULL, 1);
@@ -483,6 +488,11 @@ static void flb_test_options_use_kubelet_disabled_daemonset_json()
483488
flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled-daemonset_fluent-bit", NULL, 1);
484489
}
485490

491+
static void flb_test_options_use_kubelet_disabled_pod_json()
492+
{
493+
flb_test_options_use_kubelet_disabled("options_use-kubelet-disabled-pod_fluent-bit", NULL, 1);
494+
}
495+
486496

487497
#define flb_test_options_merge_log_enabled(target, suffix, nExpected) \
488498
kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \
@@ -1075,10 +1085,12 @@ TEST_LIST = {
10751085
{"kube_options_use-kubelet_enabled_replicateset_json", flb_test_options_use_kubelet_enabled_replicaset_json},
10761086
{"kube_options_use-kubelet_enabled_deployment_json", flb_test_options_use_kubelet_enabled_deployment_json},
10771087
{"kube_options_use-kubelet_enabled_daemonset_json", flb_test_options_use_kubelet_enabled_daemonset_json},
1088+
{"kube_options_use-kubelet_enabled_pod_json", flb_test_options_use_kubelet_enabled_pod_json},
10781089
{"kube_options_use-kubelet_disabled_json", flb_test_options_use_kubelet_disabled_json},
10791090
{"kube_options_use-kubelet_disabled_replicaset_json", flb_test_options_use_kubelet_disabled_replicaset_json},
10801091
{"kube_options_use-kubelet_disabled_deployment_json", flb_test_options_use_kubelet_disabled_deployment_json},
10811092
{"kube_options_use-kubelet_disabled_daemonset_json", flb_test_options_use_kubelet_disabled_daemonset_json},
1093+
{"kube_options_use-kubelet_disabled_pod_json", flb_test_options_use_kubelet_disabled_pod_json},
10821094
{"kube_options_merge_log_enabled_text", flb_test_options_merge_log_enabled_text},
10831095
{"kube_options_merge_log_enabled_json", flb_test_options_merge_log_enabled_json},
10841096
{"kube_options_merge_log_enabled_invalid_json", flb_test_options_merge_log_enabled_invalid_json},

0 commit comments

Comments
 (0)