Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

filter_kubernetes: add owner references information into record. #9719

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+168 −6
Open

filter_kubernetes: add owner references information into record. #9719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/cprofiles/src/cprof_profile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,4 +307,4 @@ int cprof_profile_add_comment(struct cprof_profile *profile, int64_t comment)
profile->comments_count++;

return 0;
}
}
4 changes: 2 additions & 2 deletions plugins/filter_kubernetes/kube_conf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins,
FLB_HASH_TABLE_SIZE,
FLB_HASH_TABLE_SIZE);
}

if (ctx->kube_meta_namespace_cache_ttl > 0) {
ctx->namespace_hash_table = flb_hash_table_create_with_ttl(
ctx->kube_meta_namespace_cache_ttl,
Expand All @@ -150,7 +150,7 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *ins,
FLB_HASH_TABLE_SIZE,
FLB_HASH_TABLE_SIZE);
}


if (!ctx->hash_table || !ctx->namespace_hash_table) {
flb_kube_conf_destroy(ctx);
Expand Down
1 change: 1 addition & 0 deletions plugins/filter_kubernetes/kube_conf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ struct flb_kube {
int cache_use_docker_id;
int labels;
int annotations;
int owner_references;
int namespace_labels;
int namespace_annotations;
int namespace_metadata_only;
Expand Down
18 changes: 16 additions & 2 deletions plugins/filter_kubernetes/kube_meta.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,7 @@ static int merge_pod_meta(struct flb_kube_meta *meta, struct flb_kube *ctx,
int have_uid = -1;
int have_labels = -1;
int have_annotations = -1;
int have_owner_references = -1;
int have_nodename = -1;
int have_podip = -1;
size_t off = 0;
Expand Down Expand Up @@ -1228,15 +1229,20 @@ static int merge_pod_meta(struct flb_kube_meta *meta, struct flb_kube *ctx,
map_size++;
}
}

else if (size == 11 && strncmp(ptr, "annotations", 11) == 0) {
have_annotations = i;
if (ctx->annotations == FLB_TRUE) {
map_size++;
}
}
else if (size == 15 && strncmp(ptr, "ownerReferences", 15) == 0) {
have_owner_references = i;
if (ctx->owner_references == FLB_TRUE) {
map_size++;
}
}

if (have_uid >= 0 && have_labels >= 0 && have_annotations >= 0) {
if (have_uid >= 0 && have_labels >= 0 && have_annotations >= 0 && have_owner_references >= 0) {
break;
}
}
Expand Down Expand Up @@ -1317,6 +1323,14 @@ static int merge_pod_meta(struct flb_kube_meta *meta, struct flb_kube *ctx,
msgpack_pack_object(&mp_pck, v);
}

if (have_owner_references >= 0 && ctx->owner_references == FLB_TRUE) {
k = meta_val.via.map.ptr[have_owner_references].key;
v = meta_val.via.map.ptr[have_owner_references].val;

msgpack_pack_object(&mp_pck, k);
msgpack_pack_object(&mp_pck, v);
}

if (have_nodename >= 0) {
v = spec_val.via.map.ptr[have_nodename].val;

Expand Down
9 changes: 8 additions & 1 deletion plugins/filter_kubernetes/kubernetes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ static int pack_map_content(struct flb_log_event_encoder *log_encoder,
ret = flb_log_event_encoder_append_body_cstring(
log_encoder,
"kubernetes_namespace");

off = 0;
msgpack_unpacked_init(&result);
msgpack_unpack_next(&result, namespace_kube_buf,
Expand Down Expand Up @@ -909,6 +909,13 @@ static struct flb_config_map config_map[] = {
"include Kubernetes annotations on every record"
},

/* Include Kubernetes OwnerReferences in the final record ? */
{
FLB_CONFIG_MAP_BOOL, "owner_references", "false",
0, FLB_TRUE, offsetof(struct flb_kube, owner_references),
"include Kubernetes owner references on every record"
},

/* Include Kubernetes Namespace Labels in the final record ? */
{
FLB_CONFIG_MAP_BOOL, "namespace_labels", "false",
Expand Down
126 changes: 126 additions & 0 deletions tests/runtime/data/kubernetes/meta/core_base-with-owner-references.meta
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"uid": "e9f2963f-55f2-11e9-84c5-02e422b8a84a",
"annotations": {
"prometheus.io/path": "/api/v1/metrics/prometheus",
"prometheus.io/port": "2020",
"prometheus.io/scrape": "true"
},
"creationTimestamp": "2019-04-03T09:29:00Z",
"labels": {
"app.kubernetes.io/name": "fluent-bit"
},
"ownerReferences": [
{
"apiVersion": "apps/v1",
"kind": "DaemonSet",
"name": "fluentbit-max",
"uid": "a0a2ee0a-08da-5a7f-ac1a-c0a4255f82f2",
"controller": true,
"blockOwnerDeletion": true
},
],
"name": "base",
"namespace": "core",
"resourceVersion": "74466568",
"selfLink": "/api/v1/namespaces/core/pods/base"
},
"spec": {
"containers": [
{
"image": "fluent/fluent-bit",
"imagePullPolicy": "Always",
"name": "fluent-bit",
"resources": {},
"stdin": true,
"stdinOnce": true,
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"tty": true,
"volumeMounts": [
{
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
"name": "default-token-9ffht",
"readOnly": true
}
]
}
],
"dnsPolicy": "ClusterFirst",
"nodeName": "ip-10-49-18-80.eu-west-1.compute.internal",
"restartPolicy": "Never",
"schedulerName": "default-scheduler",
"securityContext": {},
"serviceAccount": "default",
"serviceAccountName": "default",
"terminationGracePeriodSeconds": 30,
"tolerations": [
{
"effect": "NoExecute",
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"tolerationSeconds": 300
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"tolerationSeconds": 300
}
],
"volumes": [
{
"name": "default-token-9ffht",
"secret": {
"defaultMode": 420,
"secretName": "default-token-9ffht"
}
}
]
},
"status": {
"conditions": [
{
"lastProbeTime": null,
"lastTransitionTime": "2019-04-03T09:29:00Z",
"status": "True",
"type": "Initialized"
},
{
"lastProbeTime": null,
"lastTransitionTime": "2019-04-03T09:29:06Z",
"status": "True",
"type": "Ready"
},
{
"lastProbeTime": null,
"lastTransitionTime": "2019-04-03T09:29:00Z",
"status": "True",
"type": "PodScheduled"
}
],
"containerStatuses": [
{
"containerID": "docker://c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16",
"image": "fluent/fluent-bit:latest",
"imageID": "docker-pullable://fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f",
"lastState": {},
"name": "fluent-bit",
"ready": true,
"restartCount": 0,
"state": {
"running": {
"startedAt": "2019-04-03T09:29:05Z"
}
}
}
],
"hostIP": "10.49.18.80",
"phase": "Running",
"podIP": "100.116.192.42",
"qosClass": "BestEffort",
"startTime": "2019-04-03T09:29:00Z"
}
}
1 change: 1 addition & 0 deletions tests/runtime/data/kubernetes/out/core/core_base-with-owner-references_fluent-bit.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[1554141513.598656,{"log":"Fluent Bit is logging\n","stream":"stdout","kubernetes":{"pod_name":"base-with-owner-references","namespace_name":"core","pod_id":"e9f2963f-55f2-11e9-84c5-02e422b8a84a","ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"fluentbit-max","uid":"a0a2ee0a-08da-5a7f-ac1a-c0a4255f82f2","controller":true,"blockOwnerDeletion":true}],"host":"ip-10-49-18-80.eu-west-1.compute.internal","pod_ip":"100.116.192.42","container_name":"fluent-bit","docker_id":"c9898099f6d235126d564ed38a020007ea7a6fac6e25e718de683c9dd0076c16","container_hash":"fluent/fluent-bit@sha256:7ac0fd3569af866e9a6a22eb592744200d2dbe098cf066162453f8d0b06c531f","container_image":"fluent/fluent-bit:latest"}}]
13 changes: 13 additions & 0 deletions tests/runtime/filter_kubernetes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,18 @@ static void flb_test_core_base_with_namespace_labels_and_annotations()
flb_test_namespace_labels_and_annotations("core_base-with-namespace-labels-and-annotations_fluent-bit", NULL, 1);
}

#define flb_test_owner_references(target, suffix, nExpected) \
kube_test("core/" target, KUBE_TAIL, suffix, nExpected, \
"Labels", "Off", \
"Annotations", "Off", \
"Owner_References", "On", \
NULL); \

static void flb_test_core_base_with_owner_references()
{
flb_test_owner_references("core_base-with-owner-references_fluent-bit", NULL, 1);
}

#define flb_test_options_use_kubelet_enabled(target, suffix, nExpected) \
kube_test("options/" target, KUBE_TAIL, suffix, nExpected, \
"use_kubelet", "true", \
Expand Down Expand Up @@ -1005,6 +1017,7 @@ TEST_LIST = {
{"kube_core_unescaping_text", flb_test_core_unescaping_text},
{"kube_core_unescaping_json", flb_test_core_unescaping_json},
{"kube_core_base_with_namespace_labels_and_annotations", flb_test_core_base_with_namespace_labels_and_annotations},
{"kube_core_base_with_owner_references", flb_test_core_base_with_owner_references},
{"kube_options_use-kubelet_enabled_json", flb_test_options_use_kubelet_enabled_json},
{"kube_options_use-kubelet_disabled_json", flb_test_options_use_kubelet_disabled_json},
{"kube_options_merge_log_enabled_text", flb_test_options_merge_log_enabled_text},
Expand Down
Loading