Replace docker_id for container_id

Signed-off-by: Uri Sternik <[email protected]>
fluent · Nov 28, 2024 · a04eafd · a04eafd
1 parent a59c867
commit a04eafd
Show file tree

Hide file tree

Showing 23 changed files with 68 additions and 68 deletions.
diff --git a/conf/parsers.conf b/conf/parsers.conf
@@ -127,7 +127,7 @@
 [PARSER]
     Name    kube-custom
     Format  regex
-    Regex   (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
+    Regex   (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<container_id_id>[a-z0-9]{64})\.log$
 
 [PARSER]
     # Examples: TCP: https://rubular.com/r/Q8YY6fHqlqwGI0  UDP: https://rubular.com/r/B0ID69H9FvN0tp

diff --git a/plugins/filter_kubernetes/kube_conf.h b/plugins/filter_kubernetes/kube_conf.h
@@ -71,7 +71,7 @@ struct kube_meta;
 struct flb_kube {
     /* Configuration parameters */
     int use_journal;
-    int cache_use_docker_id;
+    int cache_use_container_id;
     int labels;
     int annotations;
     int namespace_labels;

diff --git a/plugins/filter_kubernetes/kube_meta.c b/plugins/filter_kubernetes/kube_meta.c
@@ -592,10 +592,10 @@ static void cb_results(const char *name, const char *value,
         meta->container_name_len = vlen;
         meta->fields++;
     }
-    else if (meta->docker_id == NULL &&
-             strcmp(name, "docker_id") == 0) {
-        meta->docker_id = flb_strndup(value, vlen);
-        meta->docker_id_len = vlen;
+    else if (meta->container_id == NULL &&
+             strcmp(name, "container_id") == 0) {
+        meta->container_id = flb_strndup(value, vlen);
+        meta->container_id_len = vlen;
         meta->fields++;
     }
     else if (meta->container_hash == NULL &&
@@ -669,18 +669,18 @@ static void extract_container_hash(struct flb_kube_meta *meta,
 {
     int i;
     msgpack_object k, v;
-    int docker_id_len = 0;
+    int container_id_len = 0;
     int container_hash_len = 0;
     int container_image_len = 0;
     const char *container_hash;
-    const char *docker_id;
+    const char *container_id;
     const char *container_image;
     const char *tmp;
     int tmp_len = 0;
     int name_found = FLB_FALSE;
-    /* Process status/containerStatus map for docker_id, container_hash, container_image */
+    /* Process status/containerStatus map for container_id, container_hash, container_image */
     for (i = 0;
-         (meta->docker_id_len == 0 || meta->container_hash_len == 0 ||
+         (meta->container_id_len == 0 || meta->container_hash_len == 0 ||
           meta->container_image_len == 0) &&
          i < status.via.map.size; i++) {
         k = status.via.map.ptr[i].key;
@@ -695,7 +695,7 @@ static void extract_container_hash(struct flb_kube_meta *meta,
             int j;
             v = status.via.map.ptr[i].val;
             for (j = 0;
-                 (meta->docker_id_len == 0 ||
+                 (meta->container_id_len == 0 ||
                   meta->container_hash_len == 0 ||
                   meta->container_image_len == 0) && j < v.via.array.size;
                  j++) {
@@ -704,7 +704,7 @@ static void extract_container_hash(struct flb_kube_meta *meta,
                 msgpack_object_str v2;
                 k1 = v.via.array.ptr[j];
                 for (l = 0;
-                     (meta->docker_id_len == 0 ||
+                     (meta->container_id_len == 0 ||
                       meta->container_hash_len == 0 ||
                       meta->container_image_len == 0) &&
                      l < k1.via.map.size; l++) {
@@ -727,8 +727,8 @@ static void extract_container_hash(struct flb_kube_meta *meta,
                                  "containerID",
                                  k2.via.str.size)) {
                         if (extract_hash(v2.ptr, v2.size, &tmp, &tmp_len) == 0) {
-                            docker_id = tmp;
-                            docker_id_len = tmp_len;
+                            container_id = tmp;
+                            container_id_len = tmp_len;
                         }
                     }
                     else if (k2.via.str.size == sizeof("imageID") - 1 &&
@@ -755,9 +755,9 @@ static void extract_container_hash(struct flb_kube_meta *meta,
                                                            container_hash_len);
                         meta->fields++;
                     }
-                    if (docker_id_len && !meta->docker_id_len) {
-                        meta->docker_id_len = docker_id_len;
-                        meta->docker_id = flb_strndup(docker_id, docker_id_len);
+                    if (container_id_len && !meta->container_id_len) {
+                        meta->container_id_len = container_id_len;
+                        meta->container_id = flb_strndup(container_id, container_id_len);
                         meta->fields++;
                     }
                     if (container_image_len && !meta->container_image_len) {
@@ -951,13 +951,13 @@ static int merge_meta_from_tag(struct flb_kube *ctx, struct flb_kube_meta *meta,
         msgpack_pack_str_body(&mp_pck, meta->container_name,
                               meta->container_name_len);
     }
-    if (meta->docker_id != NULL) {
+    if (meta->container_id != NULL) {
         flb_mp_map_header_append(&mh);
         msgpack_pack_str(&mp_pck, 9);
-        msgpack_pack_str_body(&mp_pck, "docker_id", 9);
-        msgpack_pack_str(&mp_pck, meta->docker_id_len);
-        msgpack_pack_str_body(&mp_pck, meta->docker_id,
-                              meta->docker_id_len);
+        msgpack_pack_str_body(&mp_pck, "container_id", 9);
+        msgpack_pack_str(&mp_pck, meta->container_id_len);
+        msgpack_pack_str_body(&mp_pck, meta->container_id,
+                              meta->container_id_len);
     }
 
     flb_mp_map_header_end(&mh);
@@ -1268,7 +1268,7 @@ static int merge_pod_meta(struct flb_kube_meta *meta, struct flb_kube *ctx,
                 }
             }
 
-            if ((!meta->container_hash || !meta->docker_id || !meta->container_image) && status_found) {
+            if ((!meta->container_hash || !meta->container_id || !meta->container_image) && status_found) {
                 extract_container_hash(meta, status_val);
             }
         }
@@ -1340,12 +1340,12 @@ static int merge_pod_meta(struct flb_kube_meta *meta, struct flb_kube *ctx,
         msgpack_pack_str_body(&mp_pck, meta->container_name,
                               meta->container_name_len);
     }
-    if (meta->docker_id != NULL) {
+    if (meta->container_id != NULL) {
         msgpack_pack_str(&mp_pck, 9);
-        msgpack_pack_str_body(&mp_pck, "docker_id", 9);
-        msgpack_pack_str(&mp_pck, meta->docker_id_len);
-        msgpack_pack_str_body(&mp_pck, meta->docker_id,
-                              meta->docker_id_len);
+        msgpack_pack_str_body(&mp_pck, "container_id", 9);
+        msgpack_pack_str(&mp_pck, meta->container_id_len);
+        msgpack_pack_str_body(&mp_pck, meta->container_id,
+                              meta->container_id_len);
     }
     if (meta->container_hash != NULL) {
         msgpack_pack_str(&mp_pck, 14);
@@ -1565,8 +1565,8 @@ static inline int extract_pod_meta(struct flb_kube *ctx,
         if (meta->container_name) {
             n += meta->container_name_len + 1;
         }
-        if (ctx->cache_use_docker_id && meta->docker_id) {
-            n += meta->docker_id_len + 1;
+        if (ctx->cache_use_container_id && meta->container_id) {
+            n += meta->container_id_len + 1;
         }
         meta->cache_key = flb_malloc(n);
         if (!meta->cache_key) {
@@ -1592,11 +1592,11 @@ static inline int extract_pod_meta(struct flb_kube *ctx,
             off += meta->container_name_len;
         }
 
-        if (ctx->cache_use_docker_id && meta->docker_id) {
+        if (ctx->cache_use_container_id && meta->container_id) {
             /* Separator */
             meta->cache_key[off++] = ':';
-            memcpy(meta->cache_key + off, meta->docker_id, meta->docker_id_len);
-            off += meta->docker_id_len;
+            memcpy(meta->cache_key + off, meta->container_id, meta->container_id_len);
+            off += meta->container_id_len;
         }
 
         meta->cache_key[off] = '\0';
@@ -2125,8 +2125,8 @@ int flb_kube_meta_release(struct flb_kube_meta *meta)
         r++;
     }
 
-    if (meta->docker_id) {
-        flb_free(meta->docker_id);
+    if (meta->container_id) {
+        flb_free(meta->container_id);
         r++;
     }
 

diff --git a/plugins/filter_kubernetes/kube_meta.h b/plugins/filter_kubernetes/kube_meta.h
@@ -31,15 +31,15 @@ struct flb_kube_meta {
     int podname_len;
     int cache_key_len;
     int container_name_len;
-    int docker_id_len;
+    int container_id_len;
     int container_hash_len;
     int container_image_len;
 
     char *namespace;
     char *podname;
     char *container_name;
     char *container_image;
-    char *docker_id;
+    char *container_id;
 
     char *container_hash;   /* set only on Systemd mode */
 

diff --git a/plugins/filter_kubernetes/kube_regex.h b/plugins/filter_kubernetes/kube_regex.h
@@ -22,7 +22,7 @@
 
 #include "kube_conf.h"
 
-#define KUBE_TAG_TO_REGEX "(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\\.log$"
+#define KUBE_TAG_TO_REGEX "(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<container_id>[a-z0-9]{64})\\.log$"
 
 #define KUBE_JOURNAL_TO_REGEX "^(?<name_prefix>[^_]+)_(?<container_name>[^\\._]+)(\\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace_name>[^_]+)_[^_]+_[^_]+$"
 

diff --git a/plugins/filter_kubernetes/kubernetes.c b/plugins/filter_kubernetes/kubernetes.c
@@ -995,11 +995,11 @@ static struct flb_config_map config_map[] = {
      0, FLB_TRUE, offsetof(struct flb_kube, dns_wait_time),
      "dns interval between network status checks"
     },
-    /* Fetch K8s meta when docker_id has changed */
+    /* Fetch K8s meta when container_id has changed */
     {
-     FLB_CONFIG_MAP_BOOL, "cache_use_docker_id", "false",
-     0, FLB_TRUE, offsetof(struct flb_kube, cache_use_docker_id),
-     "fetch K8s meta when docker_id is changed"
+     FLB_CONFIG_MAP_BOOL, "cache_use_container_id", "false",
+     0, FLB_TRUE, offsetof(struct flb_kube, cache_use_container_id),
+     "fetch K8s meta when container_id is changed"
     },
 
     {

diff --git a/plugins/filter_log_to_metrics/log_to_metrics.c b/plugins/filter_log_to_metrics/log_to_metrics.c
@@ -44,7 +44,7 @@ static char kubernetes_label_keys[NUMBER_OF_KUBERNETES_LABELS][16] =
         "namespace_name",
         "pod_name",
         "container_name",
-        "docker_id",
+        "container_id",
         "pod_id"
     };
 

diff --git a/plugins/in_docker/cgroup_v1.c b/plugins/in_docker/cgroup_v1.c
@@ -408,7 +408,7 @@ static mem_snapshot *get_docker_mem_snapshot(struct flb_docker *ctx, char *id)
 int in_docker_set_cgroup_api_v1(struct cgroup_api *api)
 {
     api->cgroup_version = 1;
-    api->get_active_docker_ids = get_active_dockers;
+    api->get_active_container_ids = get_active_dockers;
     api->get_container_name = get_container_name;
     api->get_cpu_snapshot = get_docker_cpu_snapshot;
     api->get_mem_snapshot = get_docker_mem_snapshot;

diff --git a/plugins/in_docker/cgroup_v2.c b/plugins/in_docker/cgroup_v2.c
@@ -458,7 +458,7 @@ static mem_snapshot *get_docker_mem_snapshot(struct flb_docker *ctx, char *id)
 int in_docker_set_cgroup_api_v2(struct cgroup_api *api)
 {
     api->cgroup_version = 2;
-    api->get_active_docker_ids = get_active_dockers;
+    api->get_active_container_ids = get_active_dockers;
     api->get_container_name = get_container_name;
     api->get_cpu_snapshot = get_docker_cpu_snapshot;
     api->get_mem_snapshot = get_docker_mem_snapshot;

diff --git a/plugins/in_docker/docker.c b/plugins/in_docker/docker.c
@@ -482,7 +482,7 @@ static int cb_docker_collect(struct flb_input_instance *ins,
     (void) config;
 
     /* Get current active dockers. */
-    active = ctx->cgroup_api.get_active_docker_ids(ctx);
+    active = ctx->cgroup_api.get_active_container_ids(ctx);
 
     filtered = apply_filters(ctx, active);
     if (!filtered) {

diff --git a/plugins/in_docker/docker.h b/plugins/in_docker/docker.h
@@ -89,7 +89,7 @@ struct flb_docker;
 
 struct cgroup_api {
     int cgroup_version;
-    struct mk_list* (*get_active_docker_ids) ();
+    struct mk_list* (*get_active_container_ids) ();
     char*           (*get_container_name) (struct flb_docker *, char *);
     cpu_snapshot*   (*get_cpu_snapshot)   (struct flb_docker *, char *);
     mem_snapshot*   (*get_mem_snapshot)   (struct flb_docker *, char *);

diff --git a/plugins/out_stackdriver/stackdriver.h b/plugins/out_stackdriver/stackdriver.h
@@ -82,7 +82,7 @@
 #define STDOUT "stdout"
 #define STDERR "stderr"
 
-#define DEFAULT_TAG_REGEX "(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\\.log$"
+#define DEFAULT_TAG_REGEX "(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<container_id>[a-z0-9]{64})\\.log$"
 
 /* Metrics */
 #ifdef FLB_HAVE_METRICS

diff --git a/scripts/rate_limit.lua b/scripts/rate_limit.lua
@@ -11,11 +11,11 @@
 
 local counter = {}
 local time = 0
-local group_key = "docker_id" -- Used to group logs. Groups are rate limited independently.
+local group_key = "container_id" -- Used to group logs. Groups are rate limited independently.
 local group_bucket_period_s = 60 -- This is the period of of time in seconds over which group_bucket_limit applies.
 local group_bucket_limit = 6000 -- Maximum number logs allowed per groups over the period of group_bucket_period_s.
 
--- with above values, each and every containers running on the kubernetes will have a limit of 6000 logs for every 60 seconds since contianers have unique kubernetes.docker_id value
+-- with above values, each and every containers running on the kubernetes will have a limit of 6000 logs for every 60 seconds since contianers have unique kubernetes.container_id value
 
 local function get_current_time(timestamp)
     return math.floor(timestamp / group_bucket_period_s)

diff --git a/tests/internal/avro.c b/tests/internal/avro.c
@@ -146,9 +146,9 @@ void test_unpack_to_avro()
 void test_parse_reordered_schema()
 {
     // test same schema but different order of fields
-    const char *ts1 = "{\"name\":\"qavrov2_record\",\"type\":\"record\",\"fields\":[{\"name\":\"log\",\"type\":\"string\"},{\"name\":\"capture\",\"type\":\"string\"},{\"name\":\"kubernetes\",\"type\":{\"name\":\"krec\",\"type\":\"record\",\"fields\":[{\"name\":\"pod_name\",\"type\":\"string\"},{\"name\":\"namespace_name\",\"type\":\"string\"},{\"name\":\"pod_id\",\"type\":\"string\"},{\"name\":\"labels\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"annotations\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"host\",\"type\":\"string\"},{\"name\":\"container_name\",\"type\":\"string\"},{\"name\":\"docker_id\",\"type\":\"string\"},{\"name\":\"container_hash\",\"type\":\"string\"},{\"name\":\"container_image\",\"type\":\"string\"}]}}]}";
-    const char *ts2 = "{\"name\":\"qavrov2_record\",\"type\":\"record\",\"fields\":[{\"name\":\"capture\",\"type\":\"string\"},{\"name\":\"log\",\"type\":\"string\"},{\"name\":\"kubernetes\",\"type\":{\"name\":\"krec\",\"type\":\"record\",\"fields\":[{\"name\":\"namespace_name\",\"type\":\"string\"},{\"name\":\"pod_name\",\"type\":\"string\"},{\"name\":\"pod_id\",\"type\":\"string\"},{\"name\":\"annotations\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"labels\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"host\",\"type\":\"string\"},{\"name\":\"container_name\",\"type\":\"string\"},{\"name\":\"docker_id\",\"type\":\"string\"},{\"name\":\"container_hash\",\"type\":\"string\"},{\"name\":\"container_image\",\"type\":\"string\"}]}}]}";
-    const char *ts3 = "{\"name\":\"qavrov2_record\",\"type\":\"record\",\"fields\":[{\"name\":\"newnovalue\",\"type\":\"string\"},{\"name\":\"capture\",\"type\":\"string\"},{\"name\":\"log\",\"type\":\"string\"},{\"name\":\"kubernetes\",\"type\":{\"name\":\"krec\",\"type\":\"record\",\"fields\":[{\"name\":\"namespace_name\",\"type\":\"string\"},{\"name\":\"pod_name\",\"type\":\"string\"},{\"name\":\"pod_id\",\"type\":\"string\"},{\"name\":\"annotations\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"labels\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"host\",\"type\":\"string\"},{\"name\":\"container_name\",\"type\":\"string\"},{\"name\":\"docker_id\",\"type\":\"string\"},{\"name\":\"container_hash\",\"type\":\"string\"},{\"name\":\"container_image\",\"type\":\"string\"}]}}]}";
+    const char *ts1 = "{\"name\":\"qavrov2_record\",\"type\":\"record\",\"fields\":[{\"name\":\"log\",\"type\":\"string\"},{\"name\":\"capture\",\"type\":\"string\"},{\"name\":\"kubernetes\",\"type\":{\"name\":\"krec\",\"type\":\"record\",\"fields\":[{\"name\":\"pod_name\",\"type\":\"string\"},{\"name\":\"namespace_name\",\"type\":\"string\"},{\"name\":\"pod_id\",\"type\":\"string\"},{\"name\":\"labels\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"annotations\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"host\",\"type\":\"string\"},{\"name\":\"container_name\",\"type\":\"string\"},{\"name\":\"container_id\",\"type\":\"string\"},{\"name\":\"container_hash\",\"type\":\"string\"},{\"name\":\"container_image\",\"type\":\"string\"}]}}]}";
+    const char *ts2 = "{\"name\":\"qavrov2_record\",\"type\":\"record\",\"fields\":[{\"name\":\"capture\",\"type\":\"string\"},{\"name\":\"log\",\"type\":\"string\"},{\"name\":\"kubernetes\",\"type\":{\"name\":\"krec\",\"type\":\"record\",\"fields\":[{\"name\":\"namespace_name\",\"type\":\"string\"},{\"name\":\"pod_name\",\"type\":\"string\"},{\"name\":\"pod_id\",\"type\":\"string\"},{\"name\":\"annotations\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"labels\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"host\",\"type\":\"string\"},{\"name\":\"container_name\",\"type\":\"string\"},{\"name\":\"container_id\",\"type\":\"string\"},{\"name\":\"container_hash\",\"type\":\"string\"},{\"name\":\"container_image\",\"type\":\"string\"}]}}]}";
+    const char *ts3 = "{\"name\":\"qavrov2_record\",\"type\":\"record\",\"fields\":[{\"name\":\"newnovalue\",\"type\":\"string\"},{\"name\":\"capture\",\"type\":\"string\"},{\"name\":\"log\",\"type\":\"string\"},{\"name\":\"kubernetes\",\"type\":{\"name\":\"krec\",\"type\":\"record\",\"fields\":[{\"name\":\"namespace_name\",\"type\":\"string\"},{\"name\":\"pod_name\",\"type\":\"string\"},{\"name\":\"pod_id\",\"type\":\"string\"},{\"name\":\"annotations\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"labels\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"host\",\"type\":\"string\"},{\"name\":\"container_name\",\"type\":\"string\"},{\"name\":\"container_id\",\"type\":\"string\"},{\"name\":\"container_hash\",\"type\":\"string\"},{\"name\":\"container_image\",\"type\":\"string\"}]}}]}";
 
     const char *schemas[] = {ts1, ts2, ts3, ts2, ts1, NULL};
 

diff --git a/tests/internal/data/avro/live-sample.json b/tests/internal/data/avro/live-sample.json
@@ -18,7 +18,7 @@
       },
       "host": "wedddd.dkdk.qqqq.com",
       "container_name": "tb1",
-      "docker_id": "50bfc67d-cd3c-410d-9369-8bda8f33b1c7",
+      "container_id": "50bfc67d-cd3c-410d-9369-8bda8f33b1c7",
       "container_hash": "qqqq.corp.qqqq.com/ai/centos/tf1.15.0-py3.7-horovod@sha256:68b6885f6d1d3fd87ce425a2b2aa687440b9578740d60996912a816ae67be85e",
       "container_image": "qqqq.corp.qqqq.com/ai/centos/tf1.15.0-py3.7-horovod:1.2"
     }

diff --git a/tests/internal/data/avro/multiline.json b/tests/internal/data/avro/multiline.json
@@ -19,7 +19,7 @@
       },
       "host": "wedddd.dkdk.qqqq.com",
       "container_name": "tb1",
-      "docker_id": "50bfc67d-cd3c-410d-9369-8bda8f33b1c7",
+      "container_id": "50bfc67d-cd3c-410d-9369-8bda8f33b1c7",
       "container_hash": "qqqq.corp.qqqq.com/ai/centos/tf1.15.0-py3.7-horovod@sha256:68b6885f6d1d3fd87ce425a2b2aa687440b9578740d60996912a816ae67be85e",
       "container_image": "qqqq.corp.qqqq.com/ai/centos/tf1.15.0-py3.7-horovod:1.2"
     }