http2: response body closed

[cedric-mgx]

Hi, I'm using the grafana agent in flow mode on kubernetes and see this error popping all the time for different pods (few times per minutes), but when i check all the logs for those pods are there.

level=warn msg="tailer stopped; will retry" target=monitoring/pod-id:grafana component=loki.source.kubernetes.pods err="http2: response body closed"

The agent are installed with the following helm chart config:

fullnameOverride: grafana-agent

# Various agent settings.
agent:
  # -- Mode to run Grafana Agent in. Can be "flow" or "static".
  mode: 'flow'
  configMap:
    # -- Create a new ConfigMap for the config file.
    create: true
    # -- Content to assign to the new ConfigMap.  This is passed into `tpl` allowing for templating from values.
    content: |
      logging {
        level  = "warn"
        format = "logfmt"
      }

      // Endpoints
      prometheus.remote_write "mimir_endpoint" {
        external_labels = {
          cluster = env("CLUSTER_NAME"),
        }
        endpoint {
          url = "http://mimir-nginx.monitoring.svc/api/v1/push"

          queue_config {
            capacity             = 5000
            max_shards           = 100
            max_samples_per_send = 1000
            batch_send_deadline  = "1m0s"
          }

          metadata_config { }

          write_relabel_config {
            source_labels = ["__name__"]
            regex         = "(etcd_request_duration_seconds_bucket|etcd_request_duration_seconds_bucket|apiserver_response_sizes_bucket|prober_probe_duration_seconds_bucket|apiserver_storage_list_duration_seconds_bucket|apiserver_watch_events_sizes_bucket|container_memory_failures_total|container_blkio_device_usage_total)"
            action        = "drop"
          }
        }
      }

      loki.write "loki_endpoint" {
        endpoint {
          url = "http://loki-gateway.monitoring.svc/loki/api/v1/push"
        }
        external_labels = {
          cluster = "observability",
        }
      }

      // Sources
      discovery.kubernetes "pods" {
        role = "pod"
      }
          
      discovery.kubernetes "nodes" {
        role = "node"
      }

      discovery.kubernetes "services" {
        role = "service"
      }

      discovery.kubernetes "endpoints" {
        role = "endpoints"
      }

      discovery.kubernetes "endpointslices" {
        role = "endpointslice"
      }

      discovery.kubernetes "ingresses" {
        role = "ingress"
      }

      // Metrics
      prometheus.operator.servicemonitors {
        forward_to = [
          prometheus.remote_write.mimir_endpoint.receiver,
        ]

        clustering {
          enabled = true
        }
      }

      // kubelet
      discovery.relabel "metrics_kubelet" {
        targets = discovery.kubernetes.nodes.targets

        rule {
          action = "replace"
          target_label = "__address__"
          replacement = "kubernetes.default.svc.cluster.local:443"
        }

        rule {
          source_labels = ["__meta_kubernetes_node_name"]
          regex = "(.+)"
          action = "replace"
          replacement = "/api/v1/nodes/${1}/proxy/metrics"
          target_label = "__metrics_path__"
        }

        rule {
          source_labels = ["__name__"]
          regex = ":node_memory_MemAvailable_bytes:sum|alertmanager_alerts|alertmanager_alerts_invalid_total"
          action = "keep"
        }
      }

      prometheus.scrape "kubelet" {
        scheme = "https"
        tls_config {
          server_name = "kubernetes"
          ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
          insecure_skip_verify = false
        }
        bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
        targets = discovery.relabel.metrics_kubelet.output
        scrape_interval = "60s"
        forward_to = [prometheus.remote_write.mimir_endpoint.receiver]
      }

      // cadvisor
      prometheus.scrape "cadvisor" {
        scheme = "https"
        tls_config {
          server_name = "kubernetes"
          ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
          insecure_skip_verify = false
        }
        bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
        targets = discovery.relabel.metrics_cadvisor.output
        scrape_interval = "60s"
        forward_to = [prometheus.remote_write.mimir_endpoint.receiver]
      }


      discovery.relabel "metrics_cadvisor" {
        targets = discovery.kubernetes.nodes.targets

        rule {
          action = "replace"
          target_label = "__address__"
          replacement = "kubernetes.default.svc.cluster.local:443"
        }

        rule {
          source_labels = ["__meta_kubernetes_node_name"]
          regex = "(.+)"
          action = "replace"
          replacement = "/api/v1/nodes/${1}/proxy/metrics/cadvisor"
          target_label = "__metrics_path__"
        }
      }

      mimir.rules.kubernetes "prometheus_rules" {
        address = "http://mimir-nginx.monitoring.svc"
      }

      // Logging
      // https://grafana.com/docs/agent/latest/flow/reference/components/discovery.kubernetes/
      discovery.relabel "logs" {
        targets = discovery.kubernetes.pods.targets

        rule {
          source_labels = ["__meta_kubernetes_namespace"]
          target_label  = "namespace"
        }

        rule {
          source_labels = ["__meta_kubernetes_pod_container_name"]
          target_label  = "container"
        }

        rule {
          source_labels = ["__meta_kubernetes_pod_name"]
          target_label  = "pod"
        }

        rule {
          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
          target_label  = "app"
        }

        rule {
          source_labels = ["__meta_kubernetes_pod_node_name"]
          target_label  = "node_name"
        }
      }

      loki.source.kubernetes "pods" {
        targets    = discovery.relabel.logs.output
        forward_to = [loki.write.loki_endpoint.receiver]

        clustering {
          enabled = true
        }
      }

      loki.source.kubernetes_events "events" {
        log_format = "logfmt"
        forward_to = [loki.write.loki_endpoint.receiver]
      }

    # -- Name of existing ConfigMap to use. Used when create is false.
    name: null
    # -- Key in ConfigMap to get config from.
    key: null

  clustering:
    # -- Deploy agents in a cluster to allow for load distribution. Only
    # applies when agent.mode=flow.
    enabled: true

  # -- Path to where Grafana Agent stores data (for example, the Write-Ahead Log).
  # By default, data is lost between reboots.
  storagePath: /tmp/agent

  # -- Address to listen for traffic on. 0.0.0.0 exposes the UI to other
  # containers.
  listenAddr: 0.0.0.0

  # -- Port to listen for traffic on.
  listenPort: 80

  # --  Base path where the UI is exposed.
  uiPathPrefix: /

  # -- Enables sending Grafana Labs anonymous usage stats to help improve Grafana
  # Agent.
  enableReporting: true

  # -- Extra environment variables to pass to the agent container.
  extraEnv:
  - name: CLUSTER_NAME
    value: observability

  # -- Maps all the keys on a ConfigMap or Secret as environment variables. https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#envfromsource-v1-core
  envFrom: []

  # -- Extra args to pass to `agent run`: https://grafana.com/docs/agent/latest/flow/reference/cli/run/
  extraArgs: []

  # -- Extra ports to expose on the Agent
  extraPorts: []
  # - name: "faro"
  #   port: 12347
  #   targetPort: 12347
  #   protocol: "TCP"

  # -- Security context to apply to the Grafana Agent container.
  securityContext: {}

  # -- Resource requests and limits to apply to the Grafana Agent container.
  resources: {}

serviceAccount:
  # -- Whether to create a service account for the Grafana Agent deployment.
  create: true
  # -- Annotations to add to the created service account.
  annotations: {}

controller:
  type: 'daemonset'

Can anyone tell me anything about those logs? Also it seems to be happening 90% of the times with the same applications, like the grafana or ansible pods for example

[rfratto]

Hi, I'm going to move this discussion to an issue. loki.source.kubernetes will regularly refresh connections to the Kubernetes API due to a bug in many versions of Kubernetes where connections can go silent after a log file rolls.

What's likely happening here is that we're improperly logging a warning when we refresh connections instead of logging nothing.

[rfratto]

Moved to #6035.