Modify crabserver process exporter conf in cmsweb prometheus

kubernetes/cmsweb/monitoring/prometheus/services/prometheus.yaml

@@ -77,10 +77,6 @@ scrape_configs:
  scrape_timeout: 110s
  static_configs:
  - targets: ['quota-exporter.http.svc.cluster.local:18000']
- - job_name: 'crabserver-processes'
- metrics_path: '/crabserver/metrics'
- static_configs:
- - targets: ['crabserver.crab.svc.cluster.local:8270']
 # - job_name: "dbsmigration-service"
 # static_configs:
 # - targets: ["dbsmigration.dbs.svc.cluster.local:18251", "dbsmigration.dbs.svc.cluster.local:19251"]
@@ -168,15 +164,12 @@ scrape_configs:
 # env: "k8s"
 
  - job_name: "kubernetes-apiservers"
-
  kubernetes_sd_configs:
  - role: endpoints
  scheme: https
-
  tls_config:
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
  relabel_configs:
  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
  action: keep
@@ -206,16 +199,12 @@ scrape_configs:
 # target_label: kubernetes_pod_name
 
  - job_name: "kubernetes-nodes"
-
  scheme: https
-
  tls_config:
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
  kubernetes_sd_configs:
  - role: node
-
  relabel_configs:
  - action: labelmap
  regex: __meta_kubernetes_node_label_(.+)
@@ -228,10 +217,8 @@ scrape_configs:
 
 
  - job_name: "kubernetes-pods"
-
  kubernetes_sd_configs:
  - role: pod
-
  # for more information about prometheus relabeling see
  # https://blog.freshtracks.io/prometheus-relabel-rules-and-the-action-parameter-39c71959354a
  relabel_configs:
@@ -263,6 +250,45 @@ scrape_configs:
  action: replace
  target_label: apod
 
+ # CRAB process exporter
+ - job_name: "kubernetes-pods-crabserver-special"
+ # Special metrics endpoint
+ metrics_path: '/crabserver/metrics'
+ kubernetes_sd_configs:
+ - role: pod
+ relabel_configs:
+ # Scrape only crab ns
+ - source_labels: [ __meta_kubernetes_namespace ]
+ action: keep
+ regex: crab
+ # Scrape only "8270" port name, only it exposes '/crabserver/metrics' endpoint
+ - source_labels: [ __meta_kubernetes_pod_container_port_number ]
+ action: keep
+ regex: 8270
+ - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_scrape ]
+ action: keep
+ regex: true
+ - source_labels: [ __address__, __meta_kubernetes_pod_container_port_number ]
+ action: replace
+ regex: ([^:]+)(?::\d+)?;(\d+)
+ replacement: $1:$2
+ target_label: __address__
+ - source_labels: [ __meta_kubernetes_pod_node_name ]
+ action: replace
+ target_label: host
+ - action: labeldrop
+ regex: __meta_kubernetes_pod_label_pod_(.+)
+ - source_labels: [ __meta_kubernetes_namespace ]
+ action: replace
+ target_label: ns
+ - action: labeldrop
+ regex: __meta_kubernetes_namespace(.+)
+ - action: labelmap
+ regex: __meta_kubernetes_pod_label_(.+)
+ - source_labels: [ __meta_kubernetes_pod_name ]
+ action: replace
+ target_label: apod
+
  - job_name: kubernetes-node-exporter
  tls_config:
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -283,16 +309,12 @@ scrape_configs:
  target_label: instance 
 
  - job_name: "kubernetes-cadvisor"
-
  scheme: https
-
  tls_config:
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
  kubernetes_sd_configs:
  - role: node
-
  relabel_configs:
  - action: labelmap
  regex: __meta_kubernetes_node_label_(.+)