Install Prometheus exporters along with the application (#291)

* test commit

* execute tests with terminal

* change snap, address nginx exporter race condition

* change snap, address nginx exporter race condition

* add layers

* remove debug terminal

* addressing comments

* create fixture

indico_rock/rockcraft.yaml

@@ -44,6 +44,9 @@ parts:
  - texlive-fonts-recommended
  - texlive-plain-generic
  - texlive-xetex
+ stage-snaps:
+ - celery-prometheus-exporter/latest/edge
+ - gtrkiller-statsd-prometheus-exporter/latest/edge
  source: plugins
  plugin: nil
  override-build: |

metadata.yaml

@@ -38,12 +38,6 @@ containers:
  resource: indico-nginx-image
  indico-celery:
  resource: indico-image
- celery-prometheus-exporter:
- resource: celery-prometheus-exporter-image
- nginx-prometheus-exporter:
- resource: nginx-prometheus-exporter-image
- statsd-prometheus-exporter:
- resource: statsd-prometheus-exporter-image
 
 resources:
  indico-image:
@@ -54,21 +48,6 @@ resources:
  type: oci-image
  description: OCI image for nginx Indico
  auto-fetch: true
- celery-prometheus-exporter-image:
- type: oci-image
- description: Prometheus exporter for celery
- auto-fetch: true
- upstream-source: danihodovic/celery-exporter:0.7.6
- nginx-prometheus-exporter-image:
- type: oci-image
- description: Prometheus exporter for nginx
- auto-fetch: true
- upstream-source: nginx/nginx-prometheus-exporter:0.11.0
- statsd-prometheus-exporter-image:
- type: oci-image
- description: Prometheus exporter for statsd
- auto-fetch: true
- upstream-source: prom/statsd-exporter:v0.22.8
 
 provides:
  grafana-dashboard:

nginx_rock/rockcraft.yaml

@@ -26,6 +26,8 @@ parts:
  nginx:
  stage-packages:
  - nginx
+ stage-snaps:
+ - gtrkiller-nginx-prometheus-exporter/latest/edge
  plugin: nil
  override-build: |
  craftctl default

requirements.txt

@@ -1,2 +1,2 @@
-ops>=2.0.0
+ops<=2.3.0
 ops-lib-pgsql

src/charm.py

@@ -70,15 +70,6 @@ def __init__(self, *args):
  self.framework.observe(self.on.indico_pebble_ready, self._on_pebble_ready)
  self.framework.observe(self.on.indico_celery_pebble_ready, self._on_pebble_ready)
  self.framework.observe(self.on.indico_nginx_pebble_ready, self._on_pebble_ready)
- self.framework.observe(
- self.on.nginx_prometheus_exporter_pebble_ready, self._on_pebble_ready
- )
- self.framework.observe(
- self.on.statsd_prometheus_exporter_pebble_ready, self._on_pebble_ready
- )
- self.framework.observe(
- self.on.celery_prometheus_exporter_pebble_ready, self._on_pebble_ready
- )
  self.framework.observe(
  self.on.refresh_external_resources_action, self._refresh_external_resources_action
  )
@@ -271,7 +262,14 @@ def _config_pebble(self, container: Container) -> None:
  pebble_config = pebble_config_func(container)
  container.add_layer(container.name, pebble_config, combine=True)
  if container.name == "indico":
+ celery_config = self._get_celery_prometheus_exporter_pebble_config(container)
+ statsd_config = self._get_statsd_prometheus_exporter_pebble_config(container)
+ container.add_layer("celery", celery_config, combine=True)
+ container.add_layer("statsd", statsd_config, combine=True)
  self._download_customization_changes(container)
+ if container.name == "indico-nginx":
+ pebble_config = self._get_nginx_prometheus_exporter_pebble_config(container)
+ container.add_layer("nginx", pebble_config, combine=True)
  self.unit.status = MaintenanceStatus(f"Starting {container.name} container")
  container.pebble.replan_services()
  if self._are_pebble_instances_ready():
@@ -308,7 +306,7 @@ def _get_indico_pebble_config(self, container: Container) -> Dict:
  "override": "replace",
  "level": "ready",
  "tcp": {"port": 8081},
- }
+ },
  },
  }
 
@@ -375,12 +373,16 @@ def _get_indico_nginx_pebble_config(self, _) -> Dict:
  },
  }
 
- def _get_celery_prometheus_exporter_pebble_config(self, _) -> Dict:
+ def _get_celery_prometheus_exporter_pebble_config(self, container) -> Dict:
  """Generate pebble config for the celery-prometheus-exporter container.
 
+ Args:
+ container: Celery container that has the target configuration.
+
  Returns:
  The pebble configuration for the container.
  """
+ indico_env_config = self._get_indico_env_config(container)
  return {
  "summary": "Celery prometheus exporter",
  "description": "Prometheus exporter for celery",
@@ -389,12 +391,11 @@ def _get_celery_prometheus_exporter_pebble_config(self, _) -> Dict:
  "override": "replace",
  "summary": "Celery Exporter",
  "command": (
- "python"
- " /app/cli.py"
+ "celery-exporter"
  f" --broker-url={self._get_celery_backend()}"
  " --retry-interval=5"
  ),
- "environment": {"CE_ACCEPT_CONTENT": "json,pickle"},
+ "environment": indico_env_config,
  "startup": "enabled",
  },
  },
@@ -417,7 +418,7 @@ def _get_nginx_prometheus_exporter_pebble_config(self, _) -> Dict:
  "summary": "Nginx prometheus exporter",
  "description": "Prometheus exporter for nginx",
  "services": {
- "nginx-exporter": {
+ "nginx-prometheus-exporter": {
  "override": "replace",
  "summary": "Nginx Exporter",
  "command": (

tests/integration/conftest.py

@@ -5,7 +5,6 @@
 
 import asyncio
 from pathlib import Path
-from typing import Dict
 
 import pytest_asyncio
 import yaml
@@ -44,23 +43,6 @@ def app_name_fixture(metadata):
  yield metadata["name"]
 
 
-@fixture(scope="module", name="prometheus_exporter_images")
-def prometheus_exporter_images_fixture(metadata):
- """Provides Prometheus exporter images from the metadata."""
- prometheus_exporter_images = {
- "nginx-prometheus-exporter-image": metadata["resources"][
- "nginx-prometheus-exporter-image"
- ]["upstream-source"],
- "statsd-prometheus-exporter-image": metadata["resources"][
- "statsd-prometheus-exporter-image"
- ]["upstream-source"],
- "celery-prometheus-exporter-image": metadata["resources"][
- "celery-prometheus-exporter-image"
- ]["upstream-source"],
- }
- yield prometheus_exporter_images
-
-
 @fixture(scope="module")
 def requests_timeout():
  """Provides a global default timeout for HTTP requests"""
@@ -72,7 +54,6 @@ async def app(
  ops_test: OpsTest,
  app_name: str,
  pytestconfig: Config,
- prometheus_exporter_images: Dict[str, str],
 ):
  """Indico charm used for integration testing.
 
@@ -91,7 +72,6 @@ async def app(
  "indico-image": pytestconfig.getoption("--indico-image"),
  "indico-nginx-image": pytestconfig.getoption("--indico-nginx-image"),
  }
- resources.update(prometheus_exporter_images)
  charm = pytestconfig.getoption("--charm-file")
  application = await ops_test.model.deploy(
  f"./{charm}",

tests/integration/test_charm.py

@@ -61,42 +61,14 @@ async def test_indico_is_up(ops_test: OpsTest, app: Application):
  assert response.status_code == 200
 
 
-@pytest.mark.asyncio
-@pytest.mark.abort_on_fail
-async def test_prom_exporters_are_up(app: Application):
- """
- arrange: given charm in its initial state
- act: when the metrics endpoints are scraped
- assert: the response is 200 (HTTP OK)
- """
- # Application actually does have units
- indico_unit = app.units[0] # type: ignore
- prometheus_targets = [
- f"localhost:{NGINX_PROMEXP_PORT}",
- f"localhost:{STATSD_PROMEXP_PORT}",
- f"localhost:{CELERY_PROMEXP_PORT}",
- ]
- # Send request to /metrics for each target and check the response
- for target in prometheus_targets:
- cmd = f"curl -m 10 http://{target}/metrics"
- action = await indico_unit.run(cmd, timeout=15)
- # Change this if upgrading Juju lib version to >= 3
- # See https://github.com/juju/python-libjuju/issues/707#issuecomment-1212296289
- result = action.data
- code = result["results"].get("Code")
- stdout = result["results"].get("Stdout")
- stderr = result["results"].get("Stderr")
- assert code == "0", f"{cmd} failed ({code}): {stderr or stdout}"
-
-
 @pytest.mark.asyncio
 @pytest.mark.abort_on_fail
 async def test_health_checks(app: Application):
  """Runs health checks for each container.
 
  Assume that the charm has already been built and is running.
  """
- container_list = ["indico", "indico-nginx", "indico-celery"]
+ container_list = ["indico-celery", "indico-nginx", "indico"]
  # Application actually does have units
  indico_unit = app.units[0] # type: ignore
  for container in container_list:
@@ -112,7 +84,7 @@ async def test_health_checks(app: Application):
  # When executing the checks, `0/3` means there are 0 errors of 3.
  # Each check has it's own `0/3`, so we will count `n` times,
  # where `n` is the number of checks for that container.
- assert stdout.count("0/3") == 1
+ assert stdout.count("0/3") == container_list.index(container) + 1
 
 
 @pytest.mark.abort_on_fail
@@ -149,6 +121,34 @@ async def add_admin(app: Application):
  assert f'Admin with email "{email_fail}" correctly created' in action2.results["output"]
 
 
+@pytest.mark.asyncio
+@pytest.mark.abort_on_fail
+async def test_prom_exporters_are_up(app: Application):
+ """
+ arrange: given charm in its initial state
+ act: when the metrics endpoints are scraped
+ assert: the response is 200 (HTTP OK)
+ """
+ # Application actually does have units
+ indico_unit = app.units[0] # type: ignore
+ prometheus_targets = [
+ f"localhost:{NGINX_PROMEXP_PORT}",
+ f"localhost:{STATSD_PROMEXP_PORT}",
+ f"localhost:{CELERY_PROMEXP_PORT}",
+ ]
+ # Send request to /metrics for each target and check the response
+ for target in prometheus_targets:
+ cmd = f"curl -m 10 http://{target}/metrics"
+ action = await indico_unit.run(cmd, timeout=15)
+ # Change this if upgrading Juju lib version to >= 3
+ # See https://github.com/juju/python-libjuju/issues/707#issuecomment-1212296289
+ result = action.data
+ code = result["results"].get("Code")
+ stdout = result["results"].get("Stdout")
+ stderr = result["results"].get("Stderr")
+ assert code == "0", f"{cmd} failed ({code}): {stderr or stdout}"
+
+
 @pytest.mark.asyncio
 @pytest.mark.abort_on_fail
 @pytest.mark.usefixtures("add_admin")

tests/unit/test_actions.py

@@ -29,17 +29,7 @@ def test_refresh_external_resources_when_customization_and_plugins_set(self, moc
  """
  mock_exec.return_value = MagicMock(wait_output=MagicMock(return_value=("", None)))
  self.harness.disable_hooks()
- self.set_up_all_relations()
- self.harness.set_leader(True)
-
- self.is_ready(
- [
- "nginx-prometheus-exporter",
- "indico",
- "indico-celery",
- "indico-nginx",
- ]
- )
+ self.set_relations_and_leader()
  self.harness.update_config(
  {
  "customization_sources_url": "https://example.com/custom",
@@ -70,19 +60,7 @@ def test_add_admin(self, mock_exec):
  """
  mock_exec.return_value = MagicMock(wait_output=MagicMock(return_value=("", None)))
 
- self.set_up_all_relations()
- self.harness.set_leader(True)
-
- self.is_ready(
- [
- "celery-prometheus-exporter",
- "statsd-prometheus-exporter",
- "nginx-prometheus-exporter",
- "indico",
- "indico-celery",
- "indico-nginx",
- ]
- )
+ self.set_relations_and_leader()
  self.harness.disable_hooks()
 
  container = self.harness.model.unit.get_container("indico")
@@ -177,19 +155,7 @@ def mock_exec_side_effect(*args, **kwargs): # pylint: disable=unused-argument
  wait_output=mock_wo,
  )
 
- self.set_up_all_relations()
- self.harness.set_leader(True)
-
- self.is_ready(
- [
- "celery-prometheus-exporter",
- "statsd-prometheus-exporter",
- "nginx-prometheus-exporter",
- "indico",
- "indico-celery",
- "indico-nginx",
- ]
- )
+ self.set_relations_and_leader()
  self.harness.disable_hooks()
 
  container = self.harness.model.unit.get_container("indico")

tests/unit/test_base.py

@@ -59,3 +59,15 @@ def is_ready(self, apps: List[str]):
  """
  for app_name in apps:
  self.harness.container_pebble_ready(app_name)
+
+ def set_relations_and_leader(self):
+ """Set Indico relations, the leader and check container readiness."""
+ self.set_up_all_relations()
+ self.harness.set_leader(True)
+ self.is_ready(
+ [
+ "indico",
+ "indico-celery",
+ "indico-nginx",
+ ]
+ )