From 40ecbd346b52749e18d5c368eac78fa795570697 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 11 Feb 2021 08:21:03 -0500 Subject: [PATCH] OTEL Collector/Multitenant Example (#497) * Added basic otel example Signed-off-by: Joe Elliott * Added auth Signed-off-by: Joe Elliott * updated authed-tenant string for clarity Signed-off-by: Joe Elliott * Added multitenancy guide Signed-off-by: Joe Elliott --- docs/tempo/website/guides/_index.md | 3 +- docs/tempo/website/guides/multitenancy.md | 51 +++++++++++++++++ .../docker-compose.multitenant.yaml | 55 +++++++++++++++++++ .../docker-compose/etc/otel-collector.yaml | 15 +++++ .../example-data/datasources/datasource.yml | 15 +++++ 5 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 docs/tempo/website/guides/multitenancy.md create mode 100644 example/docker-compose/docker-compose.multitenant.yaml create mode 100644 example/docker-compose/etc/otel-collector.yaml diff --git a/docs/tempo/website/guides/_index.md b/docs/tempo/website/guides/_index.md index a25ced7e07b..b2428b4662e 100644 --- a/docs/tempo/website/guides/_index.md +++ b/docs/tempo/website/guides/_index.md @@ -6,4 +6,5 @@ weight: 400 Because Tempo is a trace id only lookup it relies on integrations for trace discovery. Common methods of discovery are through logs and exemplars. [The examples](https://github.com/grafana/tempo/tree/master/example) are also a good place to see how some of these discovery flows work. - [Loki Derived Fields](loki-derived-fields/) -- [Pushing Spans with HTTP](pushing-spans-with-http/) \ No newline at end of file +- [Pushing Spans with HTTP](pushing-spans-with-http/) +- [Multitenancy](multitenancy/) \ No newline at end of file diff --git a/docs/tempo/website/guides/multitenancy.md b/docs/tempo/website/guides/multitenancy.md new file mode 100644 index 00000000000..8fcb1aff7a8 --- /dev/null +++ b/docs/tempo/website/guides/multitenancy.md @@ -0,0 +1,51 @@ +--- +title: Multitenancy +--- + +Tempo is a multitenant distributed tracing backend. It supports multitenancy through the use +of a header: `X-Scope-OrgID`. This guide details how to setup or disable multitenancy. + +## Multitenancy + +If you're interested in setting up multitenancy, please consult the [multitenant example](https://github.com/grafana/tempo/tree/master/example/docker-compose/docker-compose.multitenant.yaml) +in the repo. This example uses the following settings to achieve multitenancy in Tempo: + +- Configure the OTEL Collector to attach the X-Scope-OrgID header on push: +``` +exporters: + otlp: + headers: + x-scope-orgid: foo-bar-baz +``` +- Configure the Tempo datasource in Grafana to pass the tenant as a bearer token. Yes, this is weird. It works b/c it is the only header that Jaeger can be configured to pass to its GRPC plugin. +``` +- name: Tempo-Multitenant + jsonData: + httpHeaderName1: 'Authorization' + secureJsonData: + httpHeaderValue1: 'Bearer foo-bar-baz' +``` +- Configure Jaeger Query to pass the bearer token to its backend. +``` +--query.bearer-token-propagation=true +``` + +## Important Notes + +- Multitenancy on ingestion is currently [only working](https://github.com/grafana/tempo/issues/495) with GPRC and this may never change. It is strongly recommended to use the OpenTelemetry Collector to support multitenancy as described above. +- The way the read path is configured is temporary and should be much more straightforward once the [tempo-query dependency is removed](https://github.com/grafana/tempo/issues/382). + +## Disabling Multitenancy +Most Tempo installations will be single tenant. If this is desired simply set the following config +value on all Tempo components: +``` +auth_enabled: false +``` + +or from the command line: +``` +--auth.enabled=false +``` + +This option will force all Tempo components to ignore the `X-Scope-OrgID` header and use the hardcoded +value of `single-tenant`. diff --git a/example/docker-compose/docker-compose.multitenant.yaml b/example/docker-compose/docker-compose.multitenant.yaml new file mode 100644 index 00000000000..dc6401d9c39 --- /dev/null +++ b/example/docker-compose/docker-compose.multitenant.yaml @@ -0,0 +1,55 @@ +version: "3.4" +services: + # Generate fake traces... + synthetic-load-generator: + image: omnition/synthetic-load-generator:1.0.25 + volumes: + - ./etc/load-generator.json:/etc/load-generator.json + environment: + - TOPOLOGY_FILE=/etc/load-generator.json + - JAEGER_COLLECTOR_URL=http://otel-collector:14268 + + # And put them in an OTEL collector pipeline... + otel-collector: + image: otel/opentelemetry-collector:0.16.0 + volumes: + - ./etc/otel-collector.yaml:/etc/otel-collector.yaml + command: + - --config=/etc/otel-collector.yaml + + # To eventually offload to Tempo... + tempo: + image: grafana/tempo:0.5.0 + command: ["--target=all", "--storage.trace.backend=local", "--storage.trace.local.path=/var/tempo"] + ports: + - 8081:80 + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/api/prom/push' + + tempo-query: + image: grafana/tempo-query:0.5.0 + command: + - --query.bearer-token-propagation=true # required to pass auth to the grpc plugin + environment: + - BACKEND=tempo:80 + volumes: + - ./etc/tempo-query.yaml:/etc/tempo-query.yaml + ports: + - "16686:16686" # jaeger-ui + depends_on: + - tempo + + grafana: + image: grafana/grafana:7.3.0-beta1 + volumes: + - ./example-data/datasources:/etc/grafana/provisioning/datasources + - ./example-data/dashboards-provisioning:/etc/grafana/provisioning/dashboards + - ../../operations/tempo-mixin/out:/var/lib/grafana/dashboards + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + ports: + - "3000:3000" \ No newline at end of file diff --git a/example/docker-compose/etc/otel-collector.yaml b/example/docker-compose/etc/otel-collector.yaml new file mode 100644 index 00000000000..d4618460ee1 --- /dev/null +++ b/example/docker-compose/etc/otel-collector.yaml @@ -0,0 +1,15 @@ +receivers: + jaeger: + protocols: + thrift_http: +exporters: + otlp: + endpoint: tempo:55680 + insecure: true + headers: + x-scope-orgid: foo-bar-baz +service: + pipelines: + traces: + receivers: [jaeger] + exporters: [otlp] \ No newline at end of file diff --git a/example/docker-compose/example-data/datasources/datasource.yml b/example/docker-compose/example-data/datasources/datasource.yml index 538ced01171..1c0713d7638 100644 --- a/example/docker-compose/example-data/datasources/datasource.yml +++ b/example/docker-compose/example-data/datasources/datasource.yml @@ -25,6 +25,21 @@ datasources: editable: false apiVersion: 1 uid: tempo +- name: Tempo-Multitenant + type: tempo + access: proxy + orgId: 1 + url: http://tempo-query:16686 + basicAuth: false + isDefault: false + version: 1 + editable: false + apiVersion: 1 + uid: tempo-authed + jsonData: + httpHeaderName1: 'Authorization' + secureJsonData: + httpHeaderValue1: 'Bearer foo-bar-baz' - name: Loki type: loki access: proxy