From 02975e554e16bcb301fd6077b9bc166d7de3228c Mon Sep 17 00:00:00 2001 From: Molly Miller Date: Tue, 10 Sep 2024 11:38:00 +0200 Subject: [PATCH 1/5] loki: add basic role Simple configuration with static filesystem configuration. PL-132981 --- nixos/roles/default.nix | 1 + nixos/roles/loki.nix | 68 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 nixos/roles/loki.nix diff --git a/nixos/roles/default.nix b/nixos/roles/default.nix index f61f3e97c..19b261ad3 100644 --- a/nixos/roles/default.nix +++ b/nixos/roles/default.nix @@ -20,6 +20,7 @@ in { ./jitsi ./k3s ./lamp.nix + ./loki.nix ./mailout.nix ./mailserver.nix ./matomo.nix diff --git a/nixos/roles/loki.nix b/nixos/roles/loki.nix new file mode 100644 index 000000000..24c1c384f --- /dev/null +++ b/nixos/roles/loki.nix @@ -0,0 +1,68 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.flyingcircus.roles.loki; + fclib = config.fclib; +in +{ + options = with lib; { + flyingcircus.roles.loki = { + enable = mkEnableOption "Flying Circus Grafana Loki server"; + supportsContainers = fclib.mkEnableContainerSupport; + logRetentionPeriod = mkOption { + type = types.ints.unsigned; + default = 30; + description = "Global retention period for log data in days. Setting to zero disables automatic log expiry"; + }; + }; + }; + + config = lib.mkIf cfg.enable { + services.loki = { + enable = true; + configuration = { + # XXX loki only uses a single listen address. + server.http_listen_address = builtins.head fclib.network.srv.v4.addresses; + + auth_enabled = false; + + schema_config.configs = [{ + from = "2024-09-10"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + # default values in loki 3.1.1, here specified explicitly + index.prefix = ""; + index.path_prefix = "index/"; + index.period = "24h"; + }]; + + storage_config = { + # index file management + tsdb_shipper = { + active_index_directory = "/var/lib/loki/tsdb-shipper-index"; + cache_location = "/var/lib/loki/tsdb-shipper-cache"; + }; + # log data configuration + filesystem.directory = "/var/lib/loki/chunk-store"; + }; + + compactor = { + working_directory = "/var/lib/loki/compactor-workdir"; + retention_enabled = true; + delete_request_store = "filesystem"; + }; + + limits_config = { + retention_period = (builtins.toString cfg.logRetentionPeriod) + "d"; + }; + + # configuration stubs for multi-process management plane + common = { + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + }; + }; + }; +} From 18b49c9e4f903b6d2ea02f3f1f7eccc5800c8022 Mon Sep 17 00:00:00 2001 From: Molly Miller Date: Tue, 10 Sep 2024 15:03:34 +0200 Subject: [PATCH 2/5] promtail: add basic log shipping configuration This automatically enables promtail when a Loki server is present in the same resource group. Currently only a single Loki server is supported. PL-132981 --- nixos/platform/default.nix | 1 + nixos/platform/promtail.nix | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 nixos/platform/promtail.nix diff --git a/nixos/platform/default.nix b/nixos/platform/default.nix index b8f9a581e..74f077eb7 100644 --- a/nixos/platform/default.nix +++ b/nixos/platform/default.nix @@ -34,6 +34,7 @@ in { ./monitoring.nix ./network.nix ./packages.nix + ./promtail.nix ./shell.nix ./static.nix ./syslog.nix diff --git a/nixos/platform/promtail.nix b/nixos/platform/promtail.nix new file mode 100644 index 000000000..1ea8e30f0 --- /dev/null +++ b/nixos/platform/promtail.nix @@ -0,0 +1,51 @@ +{ lib, config, ... }: + +let + enc = config.flyingcircus.enc; + fclib = config.fclib; + + # XXX support multiple loki servers. the upstream docs note: "It is + # generally recommended to run multiple Promtail clients in parallel + # if you want to send to multiple remote Loki instances." + lokiServer = fclib.findOneService "loki-collector"; +in +{ + config = lib.mkIf (!builtins.isNull lokiServer) { + services.promtail = { + enable = true; + configuration = { + # don't expose the http and grpc api + server.disable = true; + + clients = [{ + url = "http://${lokiServer.address}:3100/loki/api/v1/push"; + }]; + + scrape_configs = [{ + job_name = "systemd-journal"; + journal = { + json = true; + # there are server side limits to how many labels loki + # will accept on log lines. consider them a scarce + # resource and use them sparingly. + labels = { + resource_group = enc.parameters.resource_group; + location = enc.parameters.location; + hostname = config.networking.hostName; + }; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "systemd_unit"; + } + { + source_labels = [ "__journal_syslog_identifier" ]; + target_label = "syslog_identifier"; + } + ]; + }]; + }; + }; + }; +} From ab59bc345de1a2e87c7fe40ef49cce36df7f893f Mon Sep 17 00:00:00 2001 From: Molly Miller Date: Tue, 10 Sep 2024 15:43:23 +0200 Subject: [PATCH 3/5] statshost: add loki data source to grafana configuration If the loki role is also enabled on the same host as statshost-master, then add a Loki data source to the Grafana configuration. PL-132981 --- nixos/roles/statshost/default.nix | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/nixos/roles/statshost/default.nix b/nixos/roles/statshost/default.nix index 2f0d980ac..3c6ca8a3a 100644 --- a/nixos/roles/statshost/default.nix +++ b/nixos/roles/statshost/default.nix @@ -18,6 +18,8 @@ let cfgProxyLocation = config.flyingcircus.roles.statshost-location-proxy; cfgProxyRG = config.flyingcircus.roles.statshost-relay; + cfgLokiRG = config.flyingcircus.roles.loki; + promFlags = [ "--storage.tsdb.retention.time ${toString cfgStats.prometheusRetention}d" ]; @@ -517,12 +519,30 @@ in isDefault: true ''; }; + + # support loki running on the same host as grafana in + # single-RG mode. + lokiDatasource = pkgs.writeTextFile { + name = "loki.yaml"; + text = '' + apiVersion: 1 + datasources: + - name: Loki + type: loki + access: proxy + orgId: 1 + url: http://${config.networking.hostName}:3100 + editable: false + isDefault: false + ''; + }; in '' rm -rf ${grafanaProvisioningPath} mkdir -p ${grafanaProvisioningPath}/dashboards ${grafanaProvisioningPath}/datasources ln -fs ${fcioDashboards} ${grafanaProvisioningPath}/dashboards/fcio.yaml ln -fs ${prometheusDatasource} ${grafanaProvisioningPath}/datasources/prometheus.yaml - + '' + optionalString (cfgStatsRG.enable && cfgLokiRG.enable) '' + ln -fs ${lokiDatasource} ${grafanaProvisioningPath}/datasources/loki.yaml ''; # Provide FC dashboards, and update them automatically. From c5afc77b710f690be81f623366ba82f693c70db4 Mon Sep 17 00:00:00 2001 From: Molly Miller Date: Tue, 17 Sep 2024 14:06:36 +0200 Subject: [PATCH 4/5] loki: use nginx as reverse proxy for access control Limit access to the Loki API from external machines to only the data ingestion and querying endpoints. Configure Grafana to communciate with the Loki API over localhost without restrictions. PL-132981 --- nixos/roles/loki.nix | 39 +++++++++++++++++++++++++++++-- nixos/roles/statshost/default.nix | 2 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/nixos/roles/loki.nix b/nixos/roles/loki.nix index 24c1c384f..11e99dccd 100644 --- a/nixos/roles/loki.nix +++ b/nixos/roles/loki.nix @@ -21,8 +21,7 @@ in services.loki = { enable = true; configuration = { - # XXX loki only uses a single listen address. - server.http_listen_address = builtins.head fclib.network.srv.v4.addresses; + server.http_listen_address = "127.0.0.1"; auth_enabled = false; @@ -64,5 +63,41 @@ in }; }; }; + + flyingcircus.services.nginx = { + enable = true; + virtualHosts."${config.networking.hostName}" = { + serverAliases = [ + (fclib.fqdn { vlan = "srv"; }) + "${config.networking.hostName}.${config.networking.domain}" + ]; + listen = builtins.map (addr: { inherit addr; port = 3100; }) + fclib.network.srv.dualstack.addressesQuoted; + locations = with builtins; with lib; let + proxyConfig = { proxyPass = "http://127.0.0.1:3100"; }; + in listToAttrs ( + [(nameValuePair "/" { extraConfig = "return 403;"; })] ++ + (map (path: nameValuePair path proxyConfig) [ + # https://grafana.com/docs/loki/latest/reference/loki-http-api/ + + # ingestion endpoints + "/loki/api/v1/push" + "/otlp/v1/logs" + + # query endpoints + "/loki/api/v1/query" + "/loki/api/v1/query_range" + "/loki/api/v1/labels" + "/loki/api/v1/label" + "/loki/api/v1/series" + "/loki/api/v1/index/stats" + "/loki/api/v1/index/volume" + "/loki/api/v1/index/volume_range" + "/loki/api/v1/patterns" + "/loki/api/v1/tail" + ]) + ); + }; + }; }; } diff --git a/nixos/roles/statshost/default.nix b/nixos/roles/statshost/default.nix index 3c6ca8a3a..000afcbbe 100644 --- a/nixos/roles/statshost/default.nix +++ b/nixos/roles/statshost/default.nix @@ -531,7 +531,7 @@ in type: loki access: proxy orgId: 1 - url: http://${config.networking.hostName}:3100 + url: http://localhost:3100 editable: false isDefault: false ''; From 936644b6343ee3554e5cb770c26389d8cc383634 Mon Sep 17 00:00:00 2001 From: Molly Miller Date: Wed, 11 Sep 2024 16:46:48 +0200 Subject: [PATCH 5/5] loki: add options for configuring S3 storage Provide a minimal set of options for providing configuration for an S3 endpoint used for storing log data, and for providing the required storage schedule entries in order to use the configured S3 endpoint. PL-132891 --- nixos/roles/loki.nix | 97 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/nixos/roles/loki.nix b/nixos/roles/loki.nix index 11e99dccd..e3b508a2b 100644 --- a/nixos/roles/loki.nix +++ b/nixos/roles/loki.nix @@ -3,21 +3,93 @@ let cfg = config.flyingcircus.roles.loki; fclib = config.fclib; + + storageScheduleSubmodule = with lib; with types; submodule { + options = { + startDate = mkOption { + type = strMatching "[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}"; + }; + backend = mkOption { type = enum [ "filesystem" "s3" ]; }; + schemaVersion = mkOption { type = ints.positive; default = 13; }; + }; + }; + + renderStorageSchema = opts: { + from = opts.startDate; + # always tsdb for index tables + store = "tsdb"; + schema = "v${toString opts.schemaVersion}"; + # default values in loki 3.1.1, here specified explicitly + index.prefix = ""; + index.path_prefix = "index/"; + index.period = "24h"; + + object_store = opts.backend; + }; in { options = with lib; { flyingcircus.roles.loki = { enable = mkEnableOption "Flying Circus Grafana Loki server"; supportsContainers = fclib.mkEnableContainerSupport; + logRetentionPeriod = mkOption { type = types.ints.unsigned; default = 30; description = "Global retention period for log data in days. Setting to zero disables automatic log expiry"; }; + + s3 = mkOption { + description = "Configure log storage in S3-compatible object store"; + default = {}; + type = types.submodule { + options = { + enable = mkEnableOption "store log data in S3"; + endpoint = mkOption { + description = "HTTP(S) endpoint of S3 storage server"; + type = types.str; + default = "http://rgw.local:7840"; + }; + bucketName = mkOption { + description = "S3 bucket name"; + type = types.str; + }; + credentialFile = mkOption { + description = "Path to file containing credentials used for authenticating to the S3 server (must be readable by the loki user)"; + type = types.path; + default = "/etc/local/loki/s3.cfg"; + }; + }; + }; + }; + + storageSchedule = mkOption { + description = "Log storage schedule configuration"; + default = {}; + type = with types; submodule { + options = { + default = mkOption { + visible = false; + type = listOf storageScheduleSubmodule; + default = [{ startDate = "2024-09-10"; backend = "filesystem"; }]; + }; + extra = mkOption { + description = "Additional entries to add to the log storage schedule"; + type = listOf storageScheduleSubmodule; + default = []; + defaultText = "[]"; + }; + }; + }; + }; }; }; config = lib.mkIf cfg.enable { + environment.etc."local/loki/README.txt".text = '' + This is a stub README for the Loki role. + ''; + services.loki = { enable = true; configuration = { @@ -25,16 +97,8 @@ in auth_enabled = false; - schema_config.configs = [{ - from = "2024-09-10"; - store = "tsdb"; - object_store = "filesystem"; - schema = "v13"; - # default values in loki 3.1.1, here specified explicitly - index.prefix = ""; - index.path_prefix = "index/"; - index.period = "24h"; - }]; + schema_config.configs = map renderStorageSchema + (cfg.storageSchedule.default ++ cfg.storageSchedule.extra); storage_config = { # index file management @@ -44,6 +108,13 @@ in }; # log data configuration filesystem.directory = "/var/lib/loki/chunk-store"; + } // lib.optionalAttrs (cfg.s3.enable) { + s3 = { + # authentication configured separately + endpoint = cfg.s3.endpoint; + bucketNames = cfg.s3.bucketNames; + s3forcepathstyle = true; + }; }; compactor = { @@ -53,7 +124,7 @@ in }; limits_config = { - retention_period = (builtins.toString cfg.logRetentionPeriod) + "d"; + retention_period = (toString cfg.logRetentionPeriod) + "d"; }; # configuration stubs for multi-process management plane @@ -64,6 +135,10 @@ in }; }; + systemd.services.loki.serviceConfig = lib.mkIf (cfg.s3.enable) { + Environment = "AWS_SHARED_CREDENTIALS_FILE=${cfg.s3.credentialFile}"; + }; + flyingcircus.services.nginx = { enable = true; virtualHosts."${config.networking.hostName}" = {