Skip to content

Commit

Permalink
Update Loki
Browse files Browse the repository at this point in the history
Signed-off-by: Matej Feder <[email protected]>
  • Loading branch information
matofeder committed Jun 19, 2024
1 parent db77862 commit c37e639
Show file tree
Hide file tree
Showing 11 changed files with 493 additions and 273 deletions.
1 change: 1 addition & 0 deletions .github/kind_cluster_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: worker
- role: worker
- role: worker
- role: control-plane
123 changes: 62 additions & 61 deletions .github/workflows/ci-scheduled.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,64 +13,65 @@
# limitations under the License.
#

name: Test Chart with k8s versions

on:
schedule:
# Run every 2 weeks
- cron: "0 0 1,15 * *"

jobs:
test-chart:
name: test-chart
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
include:
- k8s-version: v1.19.7
stack-version: v1.3
- k8s-version: v1.20.2
stack-version: v1.3
- k8s-version: v1.20.7
stack-version: v1.4
- k8s-version: v1.21.2
stack-version: v1.4
- k8s-version: v1.22.1
stack-version: v1.4
- k8s-version: v1.23.13
stack-version: v2.4
- k8s-version: v1.25.9
stack-version: v2.5

steps:
- name: Checkout
uses: actions/checkout@v2

- name: Fetch history
run: git fetch --prune --unshallow

- name: Complete semVer with highest patch version
id: highest-patch
run: |
tag=$(git tag -l --sort=-v:refname | grep -F ${{ matrix.stack-version }}. | head -1)
[[ -z $tag ]] && (echo "Patch version for ${{ matrix.stack-version }} not found!" && exit 1)
echo "::set-output name=version::${tag//v}"
- name: Download correct chart
run: |
echo ${{ steps.semver.outputs.version }}
wget https://dnationcloud.github.io/helm-hub/dnation-kubernetes-monitoring-stack-${{ steps.highest-patch.outputs.version }}.tgz
tar -xvf dnation-kubernetes-monitoring-stack-${{ steps.highest-patch.outputs.version }}.tgz
- name: Create kind ${{ matrix.k8s-version }} cluster
uses: helm/[email protected]
with:
node_image: kindest/node:${{ matrix.k8s-version }}

- name: Set up chart-testing
uses: helm/[email protected]

- name: Run chart-testing (install)
run: |
ct install --config .github/ct-scheduled.yaml
# TODO @mfeder: Enable and update below outdated pipeline
#name: Test Chart with k8s versions
#
#on:
# schedule:
# # Run every 2 weeks
# - cron: "0 0 1,15 * *"
#
#jobs:
# test-chart:
# name: test-chart
# runs-on: ubuntu-20.04
# strategy:
# fail-fast: false
# matrix:
# include:
# - k8s-version: v1.19.7
# stack-version: v1.3
# - k8s-version: v1.20.2
# stack-version: v1.3
# - k8s-version: v1.20.7
# stack-version: v1.4
# - k8s-version: v1.21.2
# stack-version: v1.4
# - k8s-version: v1.22.1
# stack-version: v1.4
# - k8s-version: v1.23.13
# stack-version: v2.4
# - k8s-version: v1.25.9
# stack-version: v2.5
#
# steps:
# - name: Checkout
# uses: actions/checkout@v2
#
# - name: Fetch history
# run: git fetch --prune --unshallow
#
# - name: Complete semVer with highest patch version
# id: highest-patch
# run: |
# tag=$(git tag -l --sort=-v:refname | grep -F ${{ matrix.stack-version }}. | head -1)
# [[ -z $tag ]] && (echo "Patch version for ${{ matrix.stack-version }} not found!" && exit 1)
# echo "::set-output name=version::${tag//v}"
#
# - name: Download correct chart
# run: |
# echo ${{ steps.semver.outputs.version }}
# wget https://dnationcloud.github.io/helm-hub/dnation-kubernetes-monitoring-stack-${{ steps.highest-patch.outputs.version }}.tgz
# tar -xvf dnation-kubernetes-monitoring-stack-${{ steps.highest-patch.outputs.version }}.tgz
#
# - name: Create kind ${{ matrix.k8s-version }} cluster
# uses: helm/[email protected]
# with:
# node_image: kindest/node:${{ matrix.k8s-version }}
#
# - name: Set up chart-testing
# uses: helm/[email protected]
#
# - name: Run chart-testing (install)
# run: |
# ct install --config .github/ct-scheduled.yaml
20 changes: 10 additions & 10 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,47 +23,47 @@ on:

jobs:
lint-chart:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Fetch history
run: git fetch --prune --unshallow

- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.0
uses: helm/chart-testing-action@v2.6.1

- name: Run chart-testing (install)
run: |
ct lint --config .github/ct.yaml
install-chart:
name: install-chart
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
needs:
- lint-chart
strategy:
matrix:
k8s:
- v1.23.13
- v1.24.13
- v1.25.9
- v1.28.9
- v1.29.4
- v1.30.0
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Fetch history
run: git fetch --prune --unshallow

- name: Create kind ${{ matrix.k8s }} cluster
uses: helm/kind-action@v1.7.0
uses: helm/kind-action@v1.10.0
with:
node_image: kindest/node:${{ matrix.k8s }}
config: .github/kind_cluster_config.yaml

- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.0
uses: helm/chart-testing-action@v2.6.1

- name: Run chart-testing (install)
run: |
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ on:

jobs:
release:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v4

- name: Fetch history
run: git fetch --prune --unshallow
Expand All @@ -45,7 +45,7 @@ jobs:
git config --global user.email "[email protected]"
- name: Set up Python
uses: actions/setup-python@v2
uses: actions/setup-python@v5
with:
python-version: 3.6

Expand All @@ -60,7 +60,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.CICD_TOKEN }}

- name: Tag branch
uses: anothrNick/github-tag-action@1.39.0
uses: anothrNick/github-tag-action@1.67.0
env:
GITHUB_TOKEN: ${{ secrets.CICD_TOKEN }}
CUSTOM_TAG: 'v${{ steps.version.outputs.chart }}'
Expand Down
56 changes: 24 additions & 32 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,59 +11,56 @@ An umbrella helm chart for [dNation Kubernetes Monitoring](https://github.com/dN

* [dnation-kubernetes-monitoring](https://github.com/dNationCloud/kubernetes-monitoring)
* [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)
* [thanos](https://github.com/bitnami/charts/tree/master/bitnami/thanos) (to support multicluster monitoring)
* [loki](https://github.com/grafana/helm-charts/tree/main/charts/loki)
* [loki-distributed](https://github.com/grafana/helm-charts/tree/main/charts/loki-distributed)
* [thanos](https://github.com/bitnami/charts/tree/master/bitnami/thanos)
* [loki](https://github.com/grafana/loki/tree/main/production/helm/loki)
* [promtail](https://github.com/grafana/helm-charts/tree/main/charts/promtail)
* [ssl-exporter](https://github.com/dNationCloud/ssl-exporter) # optional
* [prometheus-blackbox-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-blackbox-exporter) # optional
* [loki-distributed](https://github.com/grafana/helm-charts/tree/main/charts/loki-distributed) # deprecated, see [loki](#loki)


# Installation
Prerequisites
* [Helm3](https://helm.sh/)
* For production environment we recommend (based on our experience) a kubernetes cluster with at least 2 worker nodes and 4 GiB RAM per node or more.
* For production environment we recommend (based on our experience) a kubernetes cluster with at least 3 worker nodes and 4 GiB RAM per node or more.

dNation Kubernetes Monitoring Stack umbrella chart is hosted in the [dNation helm repository](https://artifacthub.io/packages/search?repo=dnationcloud).
By default, dNation Kubernetes Monitoring Stack installs Grafana with dNation dashboards, Prometheus with Thanos and Loki in simple scalable mode.

dNation Kubernetes Monitoring Stack umbrella chart is hosted in the [dNation helm repository](https://artifacthub.io/packages/search?repo=dnationcloud). By default, dNation Kubernetes Monitoring Stack installs Prometheus with Thanos sidecar and Thanos Query.
Thanos components are tuned according to [SCS Thanos tuning](https://github.com/SovereignCloudStack/k8s-observability/blob/main/docs/thanos-tuning.md).
For more details check [Multicluster monitoring support](#multicluster-monitoring-support) section.
```bash
# Add dNation helm repository
helm repo add dnationcloud https://dnationcloud.github.io/helm-hub/
helm repo update

# Install dNation Kubernetes Monitoring Stack (Loki in monolithic mode)
# Install dNation Kubernetes Monitoring Stack
helm install monitoring dnationcloud/dnation-kubernetes-monitoring-stack

# Install dNation Kubernetes Monitoring Stack (Loki in distributed mode with s3-compatible storage)
helm install monitoring dnationcloud/dnation-kubernetes-monitoring-stack \
-f https://raw.githubusercontent.com/dNationCloud/kubernetes-monitoring-stack/main/chart/values-loki-distributed.yaml \
--set loki-distributed.loki.storageConfig.aws.s3="<s3 path-style URL with access and secret keys>"
```

Search for `Monitoring` dashboard in the `dNation` directory. The fun starts here :).
If you want to set the `Monitoring` dashboard as a home dashboard follow [here](https://grafana.com/docs/grafana/latest/administration/change-home-dashboard/#set-the-default-dashboard-through-preferences).
Installation notes:
- Thanos components are tuned according to [SCS Thanos tuning](https://github.com/SovereignCloudStack/k8s-observability/blob/main/docs/thanos-tuning.md)
- If you're experiencing issues please read the [documentation](https://dnationcloud.github.io/kubernetes-monitoring/docs/documentation) and [FAQ](https://dnationcloud.github.io/kubernetes-monitoring/helpers/FAQ/)

For `multi-cluster centralized logging` install monitoring on your workload cluster without Loki, set `loki.enabled: false` in [values.yaml](chart/values.yaml) and also configure `promtail.config.lokiAddress` to send logs to your Loki instance. On your central cluster install it in classic way with `loki.enable: true`.
Search for `Infrastructure services monitoring` dashboard in the `dNation` directory. The fun starts here :).
If you want to set the `Infrastructure services monitoring` dashboard as a home dashboard follow [here](https://grafana.com/docs/grafana/latest/administration/change-home-dashboard/#set-the-default-dashboard-through-preferences).

If you're experiencing issues please read the [documentation](https://dnationcloud.github.io/kubernetes-monitoring/docs/documentation) and [FAQ](https://dnationcloud.github.io/kubernetes-monitoring/helpers/FAQ/).
# Loki
## loki-distributed

# Kubernetes support (tested)
This chart is deprecated and replaced by [loki](https://github.com/grafana/loki/tree/main/production/helm/loki) helm chart.
Loki helm chart is the only helm chart you should use for loki helm deployment. It supports loki deployment in monolithic, scalable
and even [distributed mode](https://grafana.com/docs/loki/next/setup/install/helm/install-microservices/).

||dNation monitoring v1.3|dNation monitoring v1.4|dNation monitoring v2.0|dNation monitoring v2.3|dNation monitoring v2.5|
|-|-|-|-|-|-|
|Kubernetes v1.19||||||
|Kubernetes v1.20||||||
|Kubernetes v1.21||||||
|Kubernetes v1.22||||||
|Kubernetes v1.23||||||
|Kubernetes v1.24||||||
|Kubernetes v1.25||||||
We recommend use the loki helm chart for all fresh installations. If you already use loki-distributed helm chart, check
the migration [guide](https://grafana.com/docs/loki/latest/setup/migrate/migrate-from-distributed/).

# Multicluster monitoring support
This chart supports also setup of multicluster monitoring using Thanos. The deployment architecture follows "observer cluster/workload clusters" pattern, where there is one observer k8s cluster which provides centralized monitoring overview of multiple workload k8s clusters. Helm values files enabling the multicluster monitoring are located inside `multicluster-config/` directory. There are 2 files in total:

- `multicluster-config/observer-values.yaml` - contains config for installation of observer cluster
- `multicluster-config/workload-values.yaml` - contains config for installation of workload cluster(s)

For `multi-cluster centralized logging` install monitoring on your workload cluster without Loki, set `loki.enabled: false` in [values.yaml](chart/values.yaml) and also configure `promtail.config.lokiAddress` to send logs to your Loki instance. On your central cluster install it in classic way with `loki.enable: true`.

## Architecture

As mentioned earlier, we are using "observer cluster/workload clusters" pattern to implement multicluster monitoring. The full architecture can be seen on following diagram:
Expand Down Expand Up @@ -220,11 +217,6 @@ dnationcloud/dnation-kubernetes-monitoring-stack \
-f <custom-observer-values-sample.yaml>
```
# Openshift support
## Tested versions

||dNation monitoring v1.3|dNation monitoring v1.4|dNation monitoring v2.0|dNation monitoring v2.3|
|-|-|-|-|-|
|Openshift v4.7||||✓|
## Installation
To install the chart on an openshift cluster, use additional [values for openshift](/chart/values-openshift.yaml)
```shell
Expand Down
8 changes: 4 additions & 4 deletions chart/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@

apiVersion: v2
name: dnation-kubernetes-monitoring-stack
version: 3.5.0
version: 3.6.0
appVersion: 2.7.0 # dnation-kubernetes-monitoring
description: An umbrella helm chart for Kubernetes monitoring based on kube-prometheus-stack, thanos, loki, loki-distributed, promtail and dnation-kubernetes-monitoring.
description: An umbrella helm chart for Kubernetes monitoring based on kube-prometheus-stack, thanos, loki, promtail and dnation-kubernetes-monitoring.
keywords:
- dnation
- kubernetes
Expand All @@ -42,15 +42,15 @@ dependencies:
repository: https://charts.bitnami.com/bitnami
condition: thanos.enabled
- name: loki
version: "5.5.*"
version: "6.6.*"
repository: https://grafana.github.io/helm-charts
condition: loki.enabled
- name: loki-distributed
version: "0.68.*"
repository: https://grafana.github.io/helm-charts
condition: loki-distributed.enabled
- name: promtail
version: "6.0.*"
version: "6.16.*"
repository: https://grafana.github.io/helm-charts
condition: promtail.enabled
- name: dnation-kubernetes-monitoring
Expand Down
2 changes: 1 addition & 1 deletion chart/templates/NOTES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ If you're experiencing issues please read the project documentation and FAQ.

4. Login with the password from step 2 and the username: '{{ index .Values "kube-prometheus-stack" "grafana" "adminUser" }}'

5. Search for `Monitoring` dashboard in the `dNation` directory. The fun starts here :)
5. Search for `Infrastructure services monitoring` dashboard in the `dNation` directory. The fun starts here :)
6 changes: 6 additions & 0 deletions chart/values-loki-distributed.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@
# with loki-distributed.
# Declare variables to be passed into helm chart dependencies.

############################################## WARNING ###############################################################
##
## DEPRECATED VALUES
##
## The following values are deprecated and will be removed in a future version of the helm chart!
############################################## WARNING ##############################################################
## Configure additional grafana datasources
grafanaDatasourcesAsConfigMap:
cluster-logs:
Expand Down
Loading

0 comments on commit c37e639

Please sign in to comment.