HPCC-29546 Grafana/loki logaccess plugin

- Provides Grafana/loki curl based logaccess plugin - Updates helm/managed/loggin/loki-stack/README - Provides mechanism to create grafana-hpcc logaccess secret - Adds encodeCSV jstring logic Signed-off-by: Rodrigo Pastrana <[email protected]>
hpcc-systems · Jun 26, 2024 · 96d641a · 96d641a
1 parent 29ad0a3
commit 96d641a
Show file tree

Hide file tree

Showing 13 changed files with 1,236 additions and 24 deletions.
diff --git a/helm/managed/logging/loki-stack/README.md b/helm/managed/logging/loki-stack/README.md
@@ -78,4 +78,57 @@ The default Loki-Stack chart will not declare permanent storage and therefore lo
 loki:
   persistence:
     enabled: true
-```
+```
+
+## Configure HPCC logAccess
+The logAccess feature allows HPCC to query and package relevant logs for various features such as ZAP report, WorkUnit helper logs, ECLWatch log viewer, etc.
+
+### Provide target Grafana/Loki access information 
+
+HPCC logAccess requires access to the Grafana username/password. Those values must be provided via a secure secret object.
+
+The secret is expected to be in the 'esp' category, and be named 'grafana-logaccess'. The following key-value pairs are required (key names must be spelled exactly as shown here)
+
+    grafana-username - This should contain the Grafana username
+    grafana-password - This should contain the Grafana password
+
+The included 'create-grafana-logaccess-secret.sh' helper can be used to create the necessary secret.
+
+Example scripted secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys):
+
+  create-grana-logaccess-secret.sh -d HPCC-Platform/helm/managed//logging/loki-stack/secrets-templates/ -n hpcc
+
+Otherwise, users can create the secret manually.
+
+Example manual secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys):
+
+  kubectl create secret generic grafana-logaccess --from-file=HPCC-Platform/helm/managed//logging/loki-stack/secrets-templates/ -n hpcc
+
+### Configure HPCC logAccess
+
+The target HPCC deployment should be directed to use the desired Grafana endpoint with the Loki datasource, and the newly created secret by providing appropriate logAccess values (such as ./grafana-hpcc-logaccess.yaml).
+
+Example use:
+
+  helm install myhpcc hpcc/hpcc -f HPCC-Platform/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml
+
+####
+
+The grafana hpcc logaccess values should provide Grafana connection information, such as the host, and port; the Loki datasource where the logs recide; the k8s namespace under which the logs were created; and the hpcc compnent log format (table|json|xml)
+
+Example use:
+  global:
+    logAccess:
+      name: "Grafana/loki stack log access"
+      type: "GrafanaCurl"
+      connection:
+        protocol: "http"
+        host: "myloki4hpcclogs-grafana.default.svc.cluster.local"
+        port: 3000
+      datasource:
+        id: "1"
+        name: "Loki"
+      namespace:
+        name: "hpcc"
+      logFormat:
+        type: "json"
diff --git a/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh b/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+WORK_DIR=$(dirname $0)
+source ${WORK_DIR}/env-loganalytics
+
+k8scommand="kubectl"
+secretname="grafana-logaccess"
+secretsdir="${WORK_DIR}/secrets-templates"
+namespace="default"
+
+usage()
+{
+    echo "Creates necessary k8s secret used by HPCC's logAccess to access Loki data source through Grafana"
+    echo "> create-grafana-logaccess-secret.sh [Options]"
+    echo ""
+    echo "Options:"
+    echo "-d       Specifies directory containing required secret values in self named files."
+    echo "         Defaults to <workingdir>/<${secretssubdir}>"
+    echo "-h       Print Usage message"
+    echo "-n       Specifies namespace for secret"
+    echo ""
+    echo "Requires directory containing secret values in dedicated files."
+    echo "Defaults to  ${secretssubdir} if not specified via -d option."
+    echo ""
+    echo "Expected directory structure:"
+    echo "${secretsdir}/"
+    echo "   grafana-password  - Should contain Grafana user name"
+    echo "   grafana-username  - Should contain Grafana password"
+}
+
+while [ "$#" -gt 0 ]; do
+  arg=$1
+  case "${arg}" in
+      -h) 
+         usage
+         exit
+         ;;
+      -d) shift
+         secretsdir=$1
+         ;;
+      -n) shift
+         namespace=$1
+         ;;
+    esac
+  shift
+done
+
+echo "Creating '${namespace}/${secretname}' secret."
+
+command -v ${k8scommand} >/dev/null 2>&1 || { echo >&2 "Aborting - '${k8scommand}' not found!"; exit 1; }
+
+errormessage=$(${k8scommand} get secret ${secretname} -n ${namespace} 2>&1)
+if [[ $? -eq 0 ]]
+then
+  echo "WARNING: Target secret '${namespace}/${secretname}' already exists! Delete it and re-run if secret update desired."
+  echo "${errormessage}"
+  exit 1
+fi
+
+errormessage=$(${k8scommand} create secret generic ${secretname} --from-file=${secretsdir} -n ${namespace} )
+if [[ $? -ne 0 ]]
+then
+  echo "Error creating: Target secret '${namespace}/${secretname}'!"
+  echo >&2
+  usage
+  exit 1
+else
+  echo "Target secret '${namespace}/${secretname}' successfully created!"
+  ${k8scommand} get secret ${secretname} -n ${namespace}
+fi
diff --git a/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml b/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml
@@ -0,0 +1,43 @@
+# Configures HPCC logAccess to target grafana/loki
+global:
+  logAccess:
+    name: "Grafana/loki stack log access"
+    type: "GrafanaCurl"
+    connection:
+      protocol: "http"
+      host: "myloki4hpcclogs-grafana.default.svc.cluster.local"
+      port: 3000
+    datasource:
+      id: "1"
+      name: "Loki"
+    namespace:
+      name: "hpcc"
+    logFormat:
+      type: "json"
+    logMaps:
+    - type: "global"
+      searchColumn: "log"
+      columnMode: "DEFAULT"
+    - type: "components"
+      storeName: "stream"
+      searchColumn: "component"
+      columnMode: "MIN"
+      columnType: "string"
+    - type: "timestamp"
+      storeName: "values"
+      searchColumn: "time"
+      columnMode: "ALL"
+      columnType: "datetime"
+    - type: "pod"
+      storeName: "stream"
+      searchColumn: "pod"
+      columnMode: "ALL"
+      columnType: "string"
+secrets:
+  esp:
+    grafana-logaccess: "grafana-logaccess"
+vaults:
+  esp:
+  - name: my-grafana-logaccess-vault
+    url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/esp/${secret}
+    kind: kv-v2
diff --git a/helm/managed/logging/loki-stack/secrets-templates/grafana-password b/helm/managed/logging/loki-stack/secrets-templates/grafana-password
@@ -0,0 +1 @@
+<grafana access password goes here>
diff --git a/helm/managed/logging/loki-stack/secrets-templates/grafana-username b/helm/managed/logging/loki-stack/secrets-templates/grafana-username
@@ -0,0 +1 @@
+admin
diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp
@@ -3213,32 +3213,21 @@ IRemoteLogAccess *queryRemoteLogAccessor()
                 {
                     const char * simulatedGlobalYaml = R"!!(global:
   logAccess:
-    name: "Azure LogAnalytics LogAccess"
-    type: "AzureLogAnalyticsCurl"
+    name: "Grafana/loki stack log access"
+    type: "GrafanaCurl"
     connection:
       #workspaceID: "ef060646-ef24-48a5-b88c-b1f3fbe40271"
-      workspaceID: "XYZ"      #ID of the Azure LogAnalytics workspace to query logs from
+      #workspaceID: "XYZ"      #ID of the Azure LogAnalytics workspace to query logs from
       #tenantID: "ABC"         #The Tenant ID, required for KQL API access
-      clientID: "DEF"         #ID of Azure Active Directory registered application with api.loganalytics.io access
-    logMaps:
-    - type: "global"
-      storeName: "ContainerLog"
-      searchColumn: "LogEntry"
-      timeStampColumn: "hpcc_log_timestamp"
-    - type: "workunits"
-      storeName: "ContainerLog"
-      searchColumn: "hpcc_log_jobid"
-    - type: "components"
-      searchColumn: "ContainerID"
-    - type: "audience"
-      searchColumn: "hpcc_log_audience"
-    - type: "class"
-      searchColumn: "hpcc_log_class"
-    - type: "instance"
-      storeName: "ContainerInventory"
-      searchColumn: "Name"
-    - type: "host"
-      searchColumn: "Computer"
+      #clientID: "DEF"         #ID of Azure Active Directory registered application with api.loganalytics.io access
+      protocol: "http"
+      host: "localhost"
+      port: "3000"
+    datasource:
+      id: "1"
+      name: "Loki"
+    namespace:
+      name: "hpcc"
                     )!!";
                     Owned<IPropertyTree> testTree = createPTreeFromYAMLString(simulatedGlobalYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr);
                     logAccessPluginConfig.setown(testTree->getPropTree("global/logAccess"));

diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp
@@ -2367,6 +2367,49 @@ StringBuffer &encodeJSON(StringBuffer &s, const char *value)
     return encodeJSON(s, strlen(value), value);
 }
 
+inline StringBuffer &encodeCSVChar(StringBuffer &s, const char *&ch, unsigned &remaining)
+{
+    byte next = *ch;
+    switch (next)
+    {
+        case '\"':
+            s.append("\"");
+            s.append(next);
+            break;
+        //case '\n':
+        //    s.append("\\n");
+        //    break;
+        //case '\r':
+        //    s.append("\\r");
+        //    break;
+        default:
+            s.append(next);
+            break;
+    }
+    ch++;
+    remaining--;
+    return s;
+}
+
+StringBuffer &encodeCSV(StringBuffer &s, unsigned size, const char *value)
+{
+    if (!value)
+        return s;
+    s.ensureCapacity(size); // Minimum size that will be written
+    s.append("\"");
+    while (size)
+        encodeCSVChar(s, value, size);
+    s.append("\"");
+    return s;
+}
+
+StringBuffer &encodeCSV(StringBuffer &s, const char *value)
+{
+    if (!value)
+        return s;
+    return encodeCSV(s, strlen(value), value);
+}
+
 bool checkUnicodeLiteral(char const * str, unsigned length, unsigned & ep, StringBuffer & msg)
 {
     unsigned i;

diff --git a/system/jlib/jstring.hpp b/system/jlib/jstring.hpp
@@ -479,6 +479,8 @@ inline StringBuffer &delimitJSON(StringBuffer &s, bool addNewline=false, bool es
     return s;
 }
 
+jlib_decl StringBuffer &encodeCSV(StringBuffer &s, const char *value);
+
 jlib_decl StringBuffer &encodeJSON(StringBuffer &s, const char *value);
 jlib_decl StringBuffer &encodeJSON(StringBuffer &s, unsigned len, const char *value);
 

diff --git a/system/logaccess/CMakeLists.txt b/system/logaccess/CMakeLists.txt
@@ -19,4 +19,5 @@ IF(NOT CLIENTTOOLS_ONLY)
     HPCC_ADD_SUBDIRECTORY (ElasticStack)
   ENDIF()
   HPCC_ADD_SUBDIRECTORY (Azure)
+  HPCC_ADD_SUBDIRECTORY (Grafana)
 ENDIF()
diff --git a/system/logaccess/Grafana/CMakeLists.txt b/system/logaccess/Grafana/CMakeLists.txt
@@ -0,0 +1,19 @@
+###############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2022 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+IF(NOT CLIENTTOOLS_ONLY)
+    HPCC_ADD_SUBDIRECTORY (CurlClient)
+ENDIF()
diff --git a/system/logaccess/Grafana/CurlClient/CMakeLists.txt b/system/logaccess/Grafana/CurlClient/CMakeLists.txt
@@ -0,0 +1,45 @@
+###############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2022 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+project(GrafanaCurllogaccess)
+
+# Required installed libraries
+find_package(CURL REQUIRED)
+
+set(srcs
+  ${CMAKE_CURRENT_SOURCE_DIR}/GrafanaCurlClient.cpp
+)
+
+include_directories(
+  ${HPCC_SOURCE_DIR}/system/include
+  ${HPCC_SOURCE_DIR}/system/jlib
+  ${CURL_INCLUDE_DIR}
+)
+
+add_definitions(-DGRAFANA_CURL_LOGACCESS_EXPORTS)
+
+HPCC_ADD_LIBRARY(${PROJECT_NAME} SHARED ${srcs})
+
+target_link_libraries(${PROJECT_NAME}
+   PRIVATE jlib
+   PRIVATE ${CURL_LIBRARIES}
+)
+
+install(TARGETS ${PROJECT_NAME}
+  RUNTIME DESTINATION ${EXEC_DIR}
+  LIBRARY DESTINATION ${LIB_DIR}
+  CALC_DEPS
+)