Merge pull request #18800 from rpastrana/HPCC-29546-GrafanaLoki-LogAc…

…cess

HPCC-29546 Grafana/loki logaccess plugin

Reviewed-by: Gavin Halliday <[email protected]>
Merged-by: Gavin Halliday <[email protected]>

helm/managed/logging/loki-stack/README.md

@@ -78,4 +78,65 @@ The default Loki-Stack chart will not declare permanent storage and therefore lo
 loki:
   persistence:
     enabled: true
-```
+```
+
+## Configure HPCC logAccess
+The logAccess feature allows HPCC to query and package relevant logs for various features such as ZAP report, WorkUnit helper logs, ECLWatch log viewer, etc.
+
+### Provide target Grafana/Loki access information 
+
+HPCC logAccess requires access to the Grafana username/password. Those values must be provided via a secure secret object.
+
+The secret is expected to be in the 'esp' category, and be named 'grafana-logaccess'. The following key-value pairs are required (key names must be spelled exactly as shown here)
+
+    username - This should contain the Grafana username
+    password - This should contain the Grafana password
+
+The included 'create-grafana-logaccess-secret.sh' helper can be used to create the necessary secret.
+
+Example scripted secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys):
+
+```
+  create-grafana-logaccess-secret.sh -d HPCC-Platform/helm/managed/logging/loki-stack/secrets-templates/ -n hpcc
+```
+
+Otherwise, users can create the secret manually.
+
+Example manual secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys):
+
+```
+  kubectl create secret generic grafana-logaccess --from-file=HPCC-Platform/helm/managed//logging/loki-stack/secrets-templates/ -n hpcc
+```
+
+### Configure HPCC logAccess
+
+The target HPCC deployment should be directed to use the desired Grafana endpoint with the Loki datasource, and the newly created secret by providing appropriate logAccess values (such as ./grafana-hpcc-logaccess.yaml).
+
+Example use:
+
+```
+  helm install myhpcc hpcc/hpcc -f HPCC-Platform/helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml
+```
+
+####
+
+The grafana hpcc logaccess values should provide Grafana connection information, such as the host, and port; the Loki datasource where the logs reside; the k8s namespace under which the logs were created (non-default namespace highly recommended); and the hpcc component log format (table|json|xml)
+
+```
+Example use:
+  global:
+    logAccess:
+      name: "Grafana/loki stack log access"
+      type: "GrafanaCurl"
+      connection:
+        protocol: "http"
+        host: "myloki4hpcclogs-grafana.default.svc.cluster.local"
+        port: 3000
+      datasource:
+        id: "1"
+        name: "Loki"
+      namespace:
+        name: "hpcc"
+      logFormat:
+        type: "json"
+```

helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+WORK_DIR=$(dirname $0)
+source ${WORK_DIR}/env-loganalytics
+
+k8scommand="kubectl"
+secretname="grafana-logaccess"
+secretsdir="${WORK_DIR}/secrets-templates"
+namespace="default"
+
+usage()
+{
+    echo "Creates necessary k8s secret used by HPCC's logAccess to access Loki data source through Grafana"
+    echo "> create-grafana-logaccess-secret.sh [Options]"
+    echo ""
+    echo "Options:"
+    echo "-d       Specifies directory containing required secret values in self named files."
+    echo "         Defaults to <workingdir>/<${secretssubdir}>"
+    echo "-h       Print Usage message"
+    echo "-n       Specifies namespace for secret"
+    echo ""
+    echo "Requires directory containing secret values in dedicated files."
+    echo "Defaults to  ${secretssubdir} if not specified via -d option."
+    echo ""
+    echo "Expected directory structure:"
+    echo "${secretsdir}/"
+    echo "   password  - Should contain Grafana user name"
+    echo "   username  - Should contain Grafana password"
+}
+
+while [ "$#" -gt 0 ]; do
+  arg=$1
+  case "${arg}" in
+      -h) 
+         usage
+         exit
+         ;;
+      -d) shift
+         secretsdir=$1
+         ;;
+      -n) shift
+         namespace=$1
+         ;;
+    esac
+  shift
+done
+
+echo "Creating '${namespace}/${secretname}' secret."
+
+command -v ${k8scommand} >/dev/null 2>&1 || { echo >&2 "Aborting - '${k8scommand}' not found!"; exit 1; }
+
+errormessage=$(${k8scommand} get secret ${secretname} -n ${namespace} 2>&1)
+if [[ $? -eq 0 ]]
+then
+  echo "WARNING: Target secret '${namespace}/${secretname}' already exists! Delete it and re-run if secret update desired."
+  echo "${errormessage}"
+  exit 1
+fi
+
+errormessage=$(${k8scommand} create secret generic ${secretname} --from-file=${secretsdir} -n ${namespace} )
+if [[ $? -ne 0 ]]
+then
+  echo "Error creating: Target secret '${namespace}/${secretname}'!"
+  echo >&2
+  usage
+  exit 1
+else
+  echo "Target secret '${namespace}/${secretname}' successfully created!"
+  ${k8scommand} get secret ${secretname} -n ${namespace}
+fi

helm/managed/logging/loki-stack/grafana-hpcc-logaccess.yaml

@@ -0,0 +1,43 @@
+# Configures HPCC logAccess to target grafana/loki
+global:
+  logAccess:
+    name: "Grafana/loki stack log access"
+    type: "GrafanaCurl"
+    connection:
+      protocol: "http"
+      host: "myloki4hpcclogs-grafana.default.svc.cluster.local"
+      port: 3000
+    datasource:
+      id: "1"
+      name: "Loki"
+    namespace:
+      name: "hpcc"
+    logFormat:
+      type: "json"
+    logMaps:
+    - type: "global"
+      searchColumn: "log"
+      columnMode: "DEFAULT"
+    - type: "components"
+      storeName: "stream"
+      searchColumn: "component"
+      columnMode: "MIN"
+      columnType: "string"
+    - type: "timestamp"
+      storeName: "values"
+      searchColumn: "time"
+      columnMode: "ALL"
+      columnType: "datetime"
+    - type: "pod"
+      storeName: "stream"
+      searchColumn: "pod"
+      columnMode: "ALL"
+      columnType: "string"
+secrets:
+  esp:
+    grafana-logaccess: "grafana-logaccess"
+vaults:
+  esp:
+  - name: my-grafana-logaccess-vault
+    url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/esp/${secret}
+    kind: kv-v2

helm/managed/logging/loki-stack/secrets-templates/password

@@ -0,0 +1 @@
+<Grafana access password goes here>

helm/managed/logging/loki-stack/secrets-templates/username

@@ -0,0 +1 @@
+admin

system/jlib/jlog.cpp

@@ -3213,32 +3213,21 @@ IRemoteLogAccess *queryRemoteLogAccessor()
                 {
                     const char * simulatedGlobalYaml = R"!!(global:
   logAccess:
-    name: "Azure LogAnalytics LogAccess"
-    type: "AzureLogAnalyticsCurl"
+    name: "Grafana/loki stack log access"
+    type: "GrafanaCurl"
     connection:
       #workspaceID: "ef060646-ef24-48a5-b88c-b1f3fbe40271"
-      workspaceID: "XYZ"      #ID of the Azure LogAnalytics workspace to query logs from
+      #workspaceID: "XYZ"      #ID of the Azure LogAnalytics workspace to query logs from
       #tenantID: "ABC"         #The Tenant ID, required for KQL API access
-      clientID: "DEF"         #ID of Azure Active Directory registered application with api.loganalytics.io access
-    logMaps:
-    - type: "global"
-      storeName: "ContainerLog"
-      searchColumn: "LogEntry"
-      timeStampColumn: "hpcc_log_timestamp"
-    - type: "workunits"
-      storeName: "ContainerLog"
-      searchColumn: "hpcc_log_jobid"
-    - type: "components"
-      searchColumn: "ContainerID"
-    - type: "audience"
-      searchColumn: "hpcc_log_audience"
-    - type: "class"
-      searchColumn: "hpcc_log_class"
-    - type: "instance"
-      storeName: "ContainerInventory"
-      searchColumn: "Name"
-    - type: "host"
-      searchColumn: "Computer"
+      #clientID: "DEF"         #ID of Azure Active Directory registered application with api.loganalytics.io access
+      protocol: "http"
+      host: "localhost"
+      port: "3000"
+    datasource:
+      id: "1"
+      name: "Loki"
+    namespace:
+      name: "hpcc"
                     )!!";
                     Owned<IPropertyTree> testTree = createPTreeFromYAMLString(simulatedGlobalYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr);
                     logAccessPluginConfig.setown(testTree->getPropTree("global/logAccess"));

system/jlib/jstring.cpp

@@ -2367,6 +2367,42 @@ StringBuffer &encodeJSON(StringBuffer &s, const char *value)
     return encodeJSON(s, strlen(value), value);
 }
 
+inline StringBuffer & encodeCSVChar(StringBuffer & encodedCSV, char ch)
+{
+    byte next = ch;
+    switch (next)
+    {
+        case '\"':
+            encodedCSV.append("\"");
+            encodedCSV.append(next);
+            break;
+        //Any other character that needs to be escaped?
+        default:
+            encodedCSV.append(next);
+            break;
+    }
+    return encodedCSV;
+}
+
+StringBuffer & encodeCSVColumn(StringBuffer & encodedCSV, unsigned size, const char *rawCSVCol)
+{
+    if (!rawCSVCol)
+        return encodedCSV;
+    encodedCSV.ensureCapacity(size+2); // Minimum size that will be written
+    encodedCSV.append("\"");
+    for (size32_t i = 0; i < size; i++)
+        encodeCSVChar(encodedCSV, rawCSVCol[i]);
+    encodedCSV.append("\"");
+    return encodedCSV;
+}
+
+StringBuffer & encodeCSVColumn(StringBuffer & encodedCSV, const char *rawCSVCol)
+{
+    if (!rawCSVCol)
+        return encodedCSV;
+    return encodeCSVColumn(encodedCSV, strlen(rawCSVCol), rawCSVCol);
+}
+
 bool checkUnicodeLiteral(char const * str, unsigned length, unsigned & ep, StringBuffer & msg)
 {
     unsigned i;

system/jlib/jstring.hpp

@@ -479,6 +479,11 @@ inline StringBuffer &delimitJSON(StringBuffer &s, bool addNewline=false, bool es
     return s;
 }
 
+/*
+* Encodes a CSV column, not an entire CSV record
+*/
+jlib_decl StringBuffer &encodeCSVColumn(StringBuffer &s, const char *value);
+
 jlib_decl StringBuffer &encodeJSON(StringBuffer &s, const char *value);
 jlib_decl StringBuffer &encodeJSON(StringBuffer &s, unsigned len, const char *value);
 

system/logaccess/CMakeLists.txt

@@ -19,4 +19,5 @@ IF(NOT CLIENTTOOLS_ONLY)
     HPCC_ADD_SUBDIRECTORY (ElasticStack)
   ENDIF()
   HPCC_ADD_SUBDIRECTORY (Azure)
+  HPCC_ADD_SUBDIRECTORY (Grafana)
 ENDIF()

system/logaccess/Grafana/CMakeLists.txt

@@ -0,0 +1,19 @@
+###############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2022 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+IF(NOT CLIENTTOOLS_ONLY)
+    HPCC_ADD_SUBDIRECTORY (CurlClient)
+ENDIF()

system/logaccess/Grafana/CurlClient/CMakeLists.txt

@@ -0,0 +1,45 @@
+###############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2022 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+project(GrafanaCurllogaccess)
+
+# Required installed libraries
+find_package(CURL REQUIRED)
+
+set(srcs
+  ${CMAKE_CURRENT_SOURCE_DIR}/GrafanaCurlClient.cpp
+)
+
+include_directories(
+  ${HPCC_SOURCE_DIR}/system/include
+  ${HPCC_SOURCE_DIR}/system/jlib
+  ${CURL_INCLUDE_DIR}
+)
+
+add_definitions(-DGRAFANA_CURL_LOGACCESS_EXPORTS)
+
+HPCC_ADD_LIBRARY(${PROJECT_NAME} SHARED ${srcs})
+
+target_link_libraries(${PROJECT_NAME}
+   PRIVATE jlib
+   PRIVATE ${CURL_LIBRARIES}
+)
+
+install(TARGETS ${PROJECT_NAME}
+  RUNTIME DESTINATION ${EXEC_DIR}
+  LIBRARY DESTINATION ${LIB_DIR}
+  CALC_DEPS
+)