Customizable logs feature (#177)

* feat: adding support for legacy XML log output (#119) * Custom Log Format (#124) * Make JSON the default log format --------- Co-authored-by: Charity Kathure <[email protected]> * Custom JSON Log Sanitization (#128) --------- Co-authored-by: Charity Kathure <[email protected]> * Formatting for scalar integer property values using TdhFormatProperty (#129) --------- Co-authored-by: Bob Sira <[email protected]> * docs: configurable / custom log format (#136) --------- Co-authored-by: Charity Kathure <[email protected]> * fix process monitor formatting (#175) --------- Co-authored-by: Charity Kathure <[email protected]> * Process Monitor Custom Logging (#176) Signed-off-by: Charity Kathure <[email protected]> --------- Signed-off-by: Charity Kathure <[email protected]> Co-authored-by: Charity Kathure <[email protected]> * Resolve build error and lint issues, and add process monitor details into the docs Signed-off-by: Charity Kathure <[email protected]> --------- Signed-off-by: Charity Kathure <[email protected]> Co-authored-by: Charity Kathure <[email protected]> Co-authored-by: Bob Sira <[email protected]> Co-authored-by: Bob Sira <[email protected]> Co-authored-by: Ian King'ori <[email protected]>
microsoft · Jul 24, 2024 · d1d39d6 · d1d39d6
1 parent c524905
commit d1d39d6
Show file tree

Hide file tree

Showing 20 changed files with 818 additions and 111 deletions.
diff --git a/LogMonitor/LogMonitorTests/ConfigFileParserTests.cpp b/LogMonitor/LogMonitorTests/ConfigFileParserTests.cpp
@@ -1712,5 +1712,48 @@ namespace LogMonitorTests
                 );
             }
         }
+
+        TEST_METHOD(TestSourceProcess)
+        {
+            //
+            // Template of a valid configuration string, with a process source.
+            //
+            std::wstring configFileStrFormat =
+                L"{    \
+                    \"LogConfig\": {    \
+                        \"logFormat\": \"%s\",\
+                        \"sources\": [ \
+                            {\
+                                \"type\": \"Process\",\
+                                \"customLogFormat\": \"%s\"\
+                            }\
+                        ]\
+                    }\
+                }";
+
+            std::wstring logFormat = L"custom";
+            std::wstring customLogFormat = L"{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Logline':'%Logline%'}";
+            {
+                std::wstring configFileStr = Utility::FormatString(
+                    configFileStrFormat.c_str(),
+                    logFormat.c_str(),
+                    customLogFormat.c_str()
+                );
+
+                JsonFileParser jsonParser(configFileStr);
+                LoggerSettings settings;
+
+                bool success = ReadConfigFile(jsonParser, settings);
+
+                std::wstring output = RecoverOuput();
+
+                //
+                // The config string was valid
+                //
+                Assert::IsTrue(success);
+                Assert::AreEqual(L"", output.c_str());
+            }
+        }
+
     };
 }
diff --git a/LogMonitor/LogMonitorTests/EtwMonitorTests.cpp b/LogMonitor/LogMonitorTests/EtwMonitorTests.cpp
@@ -74,7 +74,7 @@ namespace LogMonitorTests
                 ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
                 fflush(stdout);
 
-                EtwMonitor etwMonitor(etwProviders, true);
+                EtwMonitor etwMonitor(etwProviders, L"json", L"");
                 Sleep(WAIT_TIME_ETWMONITOR_START);
 
 
@@ -168,7 +168,7 @@ namespace LogMonitorTests
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
             fflush(stdout);
 
-            EtwMonitor etwMonitor(etwProviders, true);
+            EtwMonitor etwMonitor(etwProviders, L"json", L"");
 
             //
             // It must find the provider, and start printing events.
@@ -203,7 +203,7 @@ namespace LogMonitorTests
             fflush(stdout);
 
 
-            std::function<void(void)> f1 = [&etwProviders] { EtwMonitor etwMonitor(etwProviders, true); };
+            std::function<void(void)> f1 = [&etwProviders] { EtwMonitor etwMonitor(etwProviders, L"json", L""); };
             Assert::ExpectException<std::invalid_argument>(f1);
         }
 
@@ -223,7 +223,7 @@ namespace LogMonitorTests
             providerWithoutLevel.Keywords = 0;
 
             std::vector<ETWProvider> etwProviders = { providerWithLevel, providerWithoutLevel };
-            EtwMonitor etwMonitor(etwProviders, true);
+            EtwMonitor etwMonitor(etwProviders, L"json", L"");
 
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
             fflush(stdout);

diff --git a/LogMonitor/LogMonitorTests/EventMonitorTests.cpp b/LogMonitor/LogMonitorTests/EventMonitorTests.cpp
@@ -103,7 +103,7 @@ namespace LogMonitorTests
         {
             std::vector<EventLogChannel> eventChannels = { {L"Application", EventChannelLogLevel::Error} };
 
-            EventMonitor eventMonitor(eventChannels, true, false);
+            EventMonitor eventMonitor(eventChannels, true, false, L"json", L"");
             Sleep(WAIT_TIME_EVENTMONITOR_START);
 
             //
@@ -209,7 +209,7 @@ namespace LogMonitorTests
         {
             std::vector<EventLogChannel> eventChannels = { {L"Application", EventChannelLogLevel::Information} };
 
-            EventMonitor eventMonitor(eventChannels, true, false);
+            EventMonitor eventMonitor(eventChannels, true, false, L"json", L"");
             Sleep(WAIT_TIME_EVENTMONITOR_START);
 
             //
@@ -356,7 +356,7 @@ namespace LogMonitorTests
 
             Assert::AreEqual(0, WriteEvent(level, eventId, message));
 
-            EventMonitor eventMonitor(eventChannels, true, true);
+            EventMonitor eventMonitor(eventChannels, true, true, L"json", L"");
             Sleep(WAIT_TIME_EVENTMONITOR_START);
 
             {
@@ -382,7 +382,7 @@ namespace LogMonitorTests
         {
             std::vector<EventLogChannel> eventChannels = { {L"System", EventChannelLogLevel::Information} };
 
-            EventMonitor eventMonitor(eventChannels, false, false);
+            EventMonitor eventMonitor(eventChannels, false, false, L"json", L"");
             Sleep(WAIT_TIME_EVENTMONITOR_START);
 
             {

diff --git a/LogMonitor/LogMonitorTests/LogFileMonitorTests.cpp b/LogMonitor/LogMonitorTests/LogFileMonitorTests.cpp
@@ -176,7 +176,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //
@@ -224,7 +224,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //
@@ -293,7 +293,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //
@@ -396,7 +396,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //
@@ -572,7 +572,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //
@@ -678,7 +678,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //
@@ -798,7 +798,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //
@@ -989,7 +989,7 @@ namespace LogMonitorTests
             fflush(stdout);
             ZeroMemory(bigOutBuf, sizeof(bigOutBuf));
 
-            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds);
+            std::shared_ptr<LogFileMonitor> logfileMon = std::make_shared<LogFileMonitor>(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L"");
             Sleep(WAIT_TIME_LOGFILEMONITOR_START);
 
             //

diff --git a/LogMonitor/docs/README.md b/LogMonitor/docs/README.md
@@ -6,6 +6,7 @@
 - [Event Log Monitoring](#event-log-monitoring)
 - [Log File Monitoring](#log-file-monitoring)
 - [Process Monitoring](#process-monitoring)
+- [Log Format Customization](#log-format-customization)
 
 ## Sample Config File
 
@@ -366,3 +367,114 @@ CMD "c:\\windows\\system32\\ping.exe -n 20 localhost"
 ```
 
 The Process Monitor will stream the output for `c:\windows\system32\ping.exe -n 20 localhost`
+
+## Log Format Customization
+
+### Description
+By default, logs will be displayed in JSON format. However, users can change the log format to either `XML` or their own `custom` defined format.
+
+To specify the log format, a user needs to configure the `logFormat` field in `LogMonitorConfig.json` to either `XML`, `JSON` or `Custom` <em>(the field value is not case-insensitive)</em>
+<br>For `JSON` and `XML` log formats, no additional configurations are required. However, the `Custom` log format, needs further configuration. For custom log formats, a user needs to specify the `customLogFormat` at the source level.
+
+### Custom Log Format Pattern Layout
+To ensure the different field values are correctly displayed in the customized log outputs, ensure to wrap the field names within modulo operators (%) and the field names specified matches the correct log sources' field names.
+
+For example: `%Message%, %TimeStamp%`<br>
+
+Each log source tracked by log monitor <em>(ETW, Log File, Events, and Process Monitor logs)</em> has log field names specific to them:
+
+<strong>Event Logs:</strong>
+  - `Source`: The log source (Event Log)
+  - `TimeStamp`: Time at which the event was generated
+  - `EventID`: Unique identifier assigned to an individual event
+  - `Severity`: A label that indicates the importance or criticality of an event
+  - `Message`: The event message
+
+<strong>ETW:</strong>
+  - `Source`: The log source (ETW)
+  - `TimeStamp`: Time at which the event was generated
+  - `Severity`: A label that indicates the importance or criticality of an event
+  - `ProviderId`:  Unique identifier that is assigned to the event provider during its registration process.
+  - `ProviderName`: Unique identifier or name assigned to an event provider
+  - `DecodingSource`: Component or provider responsible for decoding and translating raw event data into a human-readable format
+  - `ExecutionProcessId`: Identifier associated with a process that is being executed at the time an event is generated
+  - `ExecutionThreadId`: Identifier associated with a thread at the time an event is generated
+  - `Keyword`:  Flag or attribute assigned to an event or a group of related events
+  - `EventId`: Unique identifier assigned to an individual event
+  - `EventData`: Payload or data associated with an event.
+
+<strong>Log Files:</strong>
+  - `Source`: The log source (File)
+  - `TimeStamp`: Time at which the change was introduced in the monitored file.
+  - `FileName`: Name of the file that the log entry is read from.
+  - `Message`: The line/change added in the monitored file.
+
+<strong>Process Monitor:</strong>
+  - `Source`: The log source (Process Monitor)
+  - `TimeStamp`: Time at which the process was executed
+  - `Logline` or `logEntry` : The output of the process/command executed
+
+### Sample Custom Log Configuration
+
+```json
+{
+  "LogConfig": {
+    "logFormat": "custom",
+    "sources": [
+      {
+        "type": "ETW",
+        "eventFormatMultiLine": false,
+        "providers": [
+          {
+            "providerName": "Microsoft-Windows-WLAN-Drive",
+            "providerGuid": "DAA6A96B-F3E7-4D4D-A0D6-31A350E6A445",
+            "level": "Information"
+          }
+        ],
+        "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Severity':'%Severity%', 'ProviderId':'%ProviderId%', 'ProviderName':'%ProviderName%', 'EventId':'%EventId%', 'EventData':'%EventData%'}"
+      },
+      {
+        "type": "File",
+        "directory": "c:\\inetpub\\logs",
+        "filter": "*.log",
+        "includeSubdirectories": true,
+        "customLogFormat": "{'message':%Message%,'source':%Source%,'fileName':%FileName%}"
+      },
+      {
+        "type": "Process",
+        "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Logline':'%Logline%'}" 
+      }
+    ]
+  }
+}
+```
+
+For advanced usage of the custom log feature, a user can choose to define their own custom JSON log format. In such a case, The `logFormat` value should be `custom`. 
+<br>To enable sanitization of the JSON output and ensure the the outputs displayed by the tool is valid, the user can add a suffix: `'|json'` after the desired custom log format.
+
+For example:
+```json
+{
+  "LogConfig": {
+	"logFormat": "custom",
+        "sources": [
+	      {
+                "type": "ETW",
+                "eventFormatMultiLine": false,
+                "providers": [
+                  {
+                     "providerName": "Microsoft-Windows-WLAN-Drive",
+                    "providerGuid": "DAA6A96B-F3E7-4D4D-A0D6-31A350E6A445",
+                    "level": "Information"
+                   }
+                 ],
+                "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Severity':'%Severity%', 'ProviderId':'%ProviderId%', 'ProviderName':'%ProviderName%', 'EventId':'%EventId%', 'EventData':'%EventData%'}|json"
+	      },
+          {
+            "type": "Process",
+            "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Logline':'%Logline%'}|JSON" 
+          }
+        ]
+  }
+}
+```
diff --git a/LogMonitor/src/LogMonitor/ConfigFileParser.cpp b/LogMonitor/src/LogMonitor/ConfigFileParser.cpp
@@ -140,9 +140,10 @@ ReadLogConfigObject(
     bool sourcesTagFound = false;
     if (Parser.BeginParseObject())
     {
+        std::wstring key;
         do
         {
-            const std::wstring key(Parser.GetKey());
+            key = Parser.GetKey();
 
             if (_wcsnicmp(key.c_str(), JSON_TAG_SOURCES, _countof(JSON_TAG_SOURCES)) == 0)
             {
@@ -188,6 +189,10 @@ ReadLogConfigObject(
                     }
                 } while (Parser.ParseNextArrayElement());
             }
+            else if (_wcsnicmp(key.c_str(), JSON_TAG_LOG_FORMAT, _countof(JSON_TAG_LOG_FORMAT)) == 0)
+            {
+                Config.LogFormat = std::wstring(Parser.ParseStringValue());
+            }
             else
             {
                 logWriter.TraceWarning(Utility::FormatString(L"Error parsing configuration file. 'Unknow key %ws in the configuration file.", key.c_str()).c_str());
@@ -306,14 +311,16 @@ ReadSourceAttributes(
             // These attributes are string type
             // * directory
             // * filter
+            // * lineLogFormat
             //
             else if (_wcsnicmp(key.c_str(), JSON_TAG_DIRECTORY, _countof(JSON_TAG_DIRECTORY)) == 0)
             {
                 std::wstring directory = Parser.ParseStringValue();
                 FileMonitorUtilities::ParseDirectoryValue(directory);
                 Attributes[key] = new std::wstring(directory);
             }
-            else if (_wcsnicmp(key.c_str(), JSON_TAG_FILTER, _countof(JSON_TAG_FILTER)) == 0)
+            else if (_wcsnicmp(key.c_str(), JSON_TAG_FILTER, _countof(JSON_TAG_FILTER)) == 0
+                || _wcsnicmp(key.c_str(), JSON_TAG_CUSTOM_LOG_FORMAT, _countof(JSON_TAG_CUSTOM_LOG_FORMAT)) == 0)
             {
                 Attributes[key] = new std::wstring(Parser.ParseStringValue());
             }
@@ -648,6 +655,21 @@ AddNewSource(
 
             break;
         }
+
+        case LogSourceType::Process:
+        {
+            std::shared_ptr<SourceProcess> sourceProcess = std::make_shared< SourceProcess>();
+
+            if (!SourceProcess::Unwrap(Attributes, *sourceProcess))
+            {
+                logWriter.TraceError(L"Error parsing configuration file. Invalid Process source)");
+                return false;
+            }
+
+            Sources.push_back(std::reinterpret_pointer_cast<LogSource>(std::move(sourceProcess)));
+
+            break;
+        }
     }
     return true;
 }