diff --git a/LogMonitor/LogMonitorTests/ConfigFileParserTests.cpp b/LogMonitor/LogMonitorTests/ConfigFileParserTests.cpp index a145fe5e..4574b1ad 100644 --- a/LogMonitor/LogMonitorTests/ConfigFileParserTests.cpp +++ b/LogMonitor/LogMonitorTests/ConfigFileParserTests.cpp @@ -1712,5 +1712,48 @@ namespace LogMonitorTests ); } } + + TEST_METHOD(TestSourceProcess) + { + // + // Template of a valid configuration string, with a process source. + // + std::wstring configFileStrFormat = + L"{ \ + \"LogConfig\": { \ + \"logFormat\": \"%s\",\ + \"sources\": [ \ + {\ + \"type\": \"Process\",\ + \"customLogFormat\": \"%s\"\ + }\ + ]\ + }\ + }"; + + std::wstring logFormat = L"custom"; + std::wstring customLogFormat = L"{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Logline':'%Logline%'}"; + { + std::wstring configFileStr = Utility::FormatString( + configFileStrFormat.c_str(), + logFormat.c_str(), + customLogFormat.c_str() + ); + + JsonFileParser jsonParser(configFileStr); + LoggerSettings settings; + + bool success = ReadConfigFile(jsonParser, settings); + + std::wstring output = RecoverOuput(); + + // + // The config string was valid + // + Assert::IsTrue(success); + Assert::AreEqual(L"", output.c_str()); + } + } + }; } diff --git a/LogMonitor/LogMonitorTests/EtwMonitorTests.cpp b/LogMonitor/LogMonitorTests/EtwMonitorTests.cpp index 1544f10c..bcfa486f 100644 --- a/LogMonitor/LogMonitorTests/EtwMonitorTests.cpp +++ b/LogMonitor/LogMonitorTests/EtwMonitorTests.cpp @@ -74,7 +74,7 @@ namespace LogMonitorTests ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); fflush(stdout); - EtwMonitor etwMonitor(etwProviders, true); + EtwMonitor etwMonitor(etwProviders, L"json", L""); Sleep(WAIT_TIME_ETWMONITOR_START); @@ -168,7 +168,7 @@ namespace LogMonitorTests ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); fflush(stdout); - EtwMonitor etwMonitor(etwProviders, true); + EtwMonitor etwMonitor(etwProviders, L"json", L""); // // It must find the provider, and start printing events. @@ -203,7 +203,7 @@ namespace LogMonitorTests fflush(stdout); - std::function f1 = [&etwProviders] { EtwMonitor etwMonitor(etwProviders, true); }; + std::function f1 = [&etwProviders] { EtwMonitor etwMonitor(etwProviders, L"json", L""); }; Assert::ExpectException(f1); } @@ -223,7 +223,7 @@ namespace LogMonitorTests providerWithoutLevel.Keywords = 0; std::vector etwProviders = { providerWithLevel, providerWithoutLevel }; - EtwMonitor etwMonitor(etwProviders, true); + EtwMonitor etwMonitor(etwProviders, L"json", L""); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); fflush(stdout); diff --git a/LogMonitor/LogMonitorTests/EventMonitorTests.cpp b/LogMonitor/LogMonitorTests/EventMonitorTests.cpp index 3529a8b2..724fe866 100644 --- a/LogMonitor/LogMonitorTests/EventMonitorTests.cpp +++ b/LogMonitor/LogMonitorTests/EventMonitorTests.cpp @@ -103,7 +103,7 @@ namespace LogMonitorTests { std::vector eventChannels = { {L"Application", EventChannelLogLevel::Error} }; - EventMonitor eventMonitor(eventChannels, true, false); + EventMonitor eventMonitor(eventChannels, true, false, L"json", L""); Sleep(WAIT_TIME_EVENTMONITOR_START); // @@ -209,7 +209,7 @@ namespace LogMonitorTests { std::vector eventChannels = { {L"Application", EventChannelLogLevel::Information} }; - EventMonitor eventMonitor(eventChannels, true, false); + EventMonitor eventMonitor(eventChannels, true, false, L"json", L""); Sleep(WAIT_TIME_EVENTMONITOR_START); // @@ -356,7 +356,7 @@ namespace LogMonitorTests Assert::AreEqual(0, WriteEvent(level, eventId, message)); - EventMonitor eventMonitor(eventChannels, true, true); + EventMonitor eventMonitor(eventChannels, true, true, L"json", L""); Sleep(WAIT_TIME_EVENTMONITOR_START); { @@ -382,7 +382,7 @@ namespace LogMonitorTests { std::vector eventChannels = { {L"System", EventChannelLogLevel::Information} }; - EventMonitor eventMonitor(eventChannels, false, false); + EventMonitor eventMonitor(eventChannels, false, false, L"json", L""); Sleep(WAIT_TIME_EVENTMONITOR_START); { diff --git a/LogMonitor/LogMonitorTests/LogFileMonitorTests.cpp b/LogMonitor/LogMonitorTests/LogFileMonitorTests.cpp index 86f1f84a..a31b871b 100644 --- a/LogMonitor/LogMonitorTests/LogFileMonitorTests.cpp +++ b/LogMonitor/LogMonitorTests/LogFileMonitorTests.cpp @@ -176,7 +176,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // @@ -224,7 +224,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // @@ -293,7 +293,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // @@ -396,7 +396,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // @@ -572,7 +572,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // @@ -678,7 +678,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // @@ -798,7 +798,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // @@ -989,7 +989,7 @@ namespace LogMonitorTests fflush(stdout); ZeroMemory(bigOutBuf, sizeof(bigOutBuf)); - std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds); + std::shared_ptr logfileMon = std::make_shared(sourceFile.Directory, sourceFile.Filter, sourceFile.IncludeSubdirectories, sourceFile.WaitInSeconds, L"json", L""); Sleep(WAIT_TIME_LOGFILEMONITOR_START); // diff --git a/LogMonitor/docs/README.md b/LogMonitor/docs/README.md index 52aa0907..d662097c 100644 --- a/LogMonitor/docs/README.md +++ b/LogMonitor/docs/README.md @@ -6,6 +6,7 @@ - [Event Log Monitoring](#event-log-monitoring) - [Log File Monitoring](#log-file-monitoring) - [Process Monitoring](#process-monitoring) +- [Log Format Customization](#log-format-customization) ## Sample Config File @@ -366,3 +367,114 @@ CMD "c:\\windows\\system32\\ping.exe -n 20 localhost" ``` The Process Monitor will stream the output for `c:\windows\system32\ping.exe -n 20 localhost` + +## Log Format Customization + +### Description +By default, logs will be displayed in JSON format. However, users can change the log format to either `XML` or their own `custom` defined format. + +To specify the log format, a user needs to configure the `logFormat` field in `LogMonitorConfig.json` to either `XML`, `JSON` or `Custom` (the field value is not case-insensitive) +
For `JSON` and `XML` log formats, no additional configurations are required. However, the `Custom` log format, needs further configuration. For custom log formats, a user needs to specify the `customLogFormat` at the source level. + +### Custom Log Format Pattern Layout +To ensure the different field values are correctly displayed in the customized log outputs, ensure to wrap the field names within modulo operators (%) and the field names specified matches the correct log sources' field names. + +For example: `%Message%, %TimeStamp%`
+ +Each log source tracked by log monitor (ETW, Log File, Events, and Process Monitor logs) has log field names specific to them: + +Event Logs: + - `Source`: The log source (Event Log) + - `TimeStamp`: Time at which the event was generated + - `EventID`: Unique identifier assigned to an individual event + - `Severity`: A label that indicates the importance or criticality of an event + - `Message`: The event message + +ETW: + - `Source`: The log source (ETW) + - `TimeStamp`: Time at which the event was generated + - `Severity`: A label that indicates the importance or criticality of an event + - `ProviderId`: Unique identifier that is assigned to the event provider during its registration process. + - `ProviderName`: Unique identifier or name assigned to an event provider + - `DecodingSource`: Component or provider responsible for decoding and translating raw event data into a human-readable format + - `ExecutionProcessId`: Identifier associated with a process that is being executed at the time an event is generated + - `ExecutionThreadId`: Identifier associated with a thread at the time an event is generated + - `Keyword`: Flag or attribute assigned to an event or a group of related events + - `EventId`: Unique identifier assigned to an individual event + - `EventData`: Payload or data associated with an event. + +Log Files: + - `Source`: The log source (File) + - `TimeStamp`: Time at which the change was introduced in the monitored file. + - `FileName`: Name of the file that the log entry is read from. + - `Message`: The line/change added in the monitored file. + +Process Monitor: + - `Source`: The log source (Process Monitor) + - `TimeStamp`: Time at which the process was executed + - `Logline` or `logEntry` : The output of the process/command executed + +### Sample Custom Log Configuration + +```json +{ + "LogConfig": { + "logFormat": "custom", + "sources": [ + { + "type": "ETW", + "eventFormatMultiLine": false, + "providers": [ + { + "providerName": "Microsoft-Windows-WLAN-Drive", + "providerGuid": "DAA6A96B-F3E7-4D4D-A0D6-31A350E6A445", + "level": "Information" + } + ], + "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Severity':'%Severity%', 'ProviderId':'%ProviderId%', 'ProviderName':'%ProviderName%', 'EventId':'%EventId%', 'EventData':'%EventData%'}" + }, + { + "type": "File", + "directory": "c:\\inetpub\\logs", + "filter": "*.log", + "includeSubdirectories": true, + "customLogFormat": "{'message':%Message%,'source':%Source%,'fileName':%FileName%}" + }, + { + "type": "Process", + "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Logline':'%Logline%'}" + } + ] + } +} +``` + +For advanced usage of the custom log feature, a user can choose to define their own custom JSON log format. In such a case, The `logFormat` value should be `custom`. +
To enable sanitization of the JSON output and ensure the the outputs displayed by the tool is valid, the user can add a suffix: `'|json'` after the desired custom log format. + +For example: +```json +{ + "LogConfig": { + "logFormat": "custom", + "sources": [ + { + "type": "ETW", + "eventFormatMultiLine": false, + "providers": [ + { + "providerName": "Microsoft-Windows-WLAN-Drive", + "providerGuid": "DAA6A96B-F3E7-4D4D-A0D6-31A350E6A445", + "level": "Information" + } + ], + "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Severity':'%Severity%', 'ProviderId':'%ProviderId%', 'ProviderName':'%ProviderName%', 'EventId':'%EventId%', 'EventData':'%EventData%'}|json" + }, + { + "type": "Process", + "customLogFormat": "{'TimeStamp':'%TimeStamp%', 'source':'%Source%', 'Logline':'%Logline%'}|JSON" + } + ] + } +} +``` diff --git a/LogMonitor/src/LogMonitor/ConfigFileParser.cpp b/LogMonitor/src/LogMonitor/ConfigFileParser.cpp index bfd188d7..c29a835a 100644 --- a/LogMonitor/src/LogMonitor/ConfigFileParser.cpp +++ b/LogMonitor/src/LogMonitor/ConfigFileParser.cpp @@ -140,9 +140,10 @@ ReadLogConfigObject( bool sourcesTagFound = false; if (Parser.BeginParseObject()) { + std::wstring key; do { - const std::wstring key(Parser.GetKey()); + key = Parser.GetKey(); if (_wcsnicmp(key.c_str(), JSON_TAG_SOURCES, _countof(JSON_TAG_SOURCES)) == 0) { @@ -188,6 +189,10 @@ ReadLogConfigObject( } } while (Parser.ParseNextArrayElement()); } + else if (_wcsnicmp(key.c_str(), JSON_TAG_LOG_FORMAT, _countof(JSON_TAG_LOG_FORMAT)) == 0) + { + Config.LogFormat = std::wstring(Parser.ParseStringValue()); + } else { logWriter.TraceWarning(Utility::FormatString(L"Error parsing configuration file. 'Unknow key %ws in the configuration file.", key.c_str()).c_str()); @@ -306,6 +311,7 @@ ReadSourceAttributes( // These attributes are string type // * directory // * filter + // * lineLogFormat // else if (_wcsnicmp(key.c_str(), JSON_TAG_DIRECTORY, _countof(JSON_TAG_DIRECTORY)) == 0) { @@ -313,7 +319,8 @@ ReadSourceAttributes( FileMonitorUtilities::ParseDirectoryValue(directory); Attributes[key] = new std::wstring(directory); } - else if (_wcsnicmp(key.c_str(), JSON_TAG_FILTER, _countof(JSON_TAG_FILTER)) == 0) + else if (_wcsnicmp(key.c_str(), JSON_TAG_FILTER, _countof(JSON_TAG_FILTER)) == 0 + || _wcsnicmp(key.c_str(), JSON_TAG_CUSTOM_LOG_FORMAT, _countof(JSON_TAG_CUSTOM_LOG_FORMAT)) == 0) { Attributes[key] = new std::wstring(Parser.ParseStringValue()); } @@ -648,6 +655,21 @@ AddNewSource( break; } + + case LogSourceType::Process: + { + std::shared_ptr sourceProcess = std::make_shared< SourceProcess>(); + + if (!SourceProcess::Unwrap(Attributes, *sourceProcess)) + { + logWriter.TraceError(L"Error parsing configuration file. Invalid Process source)"); + return false; + } + + Sources.push_back(std::reinterpret_pointer_cast(std::move(sourceProcess))); + + break; + } } return true; } diff --git a/LogMonitor/src/LogMonitor/EtwMonitor.cpp b/LogMonitor/src/LogMonitor/EtwMonitor.cpp index 5a886898..d03e26b1 100644 --- a/LogMonitor/src/LogMonitor/EtwMonitor.cpp +++ b/LogMonitor/src/LogMonitor/EtwMonitor.cpp @@ -30,9 +30,11 @@ static const std::wstring g_sessionName = L"Log Monitor ETW Session"; EtwMonitor::EtwMonitor( _In_ const std::vector& Providers, - _In_ bool EventFormatMultiLine + _In_ std::wstring LogFormat, + _In_ std::wstring CustomLogFormat = L"" ) : - m_eventFormatMultiLine(EventFormatMultiLine) + m_logFormat(LogFormat), + m_customLogFormat(CustomLogFormat) { // // This is set as 'true' to stop processing events. @@ -721,11 +723,12 @@ EtwMonitor::OnRecordEvent( /// /// Format ETW eventlog into a JSON output /// -std::wstring etwJsonFormat(EtwLogEntry* pLogEntry) +std::wstring EtwJsonFormat(EtwLogEntry* pLogEntry) { std::wostringstream oss; + // construct the JSON output - oss << L"{\"Source\":\"ETW\",\"LogEntry\":{"; + oss << L"{\"Source\":\"" << pLogEntry->source << L"\",\"LogEntry\":{"; oss << L"\"Time\":\"" << pLogEntry->Time << L"\","; oss << L"\"ProviderName\":\"" << pLogEntry->ProviderName << L"\","; oss << L"\"ProviderId\":\"" << pLogEntry->ProviderId << "\","; @@ -737,8 +740,8 @@ std::wstring etwJsonFormat(EtwLogEntry* pLogEntry) oss << L"\"Level\":\"" << pLogEntry->Level << L"\","; oss << L"\"Keyword\":\"" << pLogEntry->Keyword << L"\","; oss << L"\"EventId\":\"" << pLogEntry->EventId << "\","; - oss << L"\"EventData\":{"; + bool firstEntry = true; for (auto evtData : pLogEntry->EventData) { oss << (firstEntry ? "" : ","); @@ -748,39 +751,52 @@ std::wstring etwJsonFormat(EtwLogEntry* pLogEntry) Utility::SanitizeJson(key); oss << "\"" << key << "\":"; - // - // format (JSON) numbers without quotation marks, e.g. - /* - "EventData": { - "FrameUniqueID": 403787, - "PortNumber" : 0, - "TID" : 0, - "PeerID" : 0, - "PayloadLength" : 68, - "QueueLength" : 0, - "QueueState" : "false", - "CustomData1" : 24, - "CustomData2" : 0, - "CustomData3" : 0 - } - */ - // if (Utility::isJsonNumber(evtData.second)) { oss << evtData.second; - } - else { + } else { wstring value = evtData.second; Utility::SanitizeJson(value); oss << L"\"" << value << L"\""; } } - oss << L"}"; + oss << L"}"; oss << L"},\"SchemaVersion\":\"1.0.0\"}"; return oss.str(); } +/// +/// Format ETW eventlog into a XML output +/// +std::wstring EtwXMLFormat(EtwLogEntry* pLogEntry) +{ + std::wostringstream oss; + + // construct the XML output + oss << L"ETW"; + oss << L""; + oss << L"" << pLogEntry->ProviderName << L""; + oss << L"" << pLogEntry->ProviderId << ""; + oss << L"" << pLogEntry->DecodingSource << L""; + oss << L""; + oss << L"" << pLogEntry->ExecProcessId << L""; + oss << L"" << pLogEntry->ExecThreadId << L""; + oss << ""; + oss << L"" << pLogEntry->Level << L""; + oss << L"" << pLogEntry->Keyword << L""; + oss << L"" << pLogEntry->EventId << L""; + + oss << L""; + for (auto evtData : pLogEntry->EventData) { + wstring key = evtData.first; + wstring value = evtData.second; + oss << "<" << key << ">" << value <<""; + } + oss << L""; + + return oss.str(); +} /// /// Prints the data and metadata of the event. @@ -826,7 +842,15 @@ EtwMonitor::PrintEvent( return status; } - std::wstring formattedEvent = etwJsonFormat(pLogEntry); + std::wstring formattedEvent; + if (Utility::CompareWStrings(m_logFormat, L"XML")) { + formattedEvent = EtwXMLFormat(pLogEntry); + } else if (Utility::CompareWStrings(m_logFormat, L"Custom")) { + formattedEvent = Utility::FormatEventLineLog(m_customLogFormat, pLogEntry, pLogEntry->source); + } else { + formattedEvent = EtwJsonFormat(pLogEntry); + } + logWriter.WriteConsoleLog(formattedEvent); } catch(std::bad_alloc&) @@ -860,6 +884,7 @@ EtwMonitor::FormatMetadata( fileTime.dwHighDateTime = EventRecord->EventHeader.TimeStamp.HighPart; fileTime.dwLowDateTime = EventRecord->EventHeader.TimeStamp.LowPart; + pLogEntry->source = L"ETW"; pLogEntry->Time = Utility::FileTimeToString(fileTime).c_str(); // @@ -1248,7 +1273,8 @@ EtwMonitor::GetPropertyLength( // EVENT_PROPERTY_INFO.length field will be zero. // if (TDH_INTYPE_BINARY == EventInfo->EventPropertyInfoArray[Index].nonStructType.InType && - TDH_OUTTYPE_IPV6 == EventInfo->EventPropertyInfoArray[Index].nonStructType.OutType) + TDH_OUTTYPE_IPV6 == EventInfo->EventPropertyInfoArray[Index].nonStructType.OutType && + EventInfo->EventPropertyInfoArray[Index].length == 0) { PropertyLength = (USHORT)sizeof(IN6_ADDR); } @@ -1258,6 +1284,39 @@ EtwMonitor::GetPropertyLength( { PropertyLength = EventInfo->EventPropertyInfoArray[Index].length; } + else if (0 == (EventInfo->EventPropertyInfoArray[Index].Flags & (PropertyStruct | PropertyParamCount)) && + EventInfo->EventPropertyInfoArray[Index].count == 1 ) + { + BYTE const* pbData = static_cast(EventRecord->UserData); + BYTE const* pbDataEnd = pbData + EventRecord->UserDataLength; + + switch (EventInfo->EventPropertyInfoArray[Index].nonStructType.InType) + { + case TDH_INTYPE_INT8: + case TDH_INTYPE_UINT8: + if ((pbDataEnd - pbData) >= 1) + { + PropertyLength = *pbData; + } + break; + case TDH_INTYPE_INT16: + case TDH_INTYPE_UINT16: + if ((pbDataEnd - pbData) >= 2) + { + PropertyLength = *reinterpret_cast(pbData); + } + break; + case TDH_INTYPE_INT32: + case TDH_INTYPE_UINT32: + case TDH_INTYPE_HEXINT32: + if ((pbDataEnd - pbData) >= 4) + { + auto val = *reinterpret_cast(pbData); + PropertyLength = static_cast(val > 0xffffu ? 0xffffu : val); + } + break; + } + } else { logWriter.TraceError( @@ -1409,3 +1468,29 @@ EtwMonitor::RemoveTrailingSpace( *((LPWSTR)((PBYTE)MapName + (MapName->MapEntryArray[i].OutputOffset + byteLength))) = L'\0'; } } + +std::wstring EtwMonitor::EtwFieldsMapping(_In_ std::wstring etwFields, _In_ void* pLogEntryData) +{ + std::wostringstream oss; + EtwLogEntry* pLogEntry = (EtwLogEntry*)pLogEntryData; + + if (Utility::CompareWStrings(etwFields, L"TimeStamp")) oss << pLogEntry->Time; + if (Utility::CompareWStrings(etwFields, L"Severity")) oss << pLogEntry->Level; + if (Utility::CompareWStrings(etwFields, L"Source")) oss << pLogEntry->source; + if (Utility::CompareWStrings(etwFields, L"ProviderId")) oss << pLogEntry->ProviderId; + if (Utility::CompareWStrings(etwFields, L"ProviderName")) oss << pLogEntry->ProviderName; + if (Utility::CompareWStrings(etwFields, L"DecodingSource")) oss << pLogEntry->DecodingSource; + if (Utility::CompareWStrings(etwFields, L"ExecutionProcessId")) oss << pLogEntry->ExecProcessId; + if (Utility::CompareWStrings(etwFields, L"ExecutionThreadId")) oss << pLogEntry->ExecThreadId; + if (Utility::CompareWStrings(etwFields, L"Keyword")) oss << pLogEntry->Keyword; + if (Utility::CompareWStrings(etwFields, L"EventId")) oss << pLogEntry->EventId; + if (Utility::CompareWStrings(etwFields, L"EventData")) { + for (auto evtData : pLogEntry->EventData) { + wstring key = evtData.first; + wstring value = evtData.second; + oss << key << ": " << value << " "; + } + } + + return oss.str(); +} diff --git a/LogMonitor/src/LogMonitor/EtwMonitor.h b/LogMonitor/src/LogMonitor/EtwMonitor.h index fad672d9..50c831e3 100644 --- a/LogMonitor/src/LogMonitor/EtwMonitor.h +++ b/LogMonitor/src/LogMonitor/EtwMonitor.h @@ -14,6 +14,7 @@ typedef LPTSTR(NTAPI* PIPV6ADDRTOSTRING)( // struct to hold the ETW logEntry data // struct EtwLogEntry { + std::wstring source; std::wstring Time; std::wstring ProviderId; std::wstring ProviderName; @@ -34,16 +35,20 @@ class EtwMonitor final EtwMonitor( _In_ const std::vector& Providers, - _In_ bool EventFormatMultiLine + _In_ std::wstring LogFormat, + _In_ std::wstring CustomLogFormat ); ~EtwMonitor(); + static std::wstring EtwFieldsMapping(_In_ std::wstring etwFields, _In_ void* pLogEntryData); + private: static constexpr int ETW_MONITOR_THREAD_EXIT_MAX_WAIT_MILLIS = 5 * 1000; std::vector m_providersConfig; - bool m_eventFormatMultiLine; + std::wstring m_logFormat; + std::wstring m_customLogFormat; TRACEHANDLE m_startTraceHandle; // diff --git a/LogMonitor/src/LogMonitor/EventMonitor.cpp b/LogMonitor/src/LogMonitor/EventMonitor.cpp index 810ce719..6785b057 100644 --- a/LogMonitor/src/LogMonitor/EventMonitor.cpp +++ b/LogMonitor/src/LogMonitor/EventMonitor.cpp @@ -27,11 +27,15 @@ using namespace std; EventMonitor::EventMonitor( _In_ const std::vector& EventChannels, _In_ bool EventFormatMultiLine, - _In_ bool StartAtOldestRecord + _In_ bool StartAtOldestRecord, + _In_ std::wstring LogFormat, + _In_ std::wstring CustomLogFormat = L"" ) : m_eventChannels(EventChannels), m_eventFormatMultiLine(EventFormatMultiLine), - m_startAtOldestRecord(StartAtOldestRecord) + m_startAtOldestRecord(StartAtOldestRecord), + m_logFormat(LogFormat), + m_customLogFormat(CustomLogFormat) { m_stopEvent = NULL; m_eventMonitorThread = NULL; @@ -411,6 +415,10 @@ EventMonitor::PrintEvent( EVT_HANDLE renderContext = NULL; EVT_HANDLE publisher = NULL; + // struct to hold the Event log entry and later format print + EventLogEntry logEntry; + EventLogEntry* pLogEntry = &logEntry; + static constexpr LPCWSTR defaultValuePaths[] = { L"Event/System/Provider/@Name", L"Event/System/Channel", @@ -499,7 +507,7 @@ EventMonitor::PrintEvent( // std::wstring providerName = (EvtVarTypeString != variants[0].Type) ? L"" : variants[0].StringVal; std::wstring channelName = (EvtVarTypeString != variants[1].Type) ? L"" : variants[1].StringVal; - UINT16 eventId = (EvtVarTypeUInt16 != variants[2].Type) ? 0 : variants[2].UInt16Val; + pLogEntry->eventId = (EvtVarTypeUInt16 != variants[2].Type) ? 0 : variants[2].UInt16Val; UINT8 level = (EvtVarTypeByte != variants[3].Type) ? 0 : variants[3].ByteVal; ULARGE_INTEGER fileTimeAsInt{}; fileTimeAsInt.QuadPart = (EvtVarTypeFileTime != variants[4].Type) ? 0 : variants[4].FileTimeVal; @@ -552,20 +560,48 @@ EventMonitor::PrintEvent( if (status == ERROR_SUCCESS) { - // supporting JSON fmt by default - auto logFmt = L"{\"Source\": \"EventLog\",\"LogEntry\": {\"Time\": \"%s\",\"Channel\": \"%s\",\"Level\": \"%s\",\"EventId\": %u,\"Message\": \"%s\"}}";; - - // sanitize message - std::wstring msg(m_eventMessageBuffer.begin(), m_eventMessageBuffer.end()); - Utility::SanitizeJson(msg); + pLogEntry->source = L"EventLog"; + pLogEntry->eventTime = Utility::FileTimeToString(fileTimeCreated); + pLogEntry->eventChannel = channelName; + pLogEntry->eventLevel = c_LevelToString[static_cast(level)]; + pLogEntry->eventMessage = (LPWSTR)(&m_eventMessageBuffer[0]); + + std::wstring formattedEvent; + if (Utility::CompareWStrings(m_logFormat, L"Custom")) { + formattedEvent = Utility::FormatEventLineLog(m_customLogFormat, pLogEntry, pLogEntry->source); + } else { + std::wstring logFmt; + if (Utility::CompareWStrings(m_logFormat, L"XML")) { + logFmt = L"%s" + L"%s%s" + L"%u%s" + L""; + } else { + logFmt = L"{\"Source\": \"%s\"," + L"\"LogEntry\": {" + L"\"Time\": \"%s\"," + L"\"Channel\": \"%s\"," + L"\"Level\": \"%s\"," + L"\"EventId\": %u," + L"\"Message\": \"%s\"" + L"}}"; + + // sanitize message + std::wstring msg(m_eventMessageBuffer.begin(), m_eventMessageBuffer.end()); + Utility::SanitizeJson(msg); + pLogEntry->eventMessage = msg; + } - std::wstring formattedEvent = Utility::FormatString( - logFmt, - Utility::FileTimeToString(fileTimeCreated).c_str(), - channelName.c_str(), - c_LevelToString[static_cast(level)].c_str(), - eventId, - msg.c_str()); + formattedEvent = Utility::FormatString( + logFmt.c_str(), + pLogEntry->source.c_str(), + pLogEntry->eventTime.c_str(), + pLogEntry->eventChannel.c_str(), + pLogEntry->eventLevel.c_str(), + pLogEntry->eventId, + pLogEntry->eventMessage.c_str() + ); + } logWriter.WriteConsoleLog(formattedEvent); } @@ -784,3 +820,17 @@ EventMonitor::EnableEventLogChannel( return status; } + +std::wstring EventMonitor::EventFieldsMapping(_In_ std::wstring eventField, _In_ void* pLogEntryData) +{ + std::wostringstream oss; + EventLogEntry* pLogEntry = (EventLogEntry*)pLogEntryData; + + if (Utility::CompareWStrings(eventField, L"TimeStamp")) oss << pLogEntry->eventTime; + if (Utility::CompareWStrings(eventField, L"Severity")) oss << pLogEntry->eventLevel; + if (Utility::CompareWStrings(eventField, L"Source")) oss << pLogEntry->source; + if (Utility::CompareWStrings(eventField, L"EventID")) oss << pLogEntry->eventId; + if (Utility::CompareWStrings(eventField, L"Message")) oss << pLogEntry->eventMessage; + + return oss.str(); +} diff --git a/LogMonitor/src/LogMonitor/EventMonitor.h b/LogMonitor/src/LogMonitor/EventMonitor.h index 2d525a0e..93f49ef2 100644 --- a/LogMonitor/src/LogMonitor/EventMonitor.h +++ b/LogMonitor/src/LogMonitor/EventMonitor.h @@ -14,11 +14,15 @@ class EventMonitor final EventMonitor( _In_ const std::vector& eventChannels, _In_ bool EventFormatMultiLine, - _In_ bool StartAtOldestRecord + _In_ bool StartAtOldestRecord, + _In_ std::wstring LogFormat, + _In_ std::wstring CustomLogFormat ); ~EventMonitor(); + static std::wstring EventFieldsMapping(_In_ std::wstring eventField, _In_ void* pLogEntryData); + private: static constexpr int EVENT_MONITOR_THREAD_EXIT_MAX_WAIT_MILLIS = 5 * 1000; static constexpr int EVENT_ARRAY_SIZE = 10; @@ -26,6 +30,17 @@ class EventMonitor final const std::vector m_eventChannels; bool m_eventFormatMultiLine; bool m_startAtOldestRecord; + std::wstring m_logFormat; + std::wstring m_customLogFormat; + + struct EventLogEntry { + std::wstring source; + std::wstring eventTime; + std::wstring eventChannel; + std::wstring eventLevel; + UINT16 eventId; + std::wstring eventMessage; + }; // // Signaled by destructor to request the spawned thread to stop. diff --git a/LogMonitor/src/LogMonitor/LogFileMonitor.cpp b/LogMonitor/src/LogMonitor/LogFileMonitor.cpp index f0658e2a..75c5c930 100644 --- a/LogMonitor/src/LogMonitor/LogFileMonitor.cpp +++ b/LogMonitor/src/LogMonitor/LogFileMonitor.cpp @@ -42,12 +42,16 @@ using namespace std; LogFileMonitor::LogFileMonitor(_In_ const std::wstring& LogDirectory, _In_ const std::wstring& Filter, _In_ bool IncludeSubfolders, - _In_ const std::double_t& WaitInSeconds + _In_ const std::double_t& WaitInSeconds, + _In_ std::wstring LogFormat, + _In_ std::wstring CustomLogFormat = L"" ) : m_logDirectory(LogDirectory), m_filter(Filter), m_includeSubfolders(IncludeSubfolders), - m_waitInSeconds(WaitInSeconds) + m_waitInSeconds(WaitInSeconds), + m_logFormat(LogFormat), + m_customLogFormat(CustomLogFormat) { m_stopEvent = NULL; m_overlappedEvent = NULL; @@ -1680,11 +1684,20 @@ LogFileMonitor::ReadLogFile( } void LogFileMonitor::WriteToConsole( _In_ std::wstring Message, _In_ std::wstring FileName) { - auto logFmt = L"{\"Source\":\"File\",\"LogEntry\":{\"Logline\":\"%s\",\"FileName\":\"%s\"},\"SchemaVersion\":\"1.0.0\"}"; size_t start = 0; size_t i = 0; wstring msg; + // struct to hold the File log entry and later format print + FileLogEntry logEntry; + FileLogEntry* pLogEntry = &logEntry; + + SYSTEMTIME st; + GetSystemTime(&st); + + pLogEntry->source = L"File"; + pLogEntry->currentTime = Utility::SystemTimeToString(st).c_str(); + while (true) { i = Message.find(L"\n", start); if (i == std::string::npos) { @@ -1704,12 +1717,43 @@ void LogFileMonitor::WriteToConsole( _In_ std::wstring Message, _In_ std::wstrin if (msg.size() > 0) { // escape backslashes in FileName auto fmtFileName = Utility::ReplaceAll(FileName, L"\\", L"\\\\"); - // sanitize msg - Utility::SanitizeJson(msg); - auto log = Utility::FormatString(logFmt, msg.c_str(), fmtFileName.c_str()); - logWriter.WriteConsoleLog(log); - } + pLogEntry->fileName = fmtFileName; + pLogEntry->message = msg; + + std::wstring formattedFileEntry; + if (Utility::CompareWStrings(m_logFormat, L"Custom")) { + formattedFileEntry = Utility::FormatEventLineLog(m_customLogFormat, pLogEntry, pLogEntry->source); + } else { + std::wstring logFmt; + if (Utility::CompareWStrings(m_logFormat, L"XML")) { + logFmt = L"File" + L"" + L"%s" + L"%s" + L"" + L""; + } else { + logFmt = L"{\"Source\": \"File\"," + L"\"LogEntry\": {" + L"\"Logline\": \"%s\"," + L"\"FileName\": \"%s\"" + L"}," + L"\"SchemaVersion\":\"1.0.0\"" + L"}"; + // sanitize message + Utility::SanitizeJson(msg); + pLogEntry->message = msg; + } + + formattedFileEntry = Utility::FormatString( + logFmt.c_str(), + pLogEntry->message.c_str(), + pLogEntry->fileName.c_str() + ); + } + logWriter.WriteConsoleLog(formattedFileEntry); + } if (i >= Message.size()) break; } } @@ -2045,3 +2089,16 @@ LogFileMonitor::GetFileId( return status; } + +std::wstring LogFileMonitor::FileFieldsMapping(_In_ std::wstring fileFields, _In_ void* pLogEntryData) +{ + std::wostringstream oss; + FileLogEntry* pLogEntry = (FileLogEntry*)pLogEntryData; + + if (Utility::CompareWStrings(fileFields, L"TimeStamp")) oss << pLogEntry->currentTime; + if (Utility::CompareWStrings(fileFields, L"FileName")) oss << pLogEntry->fileName; + if (Utility::CompareWStrings(fileFields, L"Source")) oss << pLogEntry->source; + if (Utility::CompareWStrings(fileFields, L"Message")) oss << pLogEntry->message; + + return oss.str(); +} diff --git a/LogMonitor/src/LogMonitor/LogFileMonitor.h b/LogMonitor/src/LogMonitor/LogFileMonitor.h index 228e6516..6f5ba92e 100644 --- a/LogMonitor/src/LogMonitor/LogFileMonitor.h +++ b/LogMonitor/src/LogMonitor/LogFileMonitor.h @@ -63,10 +63,14 @@ class LogFileMonitor final _In_ const std::wstring &LogDirectory, _In_ const std::wstring &Filter, _In_ bool IncludeSubfolders, - _In_ const std::double_t &WaitInSeconds); + _In_ const std::double_t &WaitInSeconds, + _In_ std::wstring LogFormat, + _In_ std::wstring CustomLogFormat); ~LogFileMonitor(); + static std::wstring FileFieldsMapping(_In_ std::wstring eventFields, _In_ void* pLogEntryData); + private: static constexpr int LOG_MONITOR_THREAD_EXIT_MAX_WAIT_MILLIS = 5 * 1000; static constexpr int RECORDS_BUFFER_SIZE_BYTES = 8 * 1024; @@ -76,6 +80,15 @@ class LogFileMonitor final std::wstring m_filter; std::double_t m_waitInSeconds; bool m_includeSubfolders; + std::wstring m_logFormat; + std::wstring m_customLogFormat; + + struct FileLogEntry { + std::wstring source; + std::wstring currentTime; + std::wstring fileName; + std::wstring message; + }; // // Signaled by destructor to request the spawned thread to stop. diff --git a/LogMonitor/src/LogMonitor/Main.cpp b/LogMonitor/src/LogMonitor/Main.cpp index bc42b229..23513ee7 100644 --- a/LogMonitor/src/LogMonitor/Main.cpp +++ b/LogMonitor/src/LogMonitor/Main.cpp @@ -24,6 +24,7 @@ HANDLE g_hStopEvent = INVALID_HANDLE_VALUE; std::unique_ptr g_eventMon(nullptr); std::vector> g_logfileMonitors; std::unique_ptr g_etwMon(nullptr); +std::wstring logFormat, processMonitorCustomFormat; /// Handle signals. /// @@ -104,6 +105,10 @@ void StartMonitors(_In_ LoggerSettings& settings) bool eventMonMultiLine; bool eventMonStartAtOldestRecord; bool etwMonMultiLine; + logFormat = settings.LogFormat; + std::wstring eventCustomLogFormat; + std::wstring etwCustomLogFormat; + std::wstring processCustomLogFormat; for (auto source : settings.Sources) { @@ -121,6 +126,7 @@ void StartMonitors(_In_ LoggerSettings& settings) eventMonMultiLine = sourceEventLog->EventFormatMultiLine; eventMonStartAtOldestRecord = sourceEventLog->StartAtOldestRecord; + eventCustomLogFormat = sourceEventLog->CustomLogFormat; break; } @@ -134,7 +140,9 @@ void StartMonitors(_In_ LoggerSettings& settings) sourceFile->Directory, sourceFile->Filter, sourceFile->IncludeSubdirectories, - sourceFile->WaitInSeconds + sourceFile->WaitInSeconds, + logFormat, + sourceFile->CustomLogFormat ); g_logfileMonitors.push_back(std::move(logfileMon)); } @@ -170,17 +178,43 @@ void StartMonitors(_In_ LoggerSettings& settings) } etwMonMultiLine = sourceETW->EventFormatMultiLine; + etwCustomLogFormat = sourceETW->CustomLogFormat; break; } - } // Switch + case LogSourceType::Process: + { + std::shared_ptr sourceProcess = std::reinterpret_pointer_cast(source); + + try + { + processMonitorCustomFormat = sourceProcess->CustomLogFormat; + } + catch (std::exception& ex) + { + logWriter.TraceError( + Utility::FormatString( + L"Instantiation of a ProcessMonitor object failed. %S", ex.what() + ).c_str() + ); + } + + break; + } + }// Switch } if (!eventChannels.empty()) { try { - g_eventMon = make_unique(eventChannels, eventMonMultiLine, eventMonStartAtOldestRecord); + g_eventMon = make_unique( + eventChannels, + eventMonMultiLine, + eventMonStartAtOldestRecord, + logFormat, + eventCustomLogFormat + ); } catch (std::exception& ex) { @@ -205,7 +239,7 @@ void StartMonitors(_In_ LoggerSettings& settings) { try { - g_etwMon = make_unique(etwProviders, etwMonMultiLine); + g_etwMon = make_unique(etwProviders, logFormat, etwCustomLogFormat); } catch (...) { @@ -296,7 +330,7 @@ int __cdecl wmain(int argc, WCHAR *argv[]) cmdline += argv[i]; } - exitcode = CreateAndMonitorProcess(cmdline); + exitcode = CreateAndMonitorProcess(cmdline, logFormat, processMonitorCustomFormat); } else { diff --git a/LogMonitor/src/LogMonitor/Parser/LoggerSettings.h b/LogMonitor/src/LogMonitor/Parser/LoggerSettings.h index 081f57d5..ae0837d7 100644 --- a/LogMonitor/src/LogMonitor/Parser/LoggerSettings.h +++ b/LogMonitor/src/LogMonitor/Parser/LoggerSettings.h @@ -10,6 +10,12 @@ #define JSON_TAG_LOG_CONFIG L"LogConfig" #define JSON_TAG_SOURCES L"sources" +/// +/// Log formatting attributes +/// +#define JSON_TAG_LOG_FORMAT L"logFormat" +#define JSON_TAG_CUSTOM_LOG_FORMAT L"customLogFormat" + /// /// Valid source attributes /// @@ -132,7 +138,8 @@ enum class LogSourceType { EventLog = 0, File, - ETW + ETW, + Process }; /// @@ -141,7 +148,8 @@ enum class LogSourceType const LPCWSTR LogSourceTypeNames[] = { L"EventLog", L"File", - L"ETW" + L"ETW", + L"Process" }; /// @@ -195,6 +203,7 @@ class SourceEventLog : LogSource std::vector Channels; bool EventFormatMultiLine = true; bool StartAtOldestRecord = false; + std::wstring CustomLogFormat = L"[%TimeStamp%] [%Source%] [%Severity%] %Message%"; static bool Unwrap( _In_ AttributesMap& Attributes, @@ -234,6 +243,15 @@ class SourceEventLog : LogSource NewSource.StartAtOldestRecord = *(bool*)Attributes[JSON_TAG_START_AT_OLDEST_RECORD]; } + // + // lineLogFormat is an optional value + // + if (Attributes.find(JSON_TAG_CUSTOM_LOG_FORMAT) != Attributes.end() + && Attributes[JSON_TAG_CUSTOM_LOG_FORMAT] != nullptr) + { + NewSource.CustomLogFormat = *(std::wstring*)Attributes[JSON_TAG_CUSTOM_LOG_FORMAT]; + } + return true; } }; @@ -247,6 +265,7 @@ class SourceFile : LogSource std::wstring Directory; std::wstring Filter; bool IncludeSubdirectories = false; + std::wstring CustomLogFormat = L"[%TimeStamp%] [%Source%] [%FileName%] %Message%"; // Default wait time: 5minutes std::double_t WaitInSeconds = 300; @@ -298,6 +317,15 @@ class SourceFile : LogSource NewSource.WaitInSeconds = *(std::double_t*)Attributes[JSON_TAG_WAITINSECONDS]; } + // + // lineLogFormat is an optional value + // + if (Attributes.find(JSON_TAG_CUSTOM_LOG_FORMAT) != Attributes.end() + && Attributes[JSON_TAG_CUSTOM_LOG_FORMAT] != nullptr) + { + NewSource.CustomLogFormat = *(std::wstring*)Attributes[JSON_TAG_CUSTOM_LOG_FORMAT]; + } + return true; } }; @@ -364,6 +392,9 @@ class SourceETW : LogSource public: std::vector Providers; bool EventFormatMultiLine = true; + std::wstring CustomLogFormat = L"[%TimeStamp%] [%Source%] [%Severity%] " + L"[%ProviderId%] [%ProviderName%] " + L"[%EventId%] %EventData%"; static bool Unwrap( _In_ AttributesMap& Attributes, @@ -394,15 +425,52 @@ class SourceETW : LogSource NewSource.EventFormatMultiLine = *(bool*)Attributes[JSON_TAG_FORMAT_MULTILINE]; } + // + // lineLogFormat is an optional value + // + if (Attributes.find(JSON_TAG_CUSTOM_LOG_FORMAT) != Attributes.end() + && Attributes[JSON_TAG_CUSTOM_LOG_FORMAT] != nullptr) + { + NewSource.CustomLogFormat = *(std::wstring*)Attributes[JSON_TAG_CUSTOM_LOG_FORMAT]; + } + return true; } }; +/// +/// Represents a Source if Proccess type +/// +class SourceProcess : LogSource +{ + public: + std::wstring CustomLogFormat = L"[%TimeStamp%] [%Source%] [%LogEntry%]"; + + static bool Unwrap( + _In_ AttributesMap& Attributes, + _Out_ SourceProcess& NewSource) + { + NewSource.Type = LogSourceType::Process; + + // + // lineLogFormat is an optional value + // + if (Attributes.find(JSON_TAG_CUSTOM_LOG_FORMAT) != Attributes.end() + && Attributes[JSON_TAG_CUSTOM_LOG_FORMAT] != nullptr) + { + NewSource.CustomLogFormat = *(std::wstring*)Attributes[JSON_TAG_CUSTOM_LOG_FORMAT]; + } + + return true; + } +}; + /// /// Information about a channel Log /// typedef struct _LoggerSettings { std::vector > Sources; + std::wstring LogFormat = L"JSON"; } LoggerSettings; diff --git a/LogMonitor/src/LogMonitor/ProcessMonitor.cpp b/LogMonitor/src/LogMonitor/ProcessMonitor.cpp index 5d812520..482c7bfb 100644 --- a/LogMonitor/src/LogMonitor/ProcessMonitor.cpp +++ b/LogMonitor/src/LogMonitor/ProcessMonitor.cpp @@ -15,9 +15,10 @@ HANDLE g_hChildStd_OUT_Wr = NULL; DWORD g_processId = 0; wstring g_processName = L""; +wstring loggingformat, processCustomLogFormat; -DWORD CreateChildProcess(std::wstring& Cmdline); -DWORD ReadFromPipe(LPVOID Param); + +ProcessMonitor::ProcessMonitor(){} /// /// Creates a new process, and link its STDIN and STDOUT to the LogMonitor proccess' ones. @@ -26,8 +27,11 @@ DWORD ReadFromPipe(LPVOID Param); /// /// \return Status /// -DWORD CreateAndMonitorProcess(std::wstring& Cmdline) +DWORD CreateAndMonitorProcess(std::wstring& Cmdline, std::wstring LogFormat, std::wstring ProcessCustomLogFormat) { + loggingformat = LogFormat; + processCustomLogFormat = ProcessCustomLogFormat; + SECURITY_ATTRIBUTES saAttr; DWORD status = ERROR_SUCCESS; @@ -174,7 +178,7 @@ DWORD CreateChildProcess(std::wstring& Cmdline) /// Helper function for making a copy of the buffer. /// returns the index after the last copied byte. /// -size_t bufferCopy(char* dst, char* src, size_t start, size_t end = 0) +size_t BufferCopy(char* dst, char* src, size_t start, size_t end = 0) { char* ptr = src; size_t i = start; @@ -195,7 +199,7 @@ size_t bufferCopy(char* dst, char* src, size_t start, size_t end = 0) /// For optimization, the function also "sanitizes" the string for JSON. /// returns the index after the last copied byte. /// -size_t bufferCopyAndSanitize(char* dst, char* src) +size_t BufferCopyAndSanitize(char* dst, char* src) { char* ptr = src; size_t i = 0; @@ -229,47 +233,93 @@ size_t bufferCopyAndSanitize(char* dst, char* src) } /// -/// Helper function to formats the stdout buffer to include the other +/// Helper function to format the stdout buffer to include additional /// details from the JSON schema. /// Returns the number of bytes written to the buffer. /// -size_t formatProcessLog(char* chBuf) -{ - // {"Source":"Process","LogEntry":{"Logline":""},"SchemaVersion":"1.0.0"} - const char* prefix = "{\"Source\":\"Process\",\"LogEntry\":{\"Logline\":\""; - const char* suffix = "\"},\"SchemaVersion\":\"1.0.0\"}\n"; +size_t FormatProcessLog(char* chBuf) { + if (Utility::CompareWStrings(loggingformat, L"Custom")) { + return FormatCustomLog(chBuf); + } else { + return FormatStandardLog(chBuf); + } +} + +/// +/// Helper function to format the custom log. +/// +size_t FormatCustomLog(char* chBuf) { + ProcessLogEntry logEntry; + SYSTEMTIME st; + GetSystemTime(&st); + + char chBufCpy[BUFSIZE] = ""; + size_t chBufLen = BufferCopyAndSanitize(chBufCpy, chBuf); + + logEntry.source = L"Process"; + logEntry.currentTime = Utility::SystemTimeToString(st).c_str(); + + std::wstring_convert, wchar_t> fromBytesconverter; + logEntry.logLine = fromBytesconverter.from_bytes(chBufCpy); + + std::wstring_convert> toBytesconverter; + std::wstring formattedLog = Utility::FormatEventLineLog(processCustomLogFormat, &logEntry, logEntry.source); + std::string convertedLog = toBytesconverter.to_bytes(formattedLog); + std::string str = convertedLog + "\n"; + + const char* logLine = str.c_str(); + size_t logLineLen = strlen(logLine); + + return BufferCopy(chBuf, const_cast(logLine), 0, logLineLen); +} + +/// +/// Helper function to format the standard log (JSON or XML). +/// +size_t FormatStandardLog(char* chBuf) { + const char* prefix; + const char* suffix; + + if (Utility::CompareWStrings(loggingformat, L"XML")) { + prefix = "Process"; + suffix = "\n"; + } else { + prefix = "{\"Source\":\"Process\",\"LogEntry\":{\"Logline\":\""; + suffix = "\"},\"SchemaVersion\":\"1.0.0\"}\n"; + } char chBufCpy[BUFSIZE] = ""; // // copy valid (>0 ASCII values) bytes from chBuf to chBufCpy // - size_t chBufLen = bufferCopyAndSanitize(chBufCpy, chBuf); + size_t chBufLen = BufferCopyAndSanitize(chBufCpy, chBuf); size_t prefixLen = strlen(prefix); size_t suffixLen = strlen(suffix); - size_t index = bufferCopy(chBuf, const_cast(prefix), 0, prefixLen); + size_t index = BufferCopy(chBuf, const_cast(prefix), 0, prefixLen); - // copy over the logline after prefix - // index increments from the previous index within bufferCopy - index = bufferCopy(chBuf, chBufCpy, index); + index = BufferCopy(chBuf, chBufCpy, index); // truncate, in the unlikely event of a long logline > |BUFSIZE-85| - // leave at least 36 slots to close the JSON with `..."},\"SchemaVersion\":\"1.0.0\"}\n` + // leave at least 36 slots for JSON or 21 slots for XML // reset the start index if ((index + suffixLen) > BUFSIZE - 5) { index = BUFSIZE - 5 - suffixLen; - suffix = "...\"},\"SchemaVersion\":\"1.0.0\"}\n"; + if (Utility::CompareWStrings(loggingformat, L"XML")) + { + suffix = "...\\n"; + } else { + suffix = "...\"},\"SchemaVersion\":\"1.0.0\"}\n"; + } } - index = bufferCopy(chBuf, const_cast(suffix), index, index + suffixLen); - - return index; // same as the number of bytes read + return BufferCopy(chBuf, const_cast(suffix), index, index + suffixLen); } /// /// Helper function to clear the stdout buffer /// return number of bytes cleared /// -size_t clearBuffer(char* chBuf) { +size_t ClearBuffer(char* chBuf) { size_t count = 0; char* ptr = chBuf; @@ -303,7 +353,7 @@ DWORD ReadFromPipe(LPVOID Param) for (;;) { // clear buffer ready for read - clearBuffer(chBuf); + ClearBuffer(chBuf); // move valid chars from remainder buffer to chBuf // then ReadFile starts from the end position (chBuf + cnt) char* ptrRem = chBufRem; @@ -315,7 +365,7 @@ DWORD ReadFromPipe(LPVOID Param) bSuccess = ReadFile(g_hChildStd_OUT_Rd, chBuf + cnt, BUFSIZE - cnt, &dwRead, NULL); // clear remainder buffer for the next read - clearBuffer(chBufRem); + ClearBuffer(chBufRem); if (!bSuccess || dwRead == 0) { @@ -327,7 +377,7 @@ DWORD ReadFromPipe(LPVOID Param) size_t outSz = 0; size_t count = 0; size_t lastNewline = 0; - clearBuffer(chBufOut); + ClearBuffer(chBufOut); while (*ptr != '\0' && count < BUFSIZE) { // copy over to chBufOut till \r\n // the remaining will be reserved to be completed @@ -335,7 +385,7 @@ DWORD ReadFromPipe(LPVOID Param) if (*ptr == '\r' || *ptr == '\n') { if (outSz > 0) { // print out and reset chBufOut and outSz - size_t sz = formatProcessLog(chBufOut); + size_t sz = FormatProcessLog(chBufOut); DWORD dwRead = static_cast(sz); bSuccess = logWriter.WriteLog( @@ -346,7 +396,7 @@ DWORD ReadFromPipe(LPVOID Param) NULL); // reset outSz = 0; - clearBuffer(chBufOut); + ClearBuffer(chBufOut); lastNewline = count; } } @@ -374,3 +424,16 @@ DWORD ReadFromPipe(LPVOID Param) return ERROR_SUCCESS; } + +std::wstring ProcessMonitor::ProcessFieldsMapping(_In_ std::wstring fileFields, _In_ void* pLogEntryData) +{ + std::wostringstream oss; + ProcessLogEntry* pLogEntry = (ProcessLogEntry*)pLogEntryData; + + if (Utility::CompareWStrings(fileFields, L"TimeStamp")) oss << pLogEntry->currentTime; + if (Utility::CompareWStrings(fileFields, L"Source")) oss << pLogEntry->source; + if (Utility::CompareWStrings(fileFields, L"logLine") + || Utility::CompareWStrings(fileFields, L"logEntry")) oss << pLogEntry->logLine; + + return oss.str(); +} diff --git a/LogMonitor/src/LogMonitor/ProcessMonitor.h b/LogMonitor/src/LogMonitor/ProcessMonitor.h index 42d4c8e6..5b0ee408 100644 --- a/LogMonitor/src/LogMonitor/ProcessMonitor.h +++ b/LogMonitor/src/LogMonitor/ProcessMonitor.h @@ -5,5 +5,34 @@ #pragma once -DWORD CreateAndMonitorProcess(std::wstring& Cmdline); +struct ProcessLogEntry { + std::wstring source; + std::wstring currentTime; + std::wstring logLine; +}; +DWORD CreateAndMonitorProcess(std::wstring& Cmdline, std::wstring LogFormat, std::wstring ProcessCustomLogFormat); + +DWORD CreateChildProcess(std::wstring& Cmdline); + +static DWORD ReadFromPipe(LPVOID Param); + +static size_t ClearBuffer(char* chBuf); + +size_t FormatProcessLog(char* chBuf); + +size_t FormatCustomLog(char* chBuf); + +size_t FormatStandardLog(char* chBuf); + +static size_t BufferCopy(char* dst, char* src, size_t start, size_t end); + +static size_t BufferCopyAndSanitize(char* dst, char* src); + +class ProcessMonitor final +{ +public: + ProcessMonitor(); + + static std::wstring ProcessFieldsMapping(_In_ std::wstring eventFields, _In_ void* pLogEntryData); +}; diff --git a/LogMonitor/src/LogMonitor/Utility.cpp b/LogMonitor/src/LogMonitor/Utility.cpp index f5b08e34..d1b8737c 100644 --- a/LogMonitor/src/LogMonitor/Utility.cpp +++ b/LogMonitor/src/LogMonitor/Utility.cpp @@ -268,12 +268,16 @@ void Utility::SanitizeJson(_Inout_ std::wstring& str) size_t i = 0; while (i < str.size()) { auto sub = str.substr(i, 1); + auto s = str.substr(0, i + 1); if (sub == L"\"") { - if ((i > 0 && str.substr(i - 1, 1) != L"\\") + if ((i > 0 && str.substr(i - 1, 1) != L"\\" && str.substr(i - 1, 1) != L"~") || i == 0) { str.replace(i, 1, L"\\\""); i++; + } else if (i > 0 && str.substr(i - 1, 1) == L"~") { + str.replace(i - 1, 1, L""); + i--; } } else if (sub == L"\\") { @@ -343,3 +347,96 @@ int Utility::GetWaitInterval(_In_ std::double_t waitInSeconds, _In_ int elapsedT const auto remainingTime = static_cast(waitInSeconds - elapsedTime); return remainingTime <= WAIT_INTERVAL ? remainingTime : WAIT_INTERVAL; } + +/// +/// Comparing wstrings with ignoring the case +/// +/// +/// +/// +/// +bool Utility::CompareWStrings(wstring stringA, wstring stringB) +{ + return stringA.size() == stringB.size() && + equal( + stringA.cbegin(), + stringA.cend(), + stringB.cbegin(), + [](wstring::value_type l1, wstring::value_type r1) { + return towupper(l1) == towupper(r1); + } + ); +} + +std::wstring Utility::FormatEventLineLog( + _In_ std::wstring customLogFormat, + _In_ void* pLogEntry, + _In_ std::wstring sourceType +) +{ + bool customJsonFormat = IsCustomJsonFormat(customLogFormat); + + size_t i = 0, j = 1; + while (i < customLogFormat.size()) { + auto sub = customLogFormat.substr(i, j - i); + auto sub_length = sub.size(); + + bool startsWithPercent = sub[0] == '%'; + bool endsWithPercent = sub[sub_length - 1] == '%'; + + if (!startsWithPercent && !endsWithPercent) { + j++, i++; + } else if (startsWithPercent && endsWithPercent && sub_length > 1) { + // Valid field name found in custom log format + wstring fieldValue; + auto fieldName = sub.substr(1, sub_length - 2); + if (sourceType == L"ETW") { + fieldValue = EtwMonitor::EtwFieldsMapping(fieldName, pLogEntry); + } else if (sourceType == L"EventLog") { + fieldValue = EventMonitor::EventFieldsMapping(fieldName, pLogEntry); + } else if (sourceType == L"File") { + fieldValue = LogFileMonitor::FileFieldsMapping(fieldName, pLogEntry); + } else if (sourceType == L"Process") { + fieldValue = ProcessMonitor::ProcessFieldsMapping(fieldName, pLogEntry); + } + // Substitute the field name with value + customLogFormat.replace(i, sub_length, fieldValue); + + i += fieldValue.length(); + j = i + 1; + } else { + j++; + } + } + + if(customJsonFormat) + SanitizeJson(customLogFormat); + + return customLogFormat; +} + +/// +/// check if custom format specified in config is JSON for sanitization purposes +/// +/// +/// +bool Utility::IsCustomJsonFormat(_Inout_ std::wstring& customLogFormat) +{ + bool isCustomJSONFormat = false; + + auto npos = customLogFormat.find_last_of(L"|"); + std::wstring substr; + if (npos != std::string::npos) { + substr = customLogFormat.substr(npos + 1); + substr.erase(std::remove(substr.begin(), substr.end(), ' '), substr.end()); + + if (!substr.empty() && CompareWStrings(substr, L"JSON")) { + customLogFormat = ReplaceAll(customLogFormat, L"'", L"~\""); + isCustomJSONFormat = true; + } + + customLogFormat = customLogFormat.substr(0, customLogFormat.find_last_of(L"|")); + } + return isCustomJSONFormat; +} + diff --git a/LogMonitor/src/LogMonitor/Utility.h b/LogMonitor/src/LogMonitor/Utility.h index db469152..11d0d36f 100644 --- a/LogMonitor/src/LogMonitor/Utility.h +++ b/LogMonitor/src/LogMonitor/Utility.h @@ -77,4 +77,17 @@ class Utility final static int GetWaitInterval( _In_ std::double_t waitInSeconds, _In_ int elapsedTime); + + static bool CompareWStrings( + _In_ std::wstring stringA, + _In_ std::wstring stringB + ); + + static std::wstring FormatEventLineLog( + _In_ std::wstring customLogFormat, + _In_ void* pLogEntry, + _In_ std::wstring sourceType + ); + + static bool IsCustomJsonFormat(_Inout_ std::wstring& customLogFormat); }; diff --git a/LogMonitor/src/LogMonitor/sample-config-files/IIS/LogMonitorConfig.json b/LogMonitor/src/LogMonitor/sample-config-files/IIS/LogMonitorConfig.json index 36f9c238..19c671b1 100644 --- a/LogMonitor/src/LogMonitor/sample-config-files/IIS/LogMonitorConfig.json +++ b/LogMonitor/src/LogMonitor/sample-config-files/IIS/LogMonitorConfig.json @@ -1,5 +1,6 @@ { "LogConfig": { + "logFormat": "json", "sources": [ { "type": "EventLog", diff --git a/LogMonitor/src/LogMonitor/version.h b/LogMonitor/src/LogMonitor/version.h index db61de8d..e29cdc2e 100644 --- a/LogMonitor/src/LogMonitor/version.h +++ b/LogMonitor/src/LogMonitor/version.h @@ -7,8 +7,8 @@ #define _VERSION_H_ #define LM_MAJORNUMBER 2 -#define LM_MINORNUMBER 0 -#define LM_PATCHNUMBER 2 +#define LM_MINORNUMBER 1 +#define LM_PATCHNUMBER 0 // removed in support of semantic versioning - https://semver.org // major.minor.patch // #define LM_BUILDMINORVERSION 0