fix: JSON truncation caused by escaped zero byte (#1594)

Signed-off-by: Tao Yu <[email protected]>
alibaba · Jul 9, 2024 · 4e25935 · 4e25935
1 parent 71394f0
commit 4e25935
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 2 deletions.
diff --git a/core/processor/ProcessorParseJsonNative.cpp b/core/processor/ProcessorParseJsonNative.cpp
@@ -176,7 +176,7 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent,
 
 std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Value& value) {
  if (value.IsString())
- return value.GetString();
+ return std::string(value.GetString(), value.GetStringLength());
  else if (value.IsBool())
  return ToString(value.GetBool());
  else if (value.IsInt())
@@ -196,7 +196,7 @@ std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Va
  rapidjson::StringBuffer buffer;
  rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
  value.Accept(writer);
- return buffer.GetString();
+ return std::string(buffer.GetString(), buffer.GetLength());
  }
 }
 

diff --git a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp
@@ -29,6 +29,7 @@ class ProcessorParseJsonNativeUnittest : public ::testing::Test {
 
  void TestInit();
  void TestProcessJson();
+ void TestProcessJsonEscapedNullByte();
  void TestAddLog();
  void TestProcessEventKeepUnmatch();
  void TestProcessEventDiscardUnmatch();
@@ -45,6 +46,8 @@ UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestAddLog);
 
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJson);
 
+UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonEscapedNullByte);
+
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventKeepUnmatch);
 
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventDiscardUnmatch);
@@ -257,6 +260,63 @@ void ProcessorParseJsonNativeUnittest::TestAddLog() {
  processor.GetContext().GetProcessProfile().logGroupSize);
 }
 
+void ProcessorParseJsonNativeUnittest::TestProcessJsonEscapedNullByte() {
+ // make config
+ Json::Value config;
+ config["SourceKey"] = "content";
+ config["KeepingSourceWhenParseFail"] = true;
+ config["KeepingSourceWhenParseSucceed"] = false;
+ config["CopingRawLog"] = false;
+ config["RenamedSourceKey"] = "rawLog";
+
+ // make events
+ auto sourceBuffer = std::make_shared<SourceBuffer>();
+ PipelineEventGroup eventGroup(sourceBuffer);
+ std::string inJson = R"({
+ "events" :
+ [
+ {
+ "contents" :
+ {
+ "content" : "{\"level\":\"ERROR\",\"time\":\"2024-07-04T06:59:23.078Z\",\"msg\":\"expect { or n, but found \\u0000, error found in #0 byte of ...\"}"
+ },
+ "timestampNanosecond" : 0,
+ "timestamp" : 12345678901,
+ "type" : 1
+ }
+ ]
+ })";
+ eventGroup.FromJsonString(inJson);
+ // run function
+ ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative);
+ std::string pluginId = "testID";
+ ProcessorInstance processorInstance(&processor, pluginId);
+ APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
+ std::vector<PipelineEventGroup> eventGroupList;
+ eventGroupList.emplace_back(std::move(eventGroup));
+ processorInstance.Process(eventGroupList);
+ // judge result
+ std::string expectJson = R"({
+ "events" :
+ [
+ {
+ "contents" :
+ {
+ "level": "ERROR",
+ "msg": "expect { or n, but found \u0000, error found in #0 byte of ...",
+ "time": "2024-07-04T06:59:23.078Z"
+ },
+ "timestamp" : 12345678901,
+ "timestampNanosecond" : 0,
+ "type" : 1
+ }
+ ]
+ })";
+ std::string outJson = eventGroupList[0].ToJsonString();
+ APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
+ APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0));
+}
+
 void ProcessorParseJsonNativeUnittest::TestProcessJson() {
  // make config
  Json::Value config;