Add testing for DataLoader (#727)

* Add testing for DataLoader in PA
triton-inference-server · Jul 25, 2024 · 442915d · 442915d
1 parent c514dea
commit 442915d
Show file tree

Hide file tree

Showing 3 changed files with 196 additions and 1 deletion.
diff --git a/src/c++/perf_analyzer/perf_utils.h b/src/c++/perf_analyzer/perf_utils.h
@@ -56,7 +56,7 @@ constexpr uint64_t NANOS_PER_MILLIS = 1000000;
 
 // Will use the characters specified here to construct random strings
 std::string const character_set =
- "abcdefghijklmnaoqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890 .?!";
+ "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890 .?!";
 
 // A boolean flag to mark an interrupt and commencement of early exit
 extern volatile bool early_exit;

diff --git a/src/c++/perf_analyzer/test_dataloader.cc b/src/c++/perf_analyzer/test_dataloader.cc
@@ -28,6 +28,7 @@
 #include "doctest.h"
 #include "mock_data_loader.h"
 
+
 namespace triton { namespace perfanalyzer {
 
 /// Helper class for testing the DataLoader
@@ -104,6 +105,199 @@ TEST_CASE("dataloader: GetTotalSteps")
  CHECK_EQ(dataloader.GetTotalSteps(2), 0);
 }
 
+TEST_CASE("dataloader: ValidateIOExistsInModel")
+{
+ MockDataLoader dataloader;
+ std::shared_ptr<ModelTensorMap> inputs = std::make_shared<ModelTensorMap>();
+ std::shared_ptr<ModelTensorMap> outputs = std::make_shared<ModelTensorMap>();
+ ModelTensor input1 = TestDataLoader::CreateTensor("INPUT1");
+ ModelTensor output1 = TestDataLoader::CreateTensor("OUTPUT1");
+ inputs->insert(std::make_pair(input1.name_, input1));
+ outputs->insert(std::make_pair(output1.name_, output1));
+
+ SUBCASE("Directory does not exist")
+ {
+ std::string data_directory = "non_existent_directory";
+ cb::Error status =
+ dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+ CHECK(
+ status.Message() ==
+ "Error: Directory does not exist or is not a directory: "
+ "non_existent_directory");
+ CHECK(status.Err() == pa::GENERIC_ERROR);
+ }
+
+ SUBCASE("Directory is not a directory")
+ {
+ std::string data_directory = "tmp/test.txt";
+ std::ofstream file(data_directory);
+ cb::Error status =
+ dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+ CHECK(
+ status.Message() ==
+ "Error: Directory does not exist or is not a directory: tmp/test.txt");
+ CHECK(status.Err() == pa::GENERIC_ERROR);
+ std::remove(data_directory.c_str());
+ }
+
+ SUBCASE("Valid directory but no corresponding files")
+ {
+ std::string data_directory = "valid_directory";
+ std::filesystem::create_directory(data_directory);
+ std::ofstream(data_directory + "/invalid_file").close();
+ cb::Error status =
+ dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+ std::filesystem::remove_all(data_directory);
+ CHECK(
+ status.Message() ==
+ "Provided data file 'invalid_file' does not correspond to a valid "
+ "model input or output.");
+ CHECK(status.Err() == pa::GENERIC_ERROR);
+ }
+
+ SUBCASE("Valid directory with corresponding files")
+ {
+ std::string data_directory = "valid_directory";
+ std::filesystem::create_directory(data_directory);
+ std::ofstream(data_directory + "/INPUT1").close();
+ std::ofstream(data_directory + "/OUTPUT1").close();
+ cb::Error status =
+ dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+ std::filesystem::remove_all(data_directory);
+ CHECK(status.Message().empty());
+ CHECK(status.IsOk());
+ }
+
+ SUBCASE("Valid directory with multiple input and output tensors")
+ {
+ ModelTensor input2 = TestDataLoader::CreateTensor("INPUT2");
+ ModelTensor output2 = TestDataLoader::CreateTensor("OUTPUT2");
+
+ inputs->insert(std::make_pair(input2.name_, input2));
+ outputs->insert(std::make_pair(output2.name_, output2));
+
+ std::string data_directory = "valid_directory_multiple";
+ std::filesystem::create_directory(data_directory);
+ std::ofstream(data_directory + "/INPUT1").close();
+ std::ofstream(data_directory + "/INPUT2").close();
+ std::ofstream(data_directory + "/OUTPUT1").close();
+ std::ofstream(data_directory + "/OUTPUT2").close();
+
+ cb::Error status =
+ dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+ std::filesystem::remove_all(data_directory);
+ CHECK(status.Message().empty());
+ CHECK(status.IsOk());
+ }
+}
+
+TEST_CASE("dataloader: ReadDataFromJSON")
+{
+ DataLoader dataloader;
+ std::shared_ptr<ModelTensorMap> inputs = std::make_shared<ModelTensorMap>();
+ std::shared_ptr<ModelTensorMap> outputs = std::make_shared<ModelTensorMap>();
+ ModelTensor input1 = TestDataLoader::CreateTensor("INPUT1");
+ ModelTensor output1 = TestDataLoader::CreateTensor("OUTPUT1");
+
+ inputs->insert(std::make_pair(input1.name_, input1));
+ outputs->insert(std::make_pair(output1.name_, output1));
+
+ SUBCASE("File does not exist")
+ {
+ std::string json_file = "non_existent_file.json";
+ cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+ CHECK(status.Message() == "failed to open file for reading provided data");
+ CHECK(status.Err() == pa::GENERIC_ERROR);
+ }
+
+ SUBCASE("Valid JSON file")
+ {
+ std::string json_file = "valid_file.json";
+ std::ofstream out(json_file);
+ out << R"({
+ "data": [
+ { "INPUT1": [1] },
+ { "INPUT1": [2] },
+ { "INPUT1": [3] }
+ ],
+ "validation_data": [
+ { "OUTPUT1": [4] },
+ { "OUTPUT1": [5] },
+ { "OUTPUT1": [6] }
+ ]})";
+ out.close();
+
+ cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+ std::filesystem::remove(json_file);
+ CHECK(status.Message().empty());
+ CHECK(status.IsOk());
+ }
+
+ SUBCASE("Invalid JSON file")
+ {
+ std::string json_file = "invalid_file.json";
+ std::ofstream out(json_file);
+ out << R"({invalid_json: 1,)";
+ out.close();
+
+ cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+ std::filesystem::remove(json_file);
+
+ CHECK(
+ status.Message() ==
+ "failed to parse the specified json file for reading provided data");
+ CHECK(status.Err() == pa::GENERIC_ERROR);
+ }
+
+ SUBCASE("Multiple input and output tensors")
+ {
+ ModelTensor input2 = TestDataLoader::CreateTensor("INPUT2");
+ ModelTensor output2 = TestDataLoader::CreateTensor("OUTPUT2");
+
+ inputs->insert(std::make_pair(input2.name_, input2));
+ outputs->insert(std::make_pair(output2.name_, output2));
+
+ std::string json_file = "valid_file_multiple_input_output.json";
+ std::ofstream out(json_file);
+ out << R"({
+ "data": [
+ {
+ "INPUT1": [1],
+ "INPUT2": [4]
+ },
+ {
+ "INPUT1": [2],
+ "INPUT2": [5]
+ },
+ {
+ "INPUT1": [3],
+ "INPUT2": [6]
+ }
+ ],
+ "validation_data": [
+ {
+ "OUTPUT1": [4],
+ "OUTPUT2": [7]
+ },
+ {
+ "OUTPUT1": [5],
+ "OUTPUT2": [8]
+ },
+ {
+ "OUTPUT1": [6],
+ "OUTPUT2": [9]
+ }
+ ]
+ })";
+ out.close();
+
+ cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+ std::filesystem::remove(json_file);
+ CHECK(status.Message().empty());
+ CHECK(status.IsOk());
+ }
+}
+
 TEST_CASE("dataloader: GetInputData missing data")
 {
  MockDataLoader dataloader;

diff --git a/src/c++/perf_analyzer/test_perf_utils.cc b/src/c++/perf_analyzer/test_perf_utils.cc
@@ -144,6 +144,7 @@ TEST_CASE("perf_utils: ConvertDTypeFromTFS")
  std::make_pair("DT_DOUBLE", "FP64"),
  std::make_pair("DT_INT32", "INT32"),
  std::make_pair("DT_INT16", "INT16"),
+ std::make_pair("DT_UINT16", "UINT16"),
  std::make_pair("DT_INT8", "INT8"),
  std::make_pair("DT_UINT8", "UINT8"),
  std::make_pair("DT_STRING", "BYTES"),