diff --git a/components/tests/integration_guide.md b/components/tests/integration_guide.md index c771cd47c4..a21c7d436a 100644 --- a/components/tests/integration_guide.md +++ b/components/tests/integration_guide.md @@ -1,10 +1,9 @@ # Guide for Using Integration Test -When you want to start a new integration test, then you need to have following a few steps to set up the test. +1. Write a JSON file under the folder `/data/unit_test/`, you can refer to those existing JSON files to learn how to write it. +2. Put the mesh for your test under the folder `/data/unit_test/meshes`. Please check first, if one of the existing meshes is suitable for your integration test. +3. Add the JSON file's name to `/components/tests/integration_test.cpp`, i.e. add the line `WMTK_INTEGRATION("YOUR TEST NAME", false);`. +4. The first time you run your integration test, set the `DO_VALIDATION` parameter `false`, i.e. `WMTK_INTEGRATION("YOUR TEST NAME", false);`. The integration test will then add the test output (number of simplices) to your JSON. Afterward, set the parameter to `true`. Now, the integration test will compare the result with your original output and fail if the two do not match. +5. Push the JSON and meshes to the [wmtk data repo](https://github.com/wildmeshing/data). -- First, write a JSON file under the folder `/data/unit_test/`, you can refer to those existing JSON files to learn how to write it. -- Second, you need the model to run your integration test. Put your model under the folder `/data/unit_test/meshes`.(you may have to update the wmtkdata) -- Third, after you have done the steps above, you should add the JSON file's name to `/components/tests/integration_test.cpp`. Such as `WMTK_INTEGRATION("YOUR TEST NAME", false);` -- Fourth, for the first time you run your integration test, you should set `DO_VALIDATION` parameter `true` in `WMTK_INTEGRATION`. Then `integraion_test.cpp` will generate the original performance of your code. After that, you should set the parameter as `false`. In the later integration test, `integraion_test.cpp` will compare the result with your original performance to make sure your new version code's performance always consistent with the original one. - -NOTE: you can have a look at the `integrations_test.cpp` to get more details if you want. \ No newline at end of file +Look at `integrations_test.cpp` for more details. diff --git a/components/tests/integration_test.cpp b/components/tests/integration_test.cpp index 9ac44c4155..f636f051e3 100644 --- a/components/tests/integration_test.cpp +++ b/components/tests/integration_test.cpp @@ -14,6 +14,9 @@ using json = nlohmann::json; using namespace wmtk; + +enum IntegrationTestResult { Success = 0, Fail = 1, InvalidInput = 2 }; + namespace { bool load_json(const std::string& json_file, json& out) @@ -29,9 +32,13 @@ bool load_json(const std::string& json_file, json& out) bool contains_results(const json& in_args) { + if (!in_args.contains("tests")) { + return false; + } + const auto& tests = in_args["tests"]; - for (const auto& type : {"vertices", "edges", "faces", "tetrahedra"}) { - if (!(tests.contains(type) && tests[type].is_number())) { + for (const auto& type : {"meshes", "vertices", "edges", "faces", "tetrahedra"}) { + if (!tests.contains(type) || !tests[type].is_array()) { return false; } } @@ -43,27 +50,17 @@ bool missing_tests_data(const json& j) return !j.contains("tests") || !j.at("tests").contains("meshes"); } -int authenticate_json(const std::string& json_file, const bool compute_validation) +IntegrationTestResult authenticate_json(const std::string& json_file, const bool compute_validation) { json in_args; if (!load_json(json_file, in_args)) { - spdlog::error("unable to open {} file", json_file); - return 1; + wmtk::logger().error("unable to open {} file", json_file); + return IntegrationTestResult::InvalidInput; } - if (missing_tests_data(in_args)) { - spdlog::error("JSON file missing \"tests\" or meshes key."); - return 1; - } in_args["root_path"] = json_file; - if (compute_validation && !contains_results(in_args)) { - spdlog::error("JSON file missing vertices edges faces or tetrahedra or meshes key. Add a * " - "to the beginning of filename to allow appends."); - return 2; - } - // in_args["settings"] = R"({ // "log_level": 5, // "opt_log_level": 5 @@ -72,26 +69,32 @@ int authenticate_json(const std::string& json_file, const bool compute_validatio utils::set_random_seed(0); auto cache = wmtk::components::run_components(in_args, true); - auto meshes = in_args["tests"]["meshes"]; - if (!compute_validation) { - spdlog::info("Authenticating..."); + if (missing_tests_data(in_args)) { + wmtk::logger().error("JSON file missing \"tests\" or meshes key."); + return IntegrationTestResult::InvalidInput; + } + + REQUIRE(contains_results(in_args)); + + wmtk::logger().info("Authenticating..."); + + const std::vector meshes = in_args["tests"]["meshes"]; if (in_args["tests"].contains("skip_check") && in_args["tests"]["skip_check"]) { - spdlog::warn("Skpping checks for {}", json_file); - return 0; + wmtk::logger().warn("Skpping checks for {}", json_file); + return IntegrationTestResult::Success; } - auto vertices = in_args["tests"]["vertices"]; - auto edges = in_args["tests"]["edges"]; - auto faces = in_args["tests"]["faces"]; - auto tetrahedra = in_args["tests"]["tetrahedra"]; + const auto vertices = in_args["tests"]["vertices"]; + const auto edges = in_args["tests"]["edges"]; + const auto faces = in_args["tests"]["faces"]; + const auto tetrahedra = in_args["tests"]["tetrahedra"]; if (meshes.size() != vertices.size() || meshes.size() != edges.size() || meshes.size() != faces.size() || meshes.size() != tetrahedra.size()) { - spdlog::error("JSON size missmatch between meshes and vertices edges faces or " - "tetrahedra or meshes key. Add a * " - "to the beginning of filename to allow appends."); - return 2; + wmtk::logger().error( + "JSON size missmatch between meshes and vertices, edges, faces, or tetrahedra."); + return IntegrationTestResult::Fail; } for (int64_t i = 0; i < meshes.size(); ++i) { @@ -108,32 +111,53 @@ int authenticate_json(const std::string& json_file, const bool compute_validatio const int64_t n_tetrahedra = mesh->get_all(PrimitiveType::Tetrahedron).size(); if (n_vertices != expected_vertices) { - spdlog::error( - "Violating Authenticate for vertices {}!={}", + wmtk::logger().error( + "Violating Authenticate in mesh `{}` for vertices {} != {}", + meshes[i], n_vertices, expected_vertices); - return 2; + return IntegrationTestResult::Fail; } if (n_edges != expected_edges) { - spdlog::error("Violating Authenticate for edges {}!={}", n_edges, expected_edges); - return 2; + wmtk::logger().error( + "Violating Authenticate in mesh `{}` for edges {} != {}", + meshes[i], + n_edges, + expected_edges); + return IntegrationTestResult::Fail; } if (n_faces != expected_faces) { - spdlog::error("Violating Authenticate for faces {}!={}", n_faces, expected_faces); - return 2; + wmtk::logger().error( + "Violating Authenticate in mesh `{}` for faces {} != {}", + meshes[i], + n_faces, + expected_faces); + return IntegrationTestResult::Fail; } if (n_tetrahedra != expected_tetrahedra) { - spdlog::error( - "Violating Authenticate for tetrahedra {}!={}", + wmtk::logger().error( + "Violating Authenticate in mesh `{}` for tetrahedra {} != {}", + meshes[i], n_tetrahedra, expected_tetrahedra); - return 2; + return IntegrationTestResult::Fail; } } - spdlog::info("Authenticated ✅"); + wmtk::logger().info("Authenticated ✅"); } else { - spdlog::warn("Appending JSON..."); + if (contains_results(in_args)) { + wmtk::logger().error( + "JSON file contains results even though the test was run in `computation " + "mode`. Set DO_VALIDATION to false to compare with saved results or remove " + "results from JSON to re-compute them."); + return IntegrationTestResult::Fail; + } + + wmtk::logger().warn("Appending JSON..."); + + const std::vector meshes = cache.mesh_names(); + in_args["tests"]["meshes"] = meshes; std::vector expected_vertices, expected_edges, expected_faces, expected_tetrahedra; @@ -163,7 +187,7 @@ int authenticate_json(const std::string& json_file, const bool compute_validatio file << in_args; } - return 0; + return IntegrationTestResult::Success; } } // namespace namespace { @@ -179,9 +203,9 @@ std::string tagsrun = "[.][integration]"; { \ std::string path = std::string("unit_test/") + NAME + ".json"; \ bool compute_validation = DO_VALIDATION; \ - spdlog::info("Processing {}", NAME); \ + wmtk::logger().info("Processing {}", NAME); \ auto flag = authenticate_json(WMTK_DATA_DIR "/" + path, compute_validation); \ - REQUIRE(flag == 0); \ + REQUIRE(flag == IntegrationTestResult::Success); \ } diff --git a/components/wmtk_components/main/wmtk/components/run_components.cpp b/components/wmtk_components/main/wmtk/components/run_components.cpp index bb8114106f..314750dd0c 100644 --- a/components/wmtk_components/main/wmtk/components/run_components.cpp +++ b/components/wmtk_components/main/wmtk/components/run_components.cpp @@ -2,6 +2,7 @@ #include #include +#include #include @@ -81,6 +82,7 @@ wmtk::io::Cache run_components(const nlohmann::json& json_input_file, bool stric for (const nlohmann::json& component_json : spec_json["components"]) { for (auto& el : component_json.items()) { wmtk::logger().info("Component {}", el.key()); + wmtk::utils::StopWatch sw(el.key()); components[el.key()](paths, el.value(), cache); cache.flush_multimeshes(); } diff --git a/src/wmtk/io/Cache.cpp b/src/wmtk/io/Cache.cpp index f6d42667af..75048d4538 100644 --- a/src/wmtk/io/Cache.cpp +++ b/src/wmtk/io/Cache.cpp @@ -180,6 +180,16 @@ void Cache::flush_multimeshes() } } +std::vector Cache::mesh_names() +{ + std::vector names; + for (const auto& fp : m_file_paths) { + names.emplace_back(fp.first); + } + + return names; +} + void Cache::write_mesh( const Mesh& m, const std::string& name, diff --git a/src/wmtk/io/Cache.hpp b/src/wmtk/io/Cache.hpp index 7c7dcc448d..baad7f259a 100644 --- a/src/wmtk/io/Cache.hpp +++ b/src/wmtk/io/Cache.hpp @@ -147,6 +147,11 @@ class Cache /// Unsets the mesh held by each cached mm - useful for debugging whether cache loading works void flush_multimeshes(); + /** + * @brief Get all names of the meshes stored in cache. + */ + std::vector mesh_names(); + private: std::filesystem::path m_cache_dir; std::map m_file_paths; // name --> file location diff --git a/src/wmtk/utils/CMakeLists.txt b/src/wmtk/utils/CMakeLists.txt index 0981817bd9..206eb1535f 100644 --- a/src/wmtk/utils/CMakeLists.txt +++ b/src/wmtk/utils/CMakeLists.txt @@ -61,6 +61,9 @@ set(SRC_FILES EigenMatrixWriter.hpp EigenMatrixWriter.cpp + + Stopwatch.hpp + Stopwatch.cpp ) target_sources(wildmeshing_toolkit PRIVATE ${SRC_FILES}) diff --git a/src/wmtk/utils/Stopwatch.cpp b/src/wmtk/utils/Stopwatch.cpp new file mode 100644 index 0000000000..bc61c23e8c --- /dev/null +++ b/src/wmtk/utils/Stopwatch.cpp @@ -0,0 +1,60 @@ +#include "Stopwatch.hpp" + + +#include + +namespace wmtk::utils { + + +StopWatch::StopWatch(const std::string& name) + : StopWatch(name, spdlog::level::info) +{} + +StopWatch::StopWatch(const std::string& name, const spdlog::level::level_enum log_level) + : m_name(name) + , m_log_level(log_level) +{ + start(); +} + +StopWatch::~StopWatch() +{ + if (m_is_running) { + stop(); + log_msg(); + } +} + +void StopWatch::start() +{ + m_is_running = true; + m_start = std::chrono::high_resolution_clock::now(); +} + +void StopWatch::stop() +{ + if (!m_is_running) { + return; + } + m_is_running = false; + m_stop = std::chrono::high_resolution_clock::now(); +} + +inline void StopWatch::log_msg() +{ + int64_t h = getElapsedTime(); + int64_t m = getElapsedTime() % 60; + int64_t s = getElapsedTime() % 60; + int64_t ms = getElapsedTime() % 1000; + + logger().log( + m_log_level, + "[runtime h:m:s.ms] {}: {:02d}:{:02d}:{:02d}.{:03d}", + m_name, + h, + m, + s, + ms); +} + +} // namespace wmtk::utils \ No newline at end of file diff --git a/src/wmtk/utils/Stopwatch.hpp b/src/wmtk/utils/Stopwatch.hpp new file mode 100644 index 0000000000..4f5826496e --- /dev/null +++ b/src/wmtk/utils/Stopwatch.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include + +namespace wmtk::utils { + +class StopWatch +{ +public: + StopWatch(const std::string& name); + StopWatch(const std::string& name, const spdlog::level::level_enum log_level); + + virtual ~StopWatch(); + + void start(); + void stop(); + + template + int64_t getElapsedTime(); + + void log_msg(); + + +private: + std::string m_name; + spdlog::level::level_enum m_log_level = spdlog::level::info; + std::chrono::high_resolution_clock::time_point m_start; + std::chrono::high_resolution_clock::time_point m_stop; + + bool m_is_running = false; +}; + +template +inline int64_t StopWatch::getElapsedTime() +{ + auto duration = std::chrono::duration_cast(m_stop - m_start); + return duration.count(); +} + +} // namespace wmtk::utils \ No newline at end of file