Merge pull request #784 from wildmeshing/dzint/fixing_integration_tests

Dzint/fixing integration tests

components/tests/integration_guide.md

@@ -1,10 +1,9 @@
 # Guide for Using Integration Test
 
-When you want to start a new integration test, then you need to have following a few steps to set up the test.
+1. Write a JSON file under the folder `/data/unit_test/`, you can refer to those existing JSON files to learn how to write it.
+2. Put the mesh for your test under the folder `/data/unit_test/meshes`. Please check first, if one of the existing meshes is suitable for your integration test.
+3. Add the JSON file's name to `/components/tests/integration_test.cpp`, i.e. add the line `WMTK_INTEGRATION("YOUR TEST NAME", false);`.
+4. The first time you run your integration test, set the `DO_VALIDATION` parameter `false`, i.e. `WMTK_INTEGRATION("YOUR TEST NAME", false);`. The integration test will then add the test output (number of simplices) to your JSON. Afterward, set the parameter to `true`. Now, the integration test will compare the result with your original output and fail if the two do not match.
+5. Push the JSON and meshes to the [wmtk data repo](https://github.com/wildmeshing/data).
 
-- First, write a JSON file under the folder `/data/unit_test/`, you can refer to those existing JSON files to learn how to write it.
-- Second, you need the model to run your integration test. Put your model under the folder `/data/unit_test/meshes`.(you may have to update the wmtkdata)
-- Third, after you have done the steps above, you should add the JSON file's name to `/components/tests/integration_test.cpp`. Such as `WMTK_INTEGRATION("YOUR TEST NAME", false);`
-- Fourth, for the first time you run your integration test, you should set `DO_VALIDATION` parameter `true` in `WMTK_INTEGRATION`. Then `integraion_test.cpp` will generate the original performance of your code. After that, you should set the parameter as `false`. In the later integration test, `integraion_test.cpp` will compare the result with your original performance to make sure your new version code's performance always consistent with the original one.
-
-NOTE: you can have a look at the `integrations_test.cpp` to get more details if you want.
+Look at `integrations_test.cpp` for more details.

components/tests/integration_test.cpp

@@ -14,6 +14,9 @@
 
 using json = nlohmann::json;
 using namespace wmtk;
+
+enum IntegrationTestResult { Success = 0, Fail = 1, InvalidInput = 2 };
+
 namespace {
 
 bool load_json(const std::string& json_file, json& out)
@@ -29,9 +32,13 @@ bool load_json(const std::string& json_file, json& out)
 
 bool contains_results(const json& in_args)
 {
+    if (!in_args.contains("tests")) {
+        return false;
+    }
+
     const auto& tests = in_args["tests"];
-    for (const auto& type : {"vertices", "edges", "faces", "tetrahedra"}) {
-        if (!(tests.contains(type) && tests[type].is_number())) {
+    for (const auto& type : {"meshes", "vertices", "edges", "faces", "tetrahedra"}) {
+        if (!tests.contains(type) || !tests[type].is_array()) {
             return false;
         }
     }
@@ -43,27 +50,17 @@ bool missing_tests_data(const json& j)
     return !j.contains("tests") || !j.at("tests").contains("meshes");
 }
 
-int authenticate_json(const std::string& json_file, const bool compute_validation)
+IntegrationTestResult authenticate_json(const std::string& json_file, const bool compute_validation)
 {
     json in_args;
     if (!load_json(json_file, in_args)) {
-        spdlog::error("unable to open {} file", json_file);
-        return 1;
+        wmtk::logger().error("unable to open {} file", json_file);
+        return IntegrationTestResult::InvalidInput;
     }
 
-    if (missing_tests_data(in_args)) {
-        spdlog::error("JSON file missing \"tests\" or meshes key.");
-        return 1;
-    }
     in_args["root_path"] = json_file;
 
 
-    if (compute_validation && !contains_results(in_args)) {
-        spdlog::error("JSON file missing vertices edges faces or tetrahedra or meshes key. Add a * "
-                      "to the beginning of filename to allow appends.");
-        return 2;
-    }
-
     // in_args["settings"] = R"({
     //     "log_level": 5,
     //     "opt_log_level": 5
@@ -72,26 +69,32 @@ int authenticate_json(const std::string& json_file, const bool compute_validatio
     utils::set_random_seed(0);
     auto cache = wmtk::components::run_components(in_args, true);
 
-    auto meshes = in_args["tests"]["meshes"];
-
     if (!compute_validation) {
-        spdlog::info("Authenticating...");
+        if (missing_tests_data(in_args)) {
+            wmtk::logger().error("JSON file missing \"tests\" or meshes key.");
+            return IntegrationTestResult::InvalidInput;
+        }
+
+        REQUIRE(contains_results(in_args));
+
+        wmtk::logger().info("Authenticating...");
+
+        const std::vector<std::string> meshes = in_args["tests"]["meshes"];
 
         if (in_args["tests"].contains("skip_check") && in_args["tests"]["skip_check"]) {
-            spdlog::warn("Skpping checks for {}", json_file);
-            return 0;
+            wmtk::logger().warn("Skpping checks for {}", json_file);
+            return IntegrationTestResult::Success;
         }
 
-        auto vertices = in_args["tests"]["vertices"];
-        auto edges = in_args["tests"]["edges"];
-        auto faces = in_args["tests"]["faces"];
-        auto tetrahedra = in_args["tests"]["tetrahedra"];
+        const auto vertices = in_args["tests"]["vertices"];
+        const auto edges = in_args["tests"]["edges"];
+        const auto faces = in_args["tests"]["faces"];
+        const auto tetrahedra = in_args["tests"]["tetrahedra"];
         if (meshes.size() != vertices.size() || meshes.size() != edges.size() ||
             meshes.size() != faces.size() || meshes.size() != tetrahedra.size()) {
-            spdlog::error("JSON size missmatch between meshes and vertices edges faces or "
-                          "tetrahedra or meshes key. Add a * "
-                          "to the beginning of filename to allow appends.");
-            return 2;
+            wmtk::logger().error(
+                "JSON size missmatch between meshes and vertices, edges, faces, or tetrahedra.");
+            return IntegrationTestResult::Fail;
         }
 
         for (int64_t i = 0; i < meshes.size(); ++i) {
@@ -108,32 +111,53 @@ int authenticate_json(const std::string& json_file, const bool compute_validatio
             const int64_t n_tetrahedra = mesh->get_all(PrimitiveType::Tetrahedron).size();
 
             if (n_vertices != expected_vertices) {
-                spdlog::error(
-                    "Violating Authenticate for vertices {}!={}",
+                wmtk::logger().error(
+                    "Violating Authenticate in mesh `{}` for vertices {} != {}",
+                    meshes[i],
                     n_vertices,
                     expected_vertices);
-                return 2;
+                return IntegrationTestResult::Fail;
             }
             if (n_edges != expected_edges) {
-                spdlog::error("Violating Authenticate for edges {}!={}", n_edges, expected_edges);
-                return 2;
+                wmtk::logger().error(
+                    "Violating Authenticate in mesh `{}` for edges {} != {}",
+                    meshes[i],
+                    n_edges,
+                    expected_edges);
+                return IntegrationTestResult::Fail;
             }
             if (n_faces != expected_faces) {
-                spdlog::error("Violating Authenticate for faces {}!={}", n_faces, expected_faces);
-                return 2;
+                wmtk::logger().error(
+                    "Violating Authenticate in mesh `{}` for faces {} != {}",
+                    meshes[i],
+                    n_faces,
+                    expected_faces);
+                return IntegrationTestResult::Fail;
             }
             if (n_tetrahedra != expected_tetrahedra) {
-                spdlog::error(
-                    "Violating Authenticate for tetrahedra {}!={}",
+                wmtk::logger().error(
+                    "Violating Authenticate in mesh `{}` for tetrahedra {} != {}",
+                    meshes[i],
                     n_tetrahedra,
                     expected_tetrahedra);
-                return 2;
+                return IntegrationTestResult::Fail;
             }
         }
 
-        spdlog::info("Authenticated ✅");
+        wmtk::logger().info("Authenticated ✅");
     } else {
-        spdlog::warn("Appending JSON...");
+        if (contains_results(in_args)) {
+            wmtk::logger().error(
+                "JSON file contains results even though the test was run in `computation "
+                "mode`. Set DO_VALIDATION to false to compare with saved results or remove "
+                "results from JSON to re-compute them.");
+            return IntegrationTestResult::Fail;
+        }
+
+        wmtk::logger().warn("Appending JSON...");
+
+        const std::vector<std::string> meshes = cache.mesh_names();
+        in_args["tests"]["meshes"] = meshes;
 
         std::vector<int64_t> expected_vertices, expected_edges, expected_faces, expected_tetrahedra;
 
@@ -163,7 +187,7 @@ int authenticate_json(const std::string& json_file, const bool compute_validatio
         file << in_args;
     }
 
-    return 0;
+    return IntegrationTestResult::Success;
 }
 } // namespace
 namespace {
@@ -179,9 +203,9 @@ std::string tagsrun = "[.][integration]";
     {                                                                                \
         std::string path = std::string("unit_test/") + NAME + ".json";               \
         bool compute_validation = DO_VALIDATION;                                     \
-        spdlog::info("Processing {}", NAME);                                         \
+        wmtk::logger().info("Processing {}", NAME);                                  \
         auto flag = authenticate_json(WMTK_DATA_DIR "/" + path, compute_validation); \
-        REQUIRE(flag == 0);                                                          \
+        REQUIRE(flag == IntegrationTestResult::Success);                             \
     }
 
 

components/wmtk_components/main/wmtk/components/run_components.cpp

@@ -2,6 +2,7 @@
 
 #include <jse/jse.h>
 #include <wmtk/utils/Logger.hpp>
+#include <wmtk/utils/Stopwatch.hpp>
 
 
 #include <wmtk/components/base/Paths.hpp>
@@ -81,6 +82,7 @@ wmtk::io::Cache run_components(const nlohmann::json& json_input_file, bool stric
     for (const nlohmann::json& component_json : spec_json["components"]) {
         for (auto& el : component_json.items()) {
             wmtk::logger().info("Component {}", el.key());
+            wmtk::utils::StopWatch sw(el.key());
             components[el.key()](paths, el.value(), cache);
             cache.flush_multimeshes();
         }

src/wmtk/io/Cache.cpp

@@ -180,6 +180,16 @@ void Cache::flush_multimeshes()
     }
 }
 
+std::vector<std::string> Cache::mesh_names()
+{
+    std::vector<std::string> names;
+    for (const auto& fp : m_file_paths) {
+        names.emplace_back(fp.first);
+    }
+
+    return names;
+}
+
 void Cache::write_mesh(
     const Mesh& m,
     const std::string& name,

src/wmtk/io/Cache.hpp

@@ -147,6 +147,11 @@ class Cache
     /// Unsets the mesh held by each cached mm - useful for debugging whether cache loading works
     void flush_multimeshes();
 
+    /**
+     * @brief Get all names of the meshes stored in cache.
+     */
+    std::vector<std::string> mesh_names();
+
 private:
     std::filesystem::path m_cache_dir;
     std::map<std::string, std::filesystem::path> m_file_paths; // name --> file location

src/wmtk/utils/CMakeLists.txt

@@ -61,6 +61,9 @@ set(SRC_FILES
 
     EigenMatrixWriter.hpp
     EigenMatrixWriter.cpp
+
+    Stopwatch.hpp
+    Stopwatch.cpp
 )
 target_sources(wildmeshing_toolkit PRIVATE ${SRC_FILES})
 

src/wmtk/utils/Stopwatch.cpp

@@ -0,0 +1,60 @@
+#include "Stopwatch.hpp"
+
+
+#include <spdlog/common.h>
+
+namespace wmtk::utils {
+
+
+StopWatch::StopWatch(const std::string& name)
+    : StopWatch(name, spdlog::level::info)
+{}
+
+StopWatch::StopWatch(const std::string& name, const spdlog::level::level_enum log_level)
+    : m_name(name)
+    , m_log_level(log_level)
+{
+    start();
+}
+
+StopWatch::~StopWatch()
+{
+    if (m_is_running) {
+        stop();
+        log_msg();
+    }
+}
+
+void StopWatch::start()
+{
+    m_is_running = true;
+    m_start = std::chrono::high_resolution_clock::now();
+}
+
+void StopWatch::stop()
+{
+    if (!m_is_running) {
+        return;
+    }
+    m_is_running = false;
+    m_stop = std::chrono::high_resolution_clock::now();
+}
+
+inline void StopWatch::log_msg()
+{
+    int64_t h = getElapsedTime<std::chrono::hours>();
+    int64_t m = getElapsedTime<std::chrono::minutes>() % 60;
+    int64_t s = getElapsedTime<std::chrono::seconds>() % 60;
+    int64_t ms = getElapsedTime<std::chrono::milliseconds>() % 1000;
+
+    logger().log(
+        m_log_level,
+        "[runtime h:m:s.ms] {}: {:02d}:{:02d}:{:02d}.{:03d}",
+        m_name,
+        h,
+        m,
+        s,
+        ms);
+}
+
+} // namespace wmtk::utils

src/wmtk/utils/Stopwatch.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <spdlog/common.h>
+#include <chrono>
+#include <type_traits>
+#include <wmtk/utils/Logger.hpp>
+
+namespace wmtk::utils {
+
+class StopWatch
+{
+public:
+    StopWatch(const std::string& name);
+    StopWatch(const std::string& name, const spdlog::level::level_enum log_level);
+
+    virtual ~StopWatch();
+
+    void start();
+    void stop();
+
+    template <typename T = std::chrono::seconds>
+    int64_t getElapsedTime();
+
+    void log_msg();
+
+
+private:
+    std::string m_name;
+    spdlog::level::level_enum m_log_level = spdlog::level::info;
+    std::chrono::high_resolution_clock::time_point m_start;
+    std::chrono::high_resolution_clock::time_point m_stop;
+
+    bool m_is_running = false;
+};
+
+template <typename T>
+inline int64_t StopWatch::getElapsedTime()
+{
+    auto duration = std::chrono::duration_cast<T>(m_stop - m_start);
+    return duration.count();
+}
+
+} // namespace wmtk::utils