Merge branch 'release/0.31.0'

* release/0.31.0: (31 commits)
  Version 0.31.0
  Remove redundant code after previous commit
  Add FunctionSpace::gather and FunctionSpace::scatter
  Make Cells::global_index() using StructuredMeshGenerator for RegularLonLatGrid independent of partitioning
  Use OpenMP in MatchingMeshPartitionerLonLatPolygon
  Improve performance of BuildHalo, using unordered_map and OpenMP
  Output max threads in Library print
  Add OpenMP support to computing bounds in StructuredColumns::generate_region
  Add progress timer to StructuredColumns::generate_region
  Fix building with ATLAS_BITS_LOCAL=64
  Reduce memory peak in GatherScatter::setup
  Fix compilation warning
  Add missing this%return() in atlas_Trace_module, but likely no impact as mostly unused subroutines
  Improve performance of ATLAS_TRACE by improving hashing function
  Add interface to adjoint of direct spectral transform implemented via ectrans (#113)
  Fix compilation with ATLAS_BITS_LOCAL=64
  Fortran interfaces for PointCloud functionspace halo exchange (#112)
  Extend PointCloud functionspace to do halo exchanges (#111)
  Implement IndexViev::assign( initializer_list ) ; for now only with NativeIndexView
  Fix Github Actions missing brew
  ...

.github/workflows/build.yml

@@ -98,6 +98,10 @@ jobs:
     - name: Checkout Repository
       uses: actions/checkout@v2
 
+    - name: Set up Homebrew
+      id: set-up-homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+
     - name: Environment
       run:  |
         echo "DEPS_DIR=${{ runner.temp }}/deps" >> $GITHUB_ENV

CHANGELOG.md

@@ -7,6 +7,25 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 
 ## [Unreleased]
 
+## [0.31.0] - 2022-11-10
+### Added
+- Extend PointCloud functionspace to do halo exchanges, including Fortran API
+- Add FunctionSpace::gather and FunctionSpace::scatter abstraction
+
+### Changed
+- Improve performance of MatchingMeshPartitionerLonLatPolygon using OpenMP
+- Improve performance of BuildHalo using OpenMP and unordered_map
+- Improve performance of StructuredMeshGenerator using OpenMP
+- Improve performance of ATLAS_TRACE
+- Reduce memory peak in GatherScatter setup
+- Global element numbering of RegularLonLat grids is now following rows independent of partitioning
+
+### Fixed
+- Running CI with Github Actions
+- Fix output of atlas-grids y-range for shifted grids
+- Fix building with ATLAS_BITS_LOCAL=64
+
+
 ## [0.30.0] - 2022-08-22
 ### Added
 - Fortran API for Interpolation::execute_adjoint()
@@ -386,6 +405,7 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 ## 0.13.0 - 2018-02-16
 
 [Unreleased]: https://github.com/ecmwf/atlas/compare/master...develop
+[0.31.0]: https://github.com/ecmwf/atlas/compare/0.30.0...0.31.0
 [0.30.0]: https://github.com/ecmwf/atlas/compare/0.29.0...0.30.0
 [0.29.0]: https://github.com/ecmwf/atlas/compare/0.28.1...0.29.0
 [0.28.1]: https://github.com/ecmwf/atlas/compare/0.28.0...0.28.1

VERSION

@@ -1 +1 @@
-0.30.0
+0.31.0

atlas_io/src/atlas_io/Data.h

@@ -23,7 +23,7 @@ class Data {
 public:
     Data() = default;
     Data(void*, size_t);
-    Data(Data&&)  = default;
+    Data(Data&&)            = default;
     Data& operator=(Data&&) = default;
 
     operator const void*() const { return data(); }

atlas_io/src/atlas_io/Metadata.h

@@ -83,6 +83,14 @@ class Metadata : public eckit::LocalConfiguration {
         root.remove(name);
         return *this;
     }
+
+
+    std::vector<std::string> keys() const {
+        // Preserves order of keys
+        std::vector<std::string> result;
+        eckit::fromValue(result, get().keys());
+        return result;
+    }
 };
 
 //---------------------------------------------------------------------------------------------------------------------

atlas_io/src/atlas_io/Record.cc

@@ -132,6 +132,20 @@ Record::operator const ParsedRecord&() const {
 
 //---------------------------------------------------------------------------------------------------------------------
 
+static void parse_record(ParsedRecord& record, const std::string& key, const Metadata& metadata) {
+    if (metadata.type() || metadata.link()) {
+        record.items.emplace(key, metadata);
+        record.keys.emplace_back(key);
+    }
+    else {
+        for (auto& next_key : metadata.keys()) {
+            parse_record(record, key + "." + next_key, metadata.getSubConfiguration(next_key));
+        }
+    }
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
 Record& Record::read(Stream& in, bool read_to_end) {
     if (not empty()) {
         return *this;
@@ -206,9 +220,9 @@ Record& Record::read(Stream& in, bool read_to_end) {
 
     ATLAS_IO_ASSERT(r.metadata_format == "yaml");
     Metadata metadata = eckit::YAMLConfiguration(metadata_str);
-    record_->keys     = metadata.keys();
-    for (const auto& key : record_->keys) {
-        record_->items.emplace(key, metadata.getSubConfiguration(key));
+
+    for (auto& key : metadata.keys()) {
+        parse_record(*record_, key, metadata.getSubConfiguration(key));
     }
 
 

atlas_io/src/atlas_io/RecordWriter.h

@@ -24,6 +24,9 @@
 #include "atlas_io/detail/TypeTraits.h"
 
 #include "atlas_io/types/array/ArrayReference.h"
+#include "atlas_io/types/scalar.h"
+#include "atlas_io/types/string.h"
+
 
 #include "atlas_io/detail/Defaults.h"
 
@@ -88,6 +91,14 @@ class RecordWriter {
         set(key, Encoder{value}, config);
     }
 
+    void set(const Key& key, const char* value, const eckit::Configuration& config = NoConfig()) {
+        set(key, Encoder{std::string(value)}, config);
+    }
+
+    void set(const Key& key, const std::string& value, const eckit::Configuration& config = NoConfig()) {
+        set(key, Encoder{std::string(value)}, config);
+    }
+
     /// @brief Write new record to path
     size_t write(const eckit::PathName&, Mode = Mode::write) const;
 

atlas_io/src/atlas_io/Trace.cc

@@ -16,20 +16,26 @@ namespace atlas {
 namespace io {
 
 atlas::io::Trace::Trace(const eckit::CodeLocation& loc) {
-    for (auto& hook : TraceHookRegistry::instance().hooks) {
-        hooks_.emplace_back(hook(loc, loc.func()));
+    for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
+        if (TraceHookRegistry::enabled(id)) {
+            hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, loc.func()));
+        }
     }
 }
 
 Trace::Trace(const eckit::CodeLocation& loc, const std::string& title) {
-    for (auto& hook : TraceHookRegistry::instance().hooks) {
-        hooks_.emplace_back(hook(loc, title));
+    for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
+        if (TraceHookRegistry::enabled(id)) {
+            hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title));
+        }
     }
 }
 
 Trace::Trace(const eckit::CodeLocation& loc, const std::string& title, const Labels& labels) {
-    for (auto& hook : TraceHookRegistry::instance().hooks) {
-        hooks_.emplace_back(hook(loc, title));
+    for (size_t id = 0; id < TraceHookRegistry::size(); ++id) {
+        if (TraceHookRegistry::enabled(id)) {
+            hooks_.emplace_back(TraceHookRegistry::hook(id)(loc, title));
+        }
     }
 }
 

atlas_io/src/atlas_io/Trace.h

@@ -31,12 +31,26 @@ struct TraceHook {
 struct TraceHookRegistry {
     using TraceHookBuilder = std::function<std::unique_ptr<TraceHook>(const eckit::CodeLocation&, const std::string&)>;
     std::vector<TraceHookBuilder> hooks;
+    std::vector<int> enabled_;
     static TraceHookRegistry& instance() {
         static TraceHookRegistry instance;
         return instance;
     }
-    static void add(TraceHookBuilder&& hook) { instance().hooks.emplace_back(hook); }
-    static void add(const TraceHookBuilder& hook) { instance().hooks.emplace_back(hook); }
+    static size_t add(TraceHookBuilder&& hook) {
+        instance().hooks.emplace_back(hook);
+        instance().enabled_.emplace_back(true);
+        return instance().hooks.size() - 1;
+    }
+    static size_t add(const TraceHookBuilder& hook) {
+        instance().hooks.emplace_back(hook);
+        instance().enabled_.emplace_back(true);
+        return instance().hooks.size() - 1;
+    }
+    static void enable(size_t id) { instance().enabled_[id] = true; }
+    static void disable(size_t id) { instance().enabled_[id] = false; }
+    static bool enabled(size_t id) { return instance().enabled_[id]; }
+    static size_t size() { return instance().hooks.size(); }
+    static TraceHookBuilder& hook(size_t id) { return instance().hooks[id]; }
 
 private:
     TraceHookRegistry() = default;

atlas_io/src/atlas_io/detail/Encoder.h

@@ -38,6 +38,10 @@ class Encoder {
 
     Encoder(Encoder&& other): self_(std::move(other.self_)) {}
 
+    template <typename T, enable_if_scalar_t<T> = 0>
+    explicit Encoder(const T& x): self_(new EncodableValue<T>(x)) {}
+
+
     Encoder& operator=(Encoder&& rhs) {
         self_ = std::move(rhs.self_);
         return *this;
@@ -61,6 +65,11 @@ class Encoder {
     struct EncodableValue : Encodable {
         EncodableValue(Value&& v): value_{std::move(v)} { sfinae::encode_metadata(value_, metadata_, data_size_); }
 
+        template <bool EnableBool = true, enable_if_scalar_t<Value, EnableBool> = 0>
+        EncodableValue(const Value& v): value_{v} {
+            sfinae::encode_metadata(value_, metadata_, data_size_);
+        }
+
         size_t encode_metadata_(atlas::io::Metadata& metadata) const override {
             metadata.set(metadata_);
             return data_size_;

atlas_io/src/atlas_io/detail/TypeTraits.h

@@ -12,6 +12,7 @@
 
 #include <type_traits>
 
+#include "atlas_io/detail/DataType.h"
 
 namespace atlas {
 namespace io {
@@ -134,6 +135,20 @@ template <typename T>
 using enable_if_move_constructible_decodable_rvalue_t =
     enable_if_t<is_decodable<T>() && std::is_rvalue_reference<T&&>() && std::is_move_constructible<T>()>;
 
+template <typename T, bool EnableBool = true>
+using enable_if_scalar_t = enable_if_t<std::is_scalar<T>::value && EnableBool>;
+
+
+template <typename T>
+constexpr bool is_array_datatype() {
+    return std::is_same<T, double>::value || std::is_same<T, float>::value ||
+                                std::is_same<T, int>::value || std::is_same<T, long>::value ||
+                                std::is_same<T, size_t>::value || std::is_same<T, std::byte>::value;
+}
+
+template <typename T>
+using enable_if_array_datatype = typename std::enable_if<is_array_datatype<T>(), int>::type;
+
 
 }  // namespace io
 }  // namespace atlas

atlas_io/src/atlas_io/print/JSONFormat.cc

@@ -52,11 +52,13 @@ void JSONFormat::print(std::ostream& out) const {
         }
 
         if (print_details_) {
-            m.set("data.compression.type", item.data.compression());
-            m.set("data.compression.size", item.data.compressed_size());
-            m.set("data.size", item.data.size());
-            m.set("data.byte_order", (item.data.endian() == Endian::little) ? "little endian" : "big endian");
-            m.set("data.checksum", item.data.checksum().str());
+            if (item.data.size()) {
+                m.set("data.compression.type", item.data.compression());
+                m.set("data.compression.size", item.data.compressed_size());
+                m.set("data.size", item.data.size());
+                m.set("data.byte_order", (item.data.endian() == Endian::little) ? "little endian" : "big endian");
+                m.set("data.checksum", item.data.checksum().str());
+            }
             m.set("version", item.record.version().str());
             m.set("created", item.record.created().str());
         }

atlas_io/src/atlas_io/print/TableFormat.cc

@@ -135,7 +135,6 @@ class ScalarMetadataPrettyPrint : public MetadataPrettyPrintBase {
         std::string type = metadata_.getString("type");
         ATLAS_IO_ASSERT(type == "scalar");
         std::string datatype = metadata_.getString("datatype");
-        std::string base64   = metadata_.getString("base64");
         out << std::setw(7) << std::left << datatype << ": ";
         if (datatype == DataType::str<double>()) {
             out << decode<double>();

atlas_io/src/atlas_io/types/array/ArrayReference.cc

@@ -75,8 +75,8 @@ ArrayReference::ArrayReference(ArrayReference&& other): ArrayMetadata(std::move(
 
 ArrayReference& ArrayReference::operator=(ArrayReference&& rhs) {
     ArrayMetadata::operator=(std::move(rhs));
-    data_                  = rhs.data_;
-    rhs.data_              = nullptr;
+    data_     = rhs.data_;
+    rhs.data_ = nullptr;
     return *this;
 }
 

atlas_io/src/atlas_io/types/array/adaptors/StdVectorAdaptor.h

@@ -15,22 +15,23 @@
 #include "atlas_io/Data.h"
 #include "atlas_io/Exceptions.h"
 #include "atlas_io/Metadata.h"
+#include "atlas_io/detail/TypeTraits.h"
 #include "atlas_io/types/array/ArrayMetadata.h"
 #include "atlas_io/types/array/ArrayReference.h"
 
 namespace std {
 
 //---------------------------------------------------------------------------------------------------------------------
 
-template <typename T>
+template <typename T, atlas::io::enable_if_array_datatype<T> = 0>
 void interprete(const std::vector<T>& vector, atlas::io::ArrayReference& out) {
     using atlas::io::ArrayReference;
     out = ArrayReference{vector.data(), {int(vector.size())}};
 }
 
 //---------------------------------------------------------------------------------------------------------------------
 
-template <typename T>
+template <typename T, atlas::io::enable_if_array_datatype<T> = 0>
 void decode(const atlas::io::Metadata& m, const atlas::io::Data& encoded, std::vector<T>& out) {
     atlas::io::ArrayMetadata array(m);
     if (array.datatype().kind() != atlas::io::ArrayMetadata::DataType::kind<T>()) {

atlas_io/src/atlas_io/types/scalar.cc

@@ -135,16 +135,16 @@ void encode_data(const double&, atlas::io::Data&) {}
 //---------------------------------------------------------------------------------------------------------------------
 
 void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, int& value) {
-    decode_scalar_b64(metadata, value);
+    decode_scalar(metadata, value);
 }
 void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, long& value) {
-    decode_scalar_b64(metadata, value);
+    decode_scalar(metadata, value);
 }
 void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, long long& value) {
-    decode_scalar_b64(metadata, value);
+    decode_scalar(metadata, value);
 }
 void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, unsigned long& value) {
-    decode_scalar_b64(metadata, value);
+    decode_scalar(metadata, value);
 }
 void decode(const atlas::io::Metadata& metadata, const atlas::io::Data&, unsigned long long& value) {
     decode_scalar_b64(metadata, value);

atlas_io/tests/test_io_encoding.cc

@@ -635,11 +635,21 @@ void test_encode_decode_scalar() {
 
 CASE("Encode/Decode scalar") {
     // bit identical encoding via Base64 string within the metadata!
-    SECTION("int32") { test_encode_decode_scalar<std::int32_t>(); }
-    SECTION("int64") { test_encode_decode_scalar<std::int64_t>(); }
-    SECTION("real32") { test_encode_decode_scalar<float>(); }
-    SECTION("real64") { test_encode_decode_scalar<double>(); }
-    SECTION("uint64") { test_encode_decode_scalar<std::uint64_t>(); }
+    SECTION("int32") {
+        test_encode_decode_scalar<std::int32_t>();
+    }
+    SECTION("int64") {
+        test_encode_decode_scalar<std::int64_t>();
+    }
+    SECTION("real32") {
+        test_encode_decode_scalar<float>();
+    }
+    SECTION("real64") {
+        test_encode_decode_scalar<double>();
+    }
+    SECTION("uint64") {
+        test_encode_decode_scalar<std::uint64_t>();
+    }
 }
 
 // -------------------------------------------------------------------------------------------------------

atlas_io/tests/test_io_record.cc

@@ -137,9 +137,15 @@ CASE("Write records, each in separate file (offset=0)") {
         write_length(length, path + ".length");
     };
 
-    SECTION("record1.atlas" + suffix()) { write_record(globals::record1.data, "record1.atlas" + suffix()); }
-    SECTION("record2.atlas" + suffix()) { write_record(globals::record2.data, "record2.atlas" + suffix()); }
-    SECTION("record3.atlas" + suffix()) { write_record(globals::record3.data, "record3.atlas" + suffix()); }
+    SECTION("record1.atlas" + suffix()) {
+        write_record(globals::record1.data, "record1.atlas" + suffix());
+    }
+    SECTION("record2.atlas" + suffix()) {
+        write_record(globals::record2.data, "record2.atlas" + suffix());
+    }
+    SECTION("record3.atlas" + suffix()) {
+        write_record(globals::record3.data, "record3.atlas" + suffix());
+    }
 }
 
 //-----------------------------------------------------------------------------

cmake/features/TRANS.cmake

@@ -2,7 +2,7 @@
 
 set( atlas_HAVE_ECTRANS 0 )
 if( ENABLE_TRANS OR NOT DEFINED ENABLE_TRANS )
-    find_package( ectrans 1.0 COMPONENTS transi double QUIET )
+    find_package( ectrans 1.1 COMPONENTS transi double QUIET )
     if( TARGET transi_dp )
         set( transi_FOUND TRUE )
         if( NOT TARGET transi )

src/apps/atlas-grids.cc

@@ -282,9 +282,10 @@ int AtlasGrids::execute(const Args& args) {
                     Log::info() << "   x : [ " << std::setw(10) << std::fixed << structuredgrid.xspace().min() << " , "
                                 << std::setw(10) << std::fixed << structuredgrid.xspace().max() << " ] deg"
                                 << std::endl;
-                    Log::info() << "   y : [ " << std::setw(10) << std::fixed << structuredgrid.yspace().min() << " , "
-                                << std::setw(10) << std::fixed << structuredgrid.yspace().max() << " ] deg"
-                                << std::endl;
+                    double ymin = std::min(structuredgrid.yspace().front(), structuredgrid.yspace().back());
+                    double ymax = std::max(structuredgrid.yspace().front(), structuredgrid.yspace().back());
+                    Log::info() << "   y : [ " << std::setw(10) << std::fixed << ymin << " , " << std::setw(10)
+                                << std::fixed << ymax << " ] deg" << std::endl;
                 }
                 auto it = grid.lonlat().begin();
                 Log::info() << "   lonlat(first)     : " << *it << std::endl;