GBFS: Performance/Memory Improvements + Vehicle Types (#670)

* gbfs: partition providers, faster updates, less memory usage

* fix vehicle_docks_available

* slightly faster geofencing zone mapping

* rename provider_cache -> provider_file_infos

* ts formatting

* formatting

* clang fixes

* clang fix

* clang fixes

* trying to fix apple clang

* static_cast all the things

* review changes

* fix include

* 65k gbfs providers should be enough

* remove obsolete comment

* sort vehicle status before diff

* fill cache during gbfs update

* cleanup

* partition vehicle types by form factor + propulsion type

* rental api changes (RENTAL mode + form factor + propulsion type)

* ui fix

* rename provider segment -> products

* vehicle type id -> idx

* rename more segment -> products

* one more r-tree to speed up geofencing zone mapping

* clang fix

* fix api descriptions

* api: add rental provider filter

* ui: fix direct connection display for rental connections

* return constraint support, allow roundtrip for direct connections

* ui formatting

* fix missing gbfs data after update

* http proxy support

* share decompressed bitvecs between routing requests

* fix missing initializer

---------

Co-authored-by: Felix Gündling <[email protected]>

.pkg

@@ -5,7 +5,7 @@
 [cista]
   [email protected]:felixguendling/cista.git
   branch=master
-  commit=950f96f4ded53a6b5753824b280550b722933e55
+  commit=6362f3ad8c3133a0abf64e5d8c9ea3e21f531ee8
 [osr]
   [email protected]:motis-project/osr.git
   branch=master
@@ -54,3 +54,7 @@
   [email protected]:motis-project/mimalloc.git
   branch=dev
   commit=e2f4fe647e8aff4603a7d5119b8639fd1a47c8a6
+[lz4]
+  [email protected]:motis-project/lz4.git
+  branch=dev
+  commit=c4765545ebb14b0a56c663e21923166923f8280e

.pkg.lock

@@ -1,8 +1,9 @@
-11944721483929486774
-cista 847b27100b7e730370b810ce62206a66b0bf2d79
+11919117500806763401
+cista 6362f3ad8c3133a0abf64e5d8c9ea3e21f531ee8
 zlib-ng 68ab3e2d80253ec5dc3c83691d9ff70477b32cd3
 boost 930f38eb0365ceb7853273e03da4d9e7787abfb9
 googletest 7b64fca6ea0833628d6f86255a81424365f7cc0c
+lz4 c4765545ebb14b0a56c663e21923166923f8280e
 mimalloc e2f4fe647e8aff4603a7d5119b8639fd1a47c8a6
 libressl 24acd9e710fbe842e863572da9d738715fbc74b8
 fmt dc10f83be70ac2873d5f8d1ce317596f1fd318a2

CMakeLists.txt

@@ -121,6 +121,8 @@ target_link_libraries(motislib
     pbf_sdf_fonts_res-res
     ssl
     crypto
+    tg
+    lz4_static
 )
 
 
@@ -144,7 +146,6 @@ target_link_libraries(motis
     rtree
     geo
     cista
-    tg
     ianatzdb-res
     pbf_sdf_fonts_res-res
     tiles_server_res-res

include/motis/box_rtree.h

@@ -0,0 +1,110 @@
+#pragma once
+
+#include <array>
+
+#include "cista/strong.h"
+
+#include "rtree.h"
+
+#include "geo/box.h"
+#include "geo/latlng.h"
+
+namespace motis {
+
+template <typename T, typename Fn>
+concept BoxRtreePosHandler = requires(geo::box const& b, T const x, Fn&& f) {
+  { std::forward<Fn>(f)(b, x) };
+};
+
+template <typename T>
+struct box_rtree {
+  box_rtree() : rtree_{rtree_new()} {}
+
+  ~box_rtree() {
+    if (rtree_ != nullptr) {
+      rtree_free(rtree_);
+    }
+  }
+
+  box_rtree(box_rtree const& o) {
+    if (this != &o) {
+      if (rtree_ != nullptr) {
+        rtree_free(rtree_);
+      }
+      rtree_ = rtree_clone(o.rtree_);
+    }
+  }
+
+  box_rtree(box_rtree&& o) {
+    if (this != &o) {
+      rtree_ = o.rtree_;
+      o.rtree_ = nullptr;
+    }
+  }
+
+  box_rtree& operator=(box_rtree const& o) {
+    if (this != &o) {
+      if (rtree_ != nullptr) {
+        rtree_free(rtree_);
+      }
+      rtree_ = rtree_clone(o.rtree_);
+    }
+    return *this;
+  }
+
+  box_rtree& operator=(box_rtree&& o) {
+    if (this != &o) {
+      rtree_ = o.rtree_;
+      o.rtree_ = nullptr;
+    }
+    return *this;
+  }
+
+  void add(geo::box const& b, T const t) {
+    auto const min_corner = b.min_.lnglat();
+    auto const max_corner = b.max_.lnglat();
+    rtree_insert(
+        rtree_, min_corner.data(), max_corner.data(),
+        reinterpret_cast<void*>(static_cast<std::size_t>(cista::to_idx(t))));
+  }
+
+  void remove(geo::box const& b, T const t) {
+    auto const min_corner = b.min_.lnglat();
+    auto const max_corner = b.max_.lnglat();
+    rtree_delete(
+        rtree_, min_corner.data(), max_corner.data(),
+        reinterpret_cast<void*>(static_cast<std::size_t>(cista::to_idx(t))));
+  }
+
+  template <typename Fn>
+  void find(geo::box const& b, Fn&& fn) const {
+    auto const min = b.min_.lnglat();
+    auto const max = b.max_.lnglat();
+    rtree_search(
+        rtree_, min.data(), max.data(),
+        [](double const* min_corner, double const* max_corner, void const* item,
+           void* udata) {
+          if constexpr (BoxRtreePosHandler<T, Fn>) {
+            (*reinterpret_cast<Fn*>(udata))(
+                geo::box{geo::latlng{min_corner[1], min_corner[0]},
+                         geo::latlng{max_corner[1], max_corner[0]}},
+                T{static_cast<cista::base_t<T>>(
+                    reinterpret_cast<std::size_t>(item))});
+          } else {
+            (*reinterpret_cast<Fn*>(udata))(T{static_cast<cista::base_t<T>>(
+                reinterpret_cast<std::size_t>(item))});
+          }
+          return true;
+        },
+        &fn);
+  }
+
+  template <typename Fn>
+  void find(geo::latlng const& pos, Fn&& fn) const {
+    return find(geo::box{pos, pos}, std::forward<Fn>(fn));
+  }
+
+  rtree* rtree_{nullptr};
+};
+
+}  // namespace motis

include/motis/config.h

@@ -114,6 +114,8 @@ struct config {
     std::map<std::string, restrictions> default_restrictions_{};
     unsigned update_interval_{60};
     unsigned http_timeout_{10};
+    unsigned cache_size_{50};
+    std::optional<std::string> proxy_{};
   };
   std::optional<gbfs> gbfs_{};
 

include/motis/endpoints/routing.h

@@ -1,5 +1,9 @@
 #pragma once
 
+#include <optional>
+#include <utility>
+#include <vector>
+
 #include "osr/location.h"
 #include "osr/types.h"
 
@@ -19,10 +23,13 @@ struct routing {
       osr::location const&,
       osr::direction,
       std::vector<api::ModeEnum> const&,
+      std::optional<std::vector<api::RentalFormFactorEnum>> const&,
+      std::optional<std::vector<api::RentalPropulsionTypeEnum>> const&,
+      std::optional<std::vector<std::string>> const& rental_providers,
       bool wheelchair,
       std::chrono::seconds max,
       unsigned max_matching_distance,
-      gbfs::gbfs_data const*) const;
+      gbfs::gbfs_routing_data&) const;
 
   nigiri::hash_map<nigiri::location_idx_t,
                    std::vector<nigiri::routing::td_offset>>
@@ -35,10 +42,13 @@ struct routing {
 
   std::pair<std::vector<api::Itinerary>, nigiri::duration_t> route_direct(
       elevators const*,
-      gbfs::gbfs_data const*,
+      gbfs::gbfs_routing_data&,
       api::Place const& from,
       api::Place const& to,
       std::vector<api::ModeEnum> const&,
+      std::optional<std::vector<api::RentalFormFactorEnum>> const&,
+      std::optional<std::vector<api::RentalPropulsionTypeEnum>> const&,
+      std::optional<std::vector<std::string>> const& rental_providers,
       nigiri::unixtime_t start_time,
       bool wheelchair,
       std::chrono::seconds max) const;

include/motis/fwd.h

@@ -46,6 +46,10 @@ struct elevators;
 namespace gbfs {
 struct gbfs_data;
 struct gbfs_provider;
+struct gbfs_products_ref;
+struct gbfs_routing_data;
+struct provider_routing_data;
+struct products_routing_data;
 }  // namespace gbfs
 
 }  // namespace motis

include/motis/gbfs/compression.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+
+#include "cista/containers/bitvec.h"
+
+#include "utl/verify.h"
+
+#include "lz4.h"
+
+#include "motis/gbfs/data.h"
+
+namespace motis::gbfs {
+
+template <typename Vec, typename Key = typename Vec::size_type>
+inline compressed_bitvec compress_bitvec(
+    cista::basic_bitvec<Vec, Key> const& bv) {
+  auto const* original_data = reinterpret_cast<char const*>(bv.blocks_.data());
+  auto const original_bytes =
+      static_cast<int>(bv.blocks_.size() *
+                       sizeof(typename cista::basic_bitvec<Vec, Key>::block_t));
+  auto const max_compressed_size = LZ4_compressBound(original_bytes);
+
+  auto cbv = compressed_bitvec{
+      .data_ =
+          std::unique_ptr<char[], compressed_bitvec::free_deleter>{
+              static_cast<char*>(
+                  std::malloc(static_cast<std::size_t>(max_compressed_size)))},
+      .original_bytes_ = original_bytes,
+      .bitvec_size_ = bv.size_};
+  utl::verify(cbv.data_ != nullptr,
+              "could not allocate memory for compressed bitvec");
+
+  cbv.compressed_bytes_ = LZ4_compress_default(
+      original_data, cbv.data_.get(), original_bytes, max_compressed_size);
+  utl::verify(cbv.compressed_bytes_ > 0, "could not compress bitvec");
+
+  if (auto* compressed = std::realloc(
+          cbv.data_.get(), static_cast<std::size_t>(cbv.compressed_bytes_));
+      compressed != nullptr) {
+    cbv.data_.release();
+    cbv.data_.reset(static_cast<char*>(compressed));
+  }
+  return cbv;
+}
+
+template <typename Vec, typename Key = typename Vec::size_type>
+inline void decompress_bitvec(compressed_bitvec const& cbv,
+                              cista::basic_bitvec<Vec, Key>& bv) {
+  bv.resize(static_cast<typename cista::basic_bitvec<Vec, Key>::size_type>(
+      cbv.bitvec_size_));
+  auto const decompressed_bytes = LZ4_decompress_safe(
+      cbv.data_.get(), reinterpret_cast<char*>(bv.blocks_.data()),
+      cbv.compressed_bytes_,
+      static_cast<int>(
+          bv.blocks_.size() *
+          sizeof(typename cista::basic_bitvec<Vec, Key>::block_t)));
+  utl::verify(decompressed_bytes == cbv.original_bytes_,
+              "could not decompress bitvec");
+}
+
+}  // namespace motis::gbfs