From 9ae4b9f3b4dec11b1bc9f95161cdbcd394cd5269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20G=C3=BCndling?= Date: Mon, 4 Nov 2024 00:37:06 +0100 Subject: [PATCH 1/3] memory mapped reverse geocoder --- .pkg | 2 +- .pkg.lock | 8 ++++---- src/data.cc | 1 - src/import.cc | 17 +++++++++++++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.pkg b/.pkg index 89a5cce2a..e5d12329a 100644 --- a/.pkg +++ b/.pkg @@ -17,7 +17,7 @@ [adr] url=git@github.com:triptix-tech/adr.git branch=master - commit=b54517837bdf6d5ac4967858c2c4011f29e5789f + commit=d5ee67db28999b54e82a923fe32d7b461490f08f [googletest] url=git@github.com:motis-project/googletest.git branch=master diff --git a/.pkg.lock b/.pkg.lock index 8e9e05ced..bb2d54c39 100644 --- a/.pkg.lock +++ b/.pkg.lock @@ -1,5 +1,5 @@ -521520066768043934 -cista 5b6808fd7d7a7c1c7e4cb152b563dbf71e3efba4 +16948827567758484956 +cista 52577def055e4bdf90eaa461872fc1f7b5b1131d zlib-ng 68ab3e2d80253ec5dc3c83691d9ff70477b32cd3 boost 73549ebca677fe6214202a1ab580362b4f80e653 googletest 7b64fca6ea0833628d6f86255a81424365f7cc0c @@ -12,7 +12,7 @@ res b759b93316afeb529b6cb5b2548b24c41e382fb0 date ce88cc33b5551f66655614eeebb7c5b7189025fb oh d21c30f40e52a83d6dc09bcffd0067598b5ec069 doctest 70e8f76437b76dd5e9c0a2eb9b907df190ab71a0 -geo 5d99aeb10674a41a82d7c78f850abfd9605bf6e1 +geo de5a3586871b8b76a487d42fcc673c9487a1e233 miniz 1edbdece9d71dc65c6ff405572ee37cbdcef7af4 curl 39c8a51e8ee0ab7ea712886df79c068405a2e008 json 410c74782230daaa15054d6ee0975c0607091cb3 @@ -45,5 +45,5 @@ reflect-cpp c54fe66de4650b60c23aadd4a06d9db4ffeda22f FTXUI dd6a5d371fd7a3e2937bb579955003c54b727233 tg 20c0f298b8ce58de29a790290f44dca7c4ecc364 utf8proc 779b780da3b99d123133eb99707b65c7e4324cc8 -adr b54517837bdf6d5ac4967858c2c4011f29e5789f +adr d5ee67db28999b54e82a923fe32d7b461490f08f openapi-cpp 6b5fd40a2b552fc5656aeff56f203a34c1d9e9af diff --git a/src/data.cc b/src/data.cc index 38fd486d9..76ea7c495 100644 --- a/src/data.cc +++ b/src/data.cc @@ -209,7 +209,6 @@ void data::load_geocoder() { void data::load_reverse_geocoder() { r_ = std::make_unique(path_ / "adr", cista::mmap::protection::READ); - r_->build_rtree(*t_); } void data::load_matches() { diff --git a/src/import.cc b/src/import.cc index c39abd200..d06daa715 100644 --- a/src/import.cc +++ b/src/import.cc @@ -40,6 +40,7 @@ #include "osr/ways.h" #include "adr/adr.h" +#include "adr/reverse.h" #include "adr/typeahead.h" #include "motis/adr_extend_tt.h" @@ -249,7 +250,7 @@ data import(config const& c, fs::path const& data_path, bool const write) { utl::to_vec( t.datasets_, [&, src = n::source_idx_t{}](auto&& x) mutable - -> std::pair { + -> std::pair { auto const& [tag, dc] = x; d.tags_->add(src++, tag); return {dc.path_, @@ -305,8 +306,20 @@ data import(config const& c, fs::path const& data_path, bool const write) { if (write) { cista::write(data_path / "adr" / "t_ext.bin", *d.t_); } + d.r_.reset(); + { + auto r = adr::reverse{data_path / "adr", + cista::mmap::protection::WRITE}; + r.build_rtree(*d.t_); + r.write(); + } d.t_.get()->~typeahead(); - d.load_geocoder(); + if (c.geocoding_) { + d.load_geocoder(); + } + if (c.reverse_geocoding_) { + d.load_reverse_geocoder(); + } }, [&]() { d.load_geocoder(); }, {tt_hash, osm_hash, adr_version(), n_version()}}; From cc6d94e302a99b9ed8dba15caea9cf0123745d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20G=C3=BCndling?= Date: Mon, 4 Nov 2024 00:55:43 +0100 Subject: [PATCH 2/3] wip --- include/motis/config.h | 4 ---- include/motis/hashes.h | 2 +- src/config.cc | 3 ++- src/import.cc | 12 ++++++++++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/include/motis/config.h b/include/motis/config.h index c4fd82989..3cd70600d 100644 --- a/include/motis/config.h +++ b/include/motis/config.h @@ -125,7 +125,3 @@ struct config { }; } // namespace motis - -#include "cista/reflection/to_tuple.h" - -static_assert(std::is_aggregate_v); diff --git a/include/motis/hashes.h b/include/motis/hashes.h index 41c32b559..a4ac8a158 100644 --- a/include/motis/hashes.h +++ b/include/motis/hashes.h @@ -14,7 +14,7 @@ constexpr auto const osr_version = []() { return meta_entry_t{"osr_bin_ver", 6U}; }; constexpr auto const adr_version = []() { - return meta_entry_t{"adr_bin_ver", 1U}; + return meta_entry_t{"adr_bin_ver", 2U}; }; constexpr auto const n_version = []() { return meta_entry_t{"nigiri_bin_ver", 6U}; diff --git a/src/config.cc b/src/config.cc index 666a9672f..cfe6fc02a 100644 --- a/src/config.cc +++ b/src/config.cc @@ -65,6 +65,7 @@ config config::read_simple(std::vector const& args) { c.osm_ = p; c.street_routing_ = true; c.geocoding_ = true; + c.reverse_geocoding_ = true; c.tiles_ = {config::tiles{.profile_ = "tiles-profiles/full.lua"}}; } else { if (!c.timetable_.has_value()) { @@ -399,7 +400,7 @@ config config::read_legacy(fs::path const& p) { : std::nullopt; c.street_routing_ = is_module_active("osr") || is_module_active("osrm") || is_module_active("ppr"); - c.geocoding_ = is_module_active("adr"); + c.geocoding_ = c.reverse_geocoding_ = is_module_active("adr"); c.tiles_ = is_module_active("tiles") ? std::optional{tiles{.profile_ = cfg.profile_path_, diff --git a/src/import.cc b/src/import.cc index d06daa715..9109b0c2b 100644 --- a/src/import.cc +++ b/src/import.cc @@ -206,7 +206,10 @@ data import(config const& c, fs::path const& data_path, bool const write) { d.load_reverse_geocoder(); } }, - {osm_hash, adr_version()}}; + {osm_hash, + adr_version(), + {"geocoding", c.geocoding_}, + {"reverse_geocoding", c.reverse_geocoding_}}}; auto tt = task{ "tt", @@ -322,7 +325,12 @@ data import(config const& c, fs::path const& data_path, bool const write) { } }, [&]() { d.load_geocoder(); }, - {tt_hash, osm_hash, adr_version(), n_version()}}; + {tt_hash, + osm_hash, + adr_version(), + n_version(), + {"geocoding", c.geocoding_}, + {"reverse_geocoding", c.reverse_geocoding_}}}; auto osr_footpath = task{ "osr_footpath", From 34e2d98336ce8e55df74104d12d970e1541fd42e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20G=C3=BCndling?= Date: Mon, 4 Nov 2024 12:50:53 +0100 Subject: [PATCH 3/3] wip --- src/import.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/import.cc b/src/import.cc index 9109b0c2b..7cc662369 100644 --- a/src/import.cc +++ b/src/import.cc @@ -253,7 +253,7 @@ data import(config const& c, fs::path const& data_path, bool const write) { utl::to_vec( t.datasets_, [&, src = n::source_idx_t{}](auto&& x) mutable - -> std::pair { + -> std::pair { auto const& [tag, dc] = x; d.tags_->add(src++, tag); return {dc.path_,