fix nigiri to_extern_trip (#370)

.pkg

@@ -97,7 +97,7 @@
 [nigiri]
  [email protected]:motis-project/nigiri.git
  branch=master
- commit=89f9409ffc1dcf85986cab8e6bf06679e50a41fd
+ commit=7dc77ea6630f70900619dd71e1abc943e7bf5c9a
 [osmium]
  [email protected]:motis-project/libosmium.git
  branch=master

.pkg.lock

@@ -1,4 +1,4 @@
-867181114681065305
+10807041569219877111
 cista 04f84b0490e0e91ead20ae6eabed5add8f243ba0
 zlib fe8e13ffca867612951bc6baf114e5ac8b00f305
 boost be5235eb2258d2ec19e32546ab767a62311d9b46
@@ -28,7 +28,7 @@ net 44674d2f3917e20b7019a0f7254d332522c36fb7
 protobuf e4d1f574ea9793308da9a942b4d5c1d4628aaed6
 unordered_dense 4f380fb1d64f1843ca2c993ed39a8342a8747e5d
 wyhash 1e012b57fc2227a9e583a57e2eacb3da99816d99
-nigiri 89f9409ffc1dcf85986cab8e6bf06679e50a41fd
+nigiri 7dc77ea6630f70900619dd71e1abc943e7bf5c9a
 lua 7bb93325b26f84c7e8b51fcbd857361ce7605a1d
 luabind 9223568bbcf818ecfb1001d49f567627ee10852a
 tbb 2067af88257710d07253761655a802732a32496e

base/loader/test/gtfs/blockid_itest.cc

@@ -29,7 +29,8 @@ struct loader_graph_builder_gtfs_block_id : public motis_instance_test {
  {"--tripbased.use_data_file=false",
  fmt::format(
  "--nigiri.first_day={}-{}-{}", schedule_begin.substr(0, 4),
- schedule_begin.substr(4, 2), schedule_begin.substr(6, 2))}) {}
+ schedule_begin.substr(4, 2), schedule_begin.substr(6, 2)),
+ "--nigiri.num_days", "10"}) {}
 
  msg_ptr routing_query(std::string_view from, std::string_view to,
  std::string_view start_time, std::string_view target) {
@@ -153,14 +154,14 @@ TEST_F(loader_graph_builder_gtfs_block_id_once, search_s2_s1) {
  ASSERT_EQ(0, conns->size());
 }
 
-TEST_F(loader_graph_builder_gtfs_block_id_saturday, search_s2_s3) {
+TEST_F(loader_graph_builder_gtfs_block_id_friday, search_s2_s3) {
  auto res = routing_query("S2", "S3", "2006-07-08 00:00 Europe/Berlin");
  auto conns = motis_content(RoutingResponse, res)->connections();
  ASSERT_EQ(1, conns->size());
  expect_no_transfers(conns->Get(0));
 }
 
-TEST_F(loader_graph_builder_gtfs_block_id_saturday, search_s2_s7) {
+TEST_F(loader_graph_builder_gtfs_block_id_friday, search_s2_s7) {
  auto res = routing_query("S2", "S7", "2006-07-08 00:00 Europe/Berlin");
  auto conns = motis_content(RoutingResponse, res)->connections();
  ASSERT_EQ(1, conns->size());

modules/intermodal/src/eval/generator/intermodal_generator.cc

@@ -72,6 +72,10 @@ struct generator_settings : public conf::configuration {
  param(dest_type_, "dest_type", "destination type: coordinate|station");
  param(routers_, "routers", "routing targets");
  param(search_dir_, "search_dir", "search direction forward/backward");
+ param(extend_earlier_, "extend_earlier", "extend search interval earlier");
+ param(extend_later_, "extend_later", "extend search interval later");
+ param(min_connection_count_, "min_connection_count",
+ "min. number of connections (otherwise interval will be extended)");
  }
 
  MsgContent get_message_type() const {
@@ -127,6 +131,9 @@ struct generator_settings : public conf::configuration {
  bool large_stations_{false};
  std::vector<std::string> routers_{"/routing"};
  std::string search_dir_{"forward"};
+ bool extend_earlier_{false};
+ bool extend_later_{false};
+ unsigned min_connection_count_{0U};
 };
 
 std::string replace_target_escape(std::string const& str,
@@ -386,6 +393,8 @@ void write_query(schedule const& sched, point_generator& point_gen, int id,
  double const dest_radius, MsgContent const message_type,
  IntermodalStart const start_type,
  IntermodalDestination const destination_type, SearchDir dir,
+ bool const extend_earlier, bool const extend_later,
+ unsigned const min_connection_count,
  std::vector<std::string> const& routers,
  std::vector<std::ofstream>& out_files) {
  auto fbbs = utl::to_vec(
@@ -414,8 +423,9 @@ void write_query(schedule const& sched, point_generator& point_gen, int id,
  MsgContent_IntermodalRoutingRequest,
  CreateIntermodalRoutingRequest(
  fbb, start_type,
- CreateIntermodalPretripStart(fbb, &start_pt, &interval, 0,
- false, false)
+ CreateIntermodalPretripStart(fbb, &start_pt, &interval,
+ min_connection_count,
+ extend_earlier, extend_later)
  .Union(),
  fbb.CreateVector(create_modes(fbb, start_modes)),
  destination_type, get_destination(fbb),
@@ -535,7 +545,8 @@ void write_query(schedule const& sched, point_generator& point_gen, int id,
  fbb,
  CreateInputStation(fbb, fbb.CreateString(from->eva_nr_),
  fbb.CreateString("")),
- &interval)
+ &interval, min_connection_count, extend_earlier,
+ extend_later)
  .Union(),
  fbb.CreateVector(create_modes(fbb, start_modes)),
  destination_type, get_destination(fbb),
@@ -637,7 +648,8 @@ void write_query(schedule const& sched, point_generator& point_gen, int id,
  fbb,
  CreateInputStation(fbb, fbb.CreateString(from->eva_nr_),
  fbb.CreateString("")),
- &interval)
+ &interval, min_connection_count, extend_earlier,
+ extend_later)
  .Union(),
  CreateInputStation(fbb, fbb.CreateString(to->eva_nr_),
  fbb.CreateString("")),
@@ -849,7 +861,9 @@ int generate(int argc, char const** argv) {
  write_query(sched, point_gen, i, interval, from, to, start_modes,
  dest_modes, start_radius, dest_radius, message_type, start_type,
  dest_type, generator_opt.get_search_dir(),
- generator_opt.routers_, of_streams);
+ generator_opt.extend_earlier_, generator_opt.extend_later_,
+ generator_opt.min_connection_count_, generator_opt.routers_,
+ of_streams);
  }
 
  return 0;

modules/nigiri/include/motis/nigiri/extern_trip.h

@@ -15,6 +15,6 @@ struct tag_lookup;
 extern_trip nigiri_trip_to_extern_trip(tag_lookup const&,
  ::nigiri::timetable const&,
  ::nigiri::trip_idx_t,
- ::nigiri::day_idx_t);
+ ::nigiri::transport);
 
 } // namespace motis::nigiri

modules/nigiri/include/motis/nigiri/nigiri.h

@@ -40,6 +40,7 @@ struct nigiri : public motis::module::module {
  bool routing_{false};
  unsigned link_stop_distance_{100U};
  std::vector<std::string> gtfsrt_urls_;
+ std::vector<std::string> gtfsrt_paths_;
  unsigned gtfsrt_update_interval_{60U};
 };
 

modules/nigiri/src/extern_trip.cc

@@ -12,7 +12,7 @@ namespace motis::nigiri {
 extern_trip nigiri_trip_to_extern_trip(tag_lookup const& tags,
  n::timetable const& tt,
  n::trip_idx_t const trip_idx,
- n::day_idx_t const day) {
+ n::transport const t) {
  auto const resolve_id = [&](n::location_idx_t const x) {
  return get_station_id(
  tags, tt,
@@ -45,11 +45,11 @@ extern_trip nigiri_trip_to_extern_trip(tag_lookup const& tags,
  tt.trip_id_strings_[x].view()),
  .station_id_ = first_location,
  .train_nr_ = tt.trip_train_nr_.at(tt.trip_ids_.at(trip_idx).back()),
- .time_ = to_motis_unixtime(tt.event_time(
- {transport, day}, stop_range.from_, n::event_type::kDep)),
+ .time_ = to_motis_unixtime(
+ tt.event_time(t, stop_range.from_, n::event_type::kDep)),
  .target_station_id_ = last_location,
- .target_time_ = to_motis_unixtime(tt.event_time(
- {transport, day}, stop_range.to_ - 1, n::event_type::kArr)),
+ .target_time_ = to_motis_unixtime(
+ tt.event_time(t, stop_range.to_ - 1, n::event_type::kArr)),
  .line_id_ = std::string{line}};
 }
 

modules/nigiri/src/get_station.cc

@@ -153,6 +153,9 @@ struct rt_ev_iterator : public ev_iterator {
  n::event_type const ev_type, n::direction const dir)
  : rtt_{rtt}, stop_idx_{stop_idx}, rt_t_{rt_t}, ev_type_{ev_type} {
  finished_ = dir == n::direction::kForward ? time() < start : time() > start;
+ assert(ev_type == n::event_type::kDep &&
+ stop_idx_ < rtt.rt_transport_location_seq_[rt_t].size() - 1U ||
+ ev_type == n::event_type::kArr && stop_idx_ > 0U);
  }
 
  ~rt_ev_iterator() override = default;
@@ -282,7 +285,7 @@ mm::msg_ptr get_station(tag_lookup const& tags, n::timetable const& tt,
  fbb.CreateVector(std::vector{CreateTripInfo(
  fbb,
  to_fbs(fbb, nigiri_trip_to_extern_trip(
- tags, tt, fr[0].get_trip_idx(ev_type), fr.t_.day_)),
+ tags, tt, fr[0].get_trip_idx(ev_type), fr.t_)),
  CreateTransport(
  fbb, &range, static_cast<std::uint32_t>(fr[0].get_clasz()),
  fbb.CreateString(fr[0].line(ev_type)),

modules/nigiri/src/nigiri.cc

@@ -31,6 +31,7 @@
 #include "motis/nigiri/station_lookup.h"
 #include "motis/nigiri/trip_to_connection.h"
 #include "motis/nigiri/unixtime_conv.h"
+#include "utl/parser/split.h"
 
 namespace fs = std::filesystem;
 namespace mm = motis::module;
@@ -102,12 +103,56 @@ nigiri::nigiri() : module("Next Generation Routing", "nigiri") {
  "GTFS only: radius to connect stations, 0=skip");
  param(default_timezone_, "default_timezone",
  "tz for agencies w/o tz or routes w/o agency");
- param(gtfsrt_urls_, "gtfsrt", "list of GTFS-RT URL endpoints");
+ param(gtfsrt_urls_, "gtfsrt",
+ "list of GTFS-RT endpoints, format: tag|url|authorization");
+ param(gtfsrt_paths_, "gtfsrt_paths",
+ "list of GTFS-RT, format: tag|/path/to/file.pb");
 }
 
 nigiri::~nigiri() = default;
 
 void nigiri::init(motis::module::registry& reg) {
+ if (!gtfsrt_paths_.empty()) {
+ auto const rtt_copy = std::make_shared<n::rt_timetable>(*impl_->get_rtt());
+ auto statistics = std::vector<n::rt::statistics>{};
+ for (auto const& p : gtfsrt_paths_) {
+ auto const [tag, path] = utl::split<'|', utl::cstr, utl::cstr>(p);
+ if (path.empty()) {
+ throw utl::fail("bad GTFS-RT path: {} (required: tag|path/to/file)", p);
+ }
+ auto const src = impl_->tags_.get_src(tag.to_str() + '_');
+ if (src == n::source_idx_t::invalid()) {
+ throw utl::fail("bad GTFS-RT path: tag {} not found", tag.view());
+ }
+ auto const file =
+ cista::mmap{path.c_str(), cista::mmap::protection::READ};
+ auto stats = n::rt::statistics{};
+ try {
+ stats = n::rt::gtfsrt_update_buf(**impl_->tt_, *rtt_copy, src,
+ tag.view(), file.view());
+ } catch (std::exception const& e) {
+ stats.parser_error_ = true;
+ LOG(logging::error)
+ << "GTFS-RT update error (tag=" << tag.view() << ") " << e.what();
+ } catch (...) {
+ stats.parser_error_ = true;
+ LOG(logging::error)
+ << "Unknown GTFS-RT update error (tag= " << tag.view() << ")";
+ }
+ statistics.emplace_back(stats);
+ }
+ impl_->update_rtt(rtt_copy);
+ impl_->railviz_->update(rtt_copy);
+ for (auto const [path, stats] : utl::zip(gtfsrt_paths_, statistics)) {
+ LOG(logging::info) << "init " << path << ": "
+ << stats.total_entities_success_ << "/"
+ << stats.total_entities_ << " ("
+ << static_cast<double>(stats.total_entities_success_) /
+ stats.total_entities_ * 100
+ << "%)";
+ }
+ }
+
  reg.register_op("/nigiri",
  [&](mm::msg_ptr const& msg) {
  return route(impl_->tags_, **impl_->tt_,
@@ -361,7 +406,7 @@ void nigiri::import(motis::module::import_dispatcher& reg) {
  cista::file{dump_file_path.string().c_str(), "r"}
  .content()}));
  (**impl_->tt_).locations_.resolve_timezones();
- if (!gtfsrt_urls_.empty()) {
+ if (!gtfsrt_urls_.empty() || !gtfsrt_paths_.empty()) {
  impl_->update_rtt(std::make_shared<n::rt_timetable>(
  n::rt::create_rt_timetable(**impl_->tt_, today)));
  }

modules/nigiri/src/nigiri_to_motis_journey.cc

@@ -191,7 +191,7 @@ motis::journey nigiri_to_motis_journey(n::timetable const& tt,
  .at(tt.trip_debug_.at(trip).front().source_file_idx_)
  .view();
  extern_trips.add_entry(
- {nigiri_trip_to_extern_trip(tags, tt, trip, t.day_),
+ {nigiri_trip_to_extern_trip(tags, tt, trip, t),
  fmt::format("{}:{}:{}", src_file,
  tt.trip_debug_.at(trip).at(0).line_number_from_,
  tt.trip_debug_.at(trip).at(0).line_number_to_)},

modules/nigiri/src/railviz.cc

@@ -258,7 +258,7 @@ struct railviz::impl {
  fr.is_rt() ? TimestampReason_FORECAST : TimestampReason_SCHEDULE,
  mc.CreateVector(std::vector{
  to_fbs(mc, nigiri_trip_to_extern_trip(
- tags_, tt_, fr[0].get_trip_idx(), fr.t_.day_))}),
+ tags_, tt_, fr[0].get_trip_idx(), fr.t_))}),
  mc.CreateVector(polyline_indices));
  });
 

modules/nigiri/src/resolve_run.cc

@@ -42,19 +42,12 @@ n::rt::run resolve_run(tag_lookup const& tags, n::timetable const& tt,
  tt.event_mam(t_idx, stop_range.from_, n::event_type::kDep).days();
  auto const t = n::transport{t_idx, day_idx - day_offset};
  if (dep_time != tt.event_time(t, stop_range.from_, n::event_type::kDep)) {
- std::cout << "dep time no match: " << dep_time << ", "
- << tt.event_time(t, stop_range.from_, n::event_type::kDep)
- << "\n";
  continue;
  }
 
  auto const& traffic_days =
  tt.bitfields_[tt.transport_traffic_days_[t_idx]];
  if (!traffic_days.test(to_idx(day_idx - day_offset))) {
- std::cout << "traffic day not active: "
- << tt.internal_interval_days().from_ +
- to_idx(day_idx) * date::days{1}
- << "\n";
  continue;
  }
 

modules/nigiri/src/routing.cc

@@ -15,6 +15,7 @@
 #include "nigiri/special_stations.h"
 
 #include "motis/core/common/timing.h"
+#include "motis/core/access/error.h"
 #include "motis/core/journey/journeys_to_message.h"
 #include "motis/nigiri/location.h"
 #include "motis/nigiri/nigiri_to_motis_journey.h"
@@ -83,7 +84,7 @@ mm::msg_ptr to_routing_response(
  fbb, fbb.CreateVectorOfSortedTables(&statistics),
  fbb.CreateVector(connections),
  to_motis_unixtime(search_interval.from_),
- to_motis_unixtime(search_interval.to_ - std::chrono::minutes{1}),
+ to_motis_unixtime(search_interval.to_),
  fbb.CreateVector(std::vector<fbs::Offset<DirectConnection>>{}))
  .Union());
  return make_msg(fbb);
@@ -176,6 +177,23 @@ motis::module::msg_ptr route(tag_lookup const& tags, n::timetable const& tt,
  auto const destination_station =
  get_location_idx(tags, tt, req->destination()->id()->str());
 
+ std::visit(
+ utl::overloaded{
+ [&](n::unixtime_t const t) {
+ if (!tt.external_interval().contains(t)) {
+ throw std::system_error(access::error::timestamp_not_in_schedule);
+ }
+ },
+ [&](n::interval<n::unixtime_t>& interval) {
+ auto const tt_interval = tt.external_interval();
+ if (!interval.overlaps(tt_interval)) {
+ throw std::system_error(access::error::timestamp_not_in_schedule);
+ }
+ interval.from_ = tt_interval.clamp(interval.from_);
+ interval.to_ = tt_interval.clamp(interval.to_);
+ }},
+ start_time);
+
  auto const is_intermodal_start =
  start_station == n::get_special_station(n::special_station::kStart);
  auto const is_intermodal_dest =