From bea9623fc31d6fbecc32dce1a9d71f5949517a3c Mon Sep 17 00:00:00 2001 From: kadhikari Date: Wed, 6 Sep 2023 17:54:33 +0200 Subject: [PATCH 01/11] Init connector Forseti for bss stations --- source/jormungandr/jormungandr/instance.py | 19 +++ .../jormungandr/interfaces/v1/__init__.py | 4 + .../interfaces/v1/serializer/status.py | 7 + .../bss/bss_provider_manager.py | 5 + .../parking_space_availability/bss/forseti.py | 123 ++++++++++++++++++ .../bss/tests/forseti_test.py | 102 +++++++++++++++ .../parking_places_manager.py | 9 ++ .../navitiacommon/navitiacommon/constants.py | 2 +- .../navitiacommon/models/external_service.py | 6 + ...dd_bss_stations_in_navitia_service_type.py | 33 +++++ 10 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py create mode 100644 source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py create mode 100644 source/tyr/migrations/versions/cab8323f71bd_add_bss_stations_in_navitia_service_type.py diff --git a/source/jormungandr/jormungandr/instance.py b/source/jormungandr/jormungandr/instance.py index dd862e8b55..fa0d04e890 100644 --- a/source/jormungandr/jormungandr/instance.py +++ b/source/jormungandr/jormungandr/instance.py @@ -63,6 +63,7 @@ from navitiacommon import default_values from jormungandr.equipments import EquipmentProviderManager from jormungandr.external_services import ExternalServiceManager +from jormungandr.parking_space_availability.bss.bss_provider_manager import BssProviderManager from jormungandr.utils import ( can_connect_to_database, make_origin_destination_key, @@ -257,6 +258,13 @@ def __init__( self.external_service_provider_manager = ExternalServiceManager( self, external_service_provider_configurations, self.get_external_service_providers_from_db ) + + # Init BSS provider manager from config from external services in bdd + if disable_database: + self.bss_provider_manager = BssProviderManager(app.config['BSS_PROVIDER']) + else: + self.bss_provider_manager = BssProviderManager(app.config['BSS_PROVIDER'], self.get_bss_stations_from_db) + self.external_service_provider_manager.init_external_services() self.instance_db = instance_db self._ghost_words = ghost_words or [] @@ -333,6 +341,14 @@ def get_realtime_proxies_from_db(self): result = models.external_services if models else None return [res for res in result if res.navitia_service == 'realtime_proxies'] + def get_bss_stations_from_db(self): + """ + :return: a callable query of external services associated to the current instance in db + """ + models = self._get_models() + result = models.external_services if models else None + return [res for res in result if res.navitia_service == 'bss_stations'] + @property def autocomplete(self): if self._autocomplete_type: @@ -987,6 +1003,9 @@ def get_all_street_networks(self): def get_all_ridesharing_services(self): return self.ridesharing_services_manager.get_all_ridesharing_services() + def get_all_bss_providers(self): + return self.bss_provider_manager.get_providers() + def get_autocomplete(self, requested_autocomplete): if not requested_autocomplete: return self.autocomplete diff --git a/source/jormungandr/jormungandr/interfaces/v1/__init__.py b/source/jormungandr/jormungandr/interfaces/v1/__init__.py index 0265c65681..b6968c8ded 100644 --- a/source/jormungandr/jormungandr/interfaces/v1/__init__.py +++ b/source/jormungandr/jormungandr/interfaces/v1/__init__.py @@ -56,6 +56,10 @@ def add_common_status(response, instance): for rss in instance.get_all_ridesharing_services(): response['status']['ridesharing_services'].append(rss.status()) + response['status']['bss_providers'] = [] + for bp in instance.get_all_bss_providers(): + response['status']['bss_providers'].append(bp.status()) + response['status']['equipment_providers_services'] = {} response['status']['equipment_providers_services'][ 'equipment_providers_keys' diff --git a/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py b/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py index eabba34b25..b693982797 100644 --- a/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py +++ b/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py @@ -204,6 +204,12 @@ def get_arrival_radius(self, obj): return obj.get('arrival_radius') +class BSSStationsSerializer(OutsideServiceCommon): + id = Field(display_none=True) + url = Field(display_none=True) + class_ = Field(schema_type=str, label='class', attr='class') + + class EquipmentProvidersSerializer(NullableDictSerializer): key = Field(schema_type=str, display_none=False) codes_types = Field(schema_type=str, many=True, display_none=True) @@ -255,6 +261,7 @@ class CommonStatusSerializer(NullableDictSerializer): publication_date = Field(schema_type=str, display_none=False) street_networks = StreetNetworkSerializer(many=True, display_none=False) ridesharing_services = RidesharingServicesSerializer(many=True, display_none=False) + bss_providers = BSSStationsSerializer(many=True, display_none=False) equipment_providers_services = EquipmentProvidersServicesSerializer(display_none=False) external_providers_services = ExternalServiceProvidersServicesSerializer(display_none=False) start_production_date = Field(schema_type=str, display_none=False) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py b/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py index 021fc11992..d2b3ec58d2 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py @@ -67,6 +67,7 @@ def update_config(self): self._last_update = datetime.datetime.utcnow() try: + # BSS provider list form the database (table bss_provider) providers = self._providers_getter() except Exception as e: logger.exception('No access to table bss_provider (error: {})'.format(e)) @@ -119,3 +120,7 @@ def _get_providers(self): def get_providers(self): return self._get_providers() + + def exist_provider(self): + self.update_config() + return any((self._bss_providers.values(), self._bss_providers_legacy)) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py new file mode 100644 index 0000000000..490898b367 --- /dev/null +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -0,0 +1,123 @@ +# coding: utf-8 + +# Copyright (c) 2001-2022, Hove and/or its affiliates. All rights reserved. +# +# This file is part of Navitia, +# the software to build cool stuff with public transport. +# +# Hope you'll enjoy and contribute to this project, +# powered by Hove (www.hove.com). +# Help us simplify mobility and open public transport: +# a non ending quest to the responsive locomotion way of traveling! +# +# LICENCE: This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# Stay tuned using +# twitter @navitia +# channel `#navitia` on riot https://riot.im/app/#/room/#navitia:matrix.org +# https://groups.google.com/d/forum/navitia +# www.navitia.io +from __future__ import absolute_import, print_function, unicode_literals, division +from jormungandr import cache, app +import pybreaker +import logging +import requests as requests +from jormungandr.ptref import FeedPublisher +from jormungandr.parking_space_availability.bss.stands import Stands, StandsStatus +from jormungandr.parking_space_availability.bss.common_bss_provider import CommonBssProvider, BssProxyError +import six + +DEFAULT_FORSETI_FEED_PUBLISHER = {'id': 'forseti', 'name': 'forseti', 'license': 'Private', 'url': 'www.forseti.fr'} + + +class ForsetiProvider(CommonBssProvider): + """ + class managing calls to Forseti external service providing real-time BSS stands availability + + """ + + def __init__(self, service_url, feed_publisher=DEFAULT_FORSETI_FEED_PUBLISHER, timeout=2, **kwargs): + self.logger = logging.getLogger(__name__) + self.service_url = service_url + self.timeout = timeout + self.network = "Forseti" + self.breaker = pybreaker.CircuitBreaker( + fail_max=kwargs.get('circuit_breaker_max_fail', app.config['CIRCUIT_BREAKER_MAX_FORSETI_FAIL']), + reset_timeout=kwargs.get( + 'circuit_breaker_reset_timeout', app.config['CIRCUIT_BREAKER_FORSETI_TIMEOUT_S'] + ), + ) + + self._feed_publisher = FeedPublisher(**feed_publisher) if feed_publisher else None + + def service_caller(self, method, url): + try: + response = self.breaker.call(method, url, timeout=self.timeout, verify=False) + if not response or response.status_code != 200: + logging.getLogger(__name__).error( + 'Forseti, Invalid response, status_code: {}'.format(response.status_code) + ) + raise BssProxyError('non 200 response') + return response + except pybreaker.CircuitBreakerError as e: + logging.getLogger(__name__).error('forseti service dead (error: {})'.format(e)) + raise BssProxyError('circuit breaker open') + except requests.Timeout as t: + logging.getLogger(__name__).error('forseti service timeout (error: {})'.format(t)) + raise BssProxyError('timeout') + except Exception as e: + logging.getLogger(__name__).exception('forseti error : {}'.format(str(e))) + raise BssProxyError(str(e)) + + @cache.memoize(app.config.get(str('CACHE_CONFIGURATION'), {}).get(str('TIMEOUT_FORSETI'), 30)) + def _call_webservice(self, arguments): + url = "{}?{}".format(self.service_url, arguments) + data = self.service_caller(method=requests.get, url=url) + return data.json() + + def support_poi(self, poi): + return True + + def status(self): + # return {'network': self.network, 'operators': self.operators} + return { + 'id': six.text_type(self.network), + 'url': self.service_url, + 'class': self.__class__.__name__, + } + + def feed_publisher(self): + return self._feed_publisher + + def _get_informations(self, poi): + longitude = poi.get('coord', {}).get('lon', None) + latitude = poi.get('coord', {}).get('lat', None) + if latitude is None or latitude is None: + return Stands(0, 0, StandsStatus.unavailable) + + arguments = 'coord={}%3B{}&distance=50'.format(longitude, latitude) + data = self._call_webservice(arguments) + + if not data: + return Stands(0, 0, StandsStatus.unavailable) + obj_stations = data.get('stations', []) + + if not obj_stations: + return Stands(0, 0, StandsStatus.unavailable) + vehicle_count = 0 + for v in obj_stations[0].get('vehicles'): + vehicle_count = vehicle_count + v.get('count', 0) + + stand = Stands(obj_stations[0].get('docks', {}).get('available', 0), vehicle_count, StandsStatus.open) + return stand diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py new file mode 100644 index 0000000000..b1b90ba1b7 --- /dev/null +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py @@ -0,0 +1,102 @@ +# encoding: utf-8 +# Copyright (c) 2001-2022, Hove and/or its affiliates. All rights reserved. +# +# This file is part of Navitia, +# the software to build cool stuff with public transport. +# +# Hope you'll enjoy and contribute to this project, +# powered by Hove (www.hove.com). +# Help us simplify mobility and open public transport: +# a non ending quest to the responsive locomotion way of traveling! +# +# LICENCE: This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# Stay tuned using +# twitter @navitia +# channel `#navitia` on riot https://riot.im/app/#/room/#navitia:matrix.org +# https://groups.google.com/d/forum/navitia +# www.navitia.io + +from __future__ import absolute_import, print_function, unicode_literals, division +from copy import deepcopy +from jormungandr.parking_space_availability.bss.forseti import ForsetiProvider +from jormungandr.parking_space_availability.bss.stands import Stands, StandsStatus +from mock import MagicMock + +poi = { + 'poi_type': {'name': 'station vls', 'id': 'poi_type:amenity:bicycle_rental'}, + 'coord': {'lat': '48.0981147', 'lon': '-1.6552921'} +} + +BSS_PROVIDER = [ + { + "id": "forseti_stations", + "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", + "args": { + "service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", + "timeout": 20 + } + } + ] + + +def parking_space_availability_forseti_support_poi_test(): + """ + ForsetiProvider bss provider support + Since we search bss station in forseti with coordinate, it is always True + """ + provider = ForsetiProvider('http://forseti') + poi_copy = deepcopy(poi) + assert provider.support_poi(poi_copy) + + +def parking_space_availability_forseti_get_informations_test(): + webservice_response = { + "stations": [ + { + "id": "TAN:Station:18", + "name": "018-VIARME", + "coord": { + "lat": 48.0981147, + "lon": -1.6552921 + }, + "vehicles": [ + { + "type": "bicycle", + "count": 9 + } + ], + "docks": { + "available": 4, + "total": 13 + }, + "status": "OPEN" + } + ], + "pagination": { + "start_page": 0, + "items_on_page": 2, + "items_per_page": 25, + "total_result": 2 + } + } + + provider = ForsetiProvider('http://forseti') + provider._call_webservice = MagicMock(return_value=webservice_response) + assert provider.get_informations(poi) == Stands(4, 9, StandsStatus.open) + + provider._call_webservice = MagicMock(return_value=None) + assert provider.get_informations(poi) == Stands(0, 0, StandsStatus.unavailable) + invalid_poi = {} + assert provider.get_informations(invalid_poi) == Stands(0, 0, StandsStatus.unavailable) diff --git a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py index 31ef2858db..28386e77e4 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py +++ b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py @@ -83,6 +83,15 @@ def wrapper(*args, **kwargs): 'Error while handling BSS realtime availability', ) + if show_bss_stands and instance and instance.bss_provider_manager.exist_provider(): + _handle( + response, + instance.bss_provider_manager, + self.attribute, + self.logger, + 'Error while handling BSS realtime availability', + ) + if show_car_park and instance and instance.car_park_provider: _handle( response, diff --git a/source/navitiacommon/navitiacommon/constants.py b/source/navitiacommon/navitiacommon/constants.py index 0630c25ec2..1ffafd75e1 100644 --- a/source/navitiacommon/navitiacommon/constants.py +++ b/source/navitiacommon/navitiacommon/constants.py @@ -34,4 +34,4 @@ ENUM_SHAPE_SCOPE = ('admin', 'street', 'addr', 'poi', 'stop') DEFAULT_SHAPE_SCOPE = ('admin', 'street', 'addr', 'poi') -ENUM_EXTERNAL_SERVICE = ('free_floatings', 'vehicle_occupancies', 'realtime_proxies', 'vehicle_positions') +ENUM_EXTERNAL_SERVICE = ('free_floatings', 'vehicle_occupancies', 'realtime_proxies', 'vehicle_positions', 'bss_stations') diff --git a/source/navitiacommon/navitiacommon/models/external_service.py b/source/navitiacommon/navitiacommon/models/external_service.py index a2d6055658..5fb9c2eab4 100644 --- a/source/navitiacommon/navitiacommon/models/external_service.py +++ b/source/navitiacommon/navitiacommon/models/external_service.py @@ -77,3 +77,9 @@ def get_default(cls, navitia_service=None): def last_update(self): return self.updated_at if self.updated_at else self.created_at + + def full_args(self): + """ + generate args form jormungandr implementation of a bss providers from configuration in external service + """ + return self.args diff --git a/source/tyr/migrations/versions/cab8323f71bd_add_bss_stations_in_navitia_service_type.py b/source/tyr/migrations/versions/cab8323f71bd_add_bss_stations_in_navitia_service_type.py new file mode 100644 index 0000000000..85599b4175 --- /dev/null +++ b/source/tyr/migrations/versions/cab8323f71bd_add_bss_stations_in_navitia_service_type.py @@ -0,0 +1,33 @@ +"""Add bss_stations in navitia_service_type + +Revision ID: cab8323f71bd +Revises: 20d8caa23b26 +Create Date: 2023-09-06 14:19:03.579738 + +""" + +# revision identifiers, used by Alembic. +revision = 'cab8323f71bd' +down_revision = '20d8caa23b26' + +from alembic import op + + +def upgrade(): + op.execute("COMMIT") # See https://bitbucket.org/zzzeek/alembic/issue/123 + op.execute("ALTER TYPE navitia_service_type ADD VALUE 'bss_stations'") + + +def downgrade(): + op.execute( + "DELETE FROM associate_instance_external_service WHERE external_service_id in (SELECT ID FROM external_service WHERE navitia_service = 'bss_stations')" + ) + op.execute("DELETE FROM external_service WHERE navitia_service = 'bss_stations'") + op.execute("ALTER TABLE external_service ALTER COLUMN navitia_service TYPE text") + op.execute("DROP TYPE navitia_service_type CASCADE") + op.execute( + "CREATE TYPE navitia_service_type AS ENUM ('free_floatings', 'vehicle_occupancies', 'realtime_proxies', 'vehicle_positions')" + ) + op.execute( + "ALTER TABLE external_service ALTER COLUMN navitia_service TYPE navitia_service_type USING navitia_service::navitia_service_type" + ) From 5afe2cb81c783be239f9a40971b103abf678a124 Mon Sep 17 00:00:00 2001 From: kadhikari Date: Thu, 7 Sep 2023 09:44:36 +0200 Subject: [PATCH 02/11] Precommit error correction --- source/jormungandr/jormungandr/instance.py | 4 ++- .../parking_space_availability/bss/forseti.py | 7 ++++- .../bss/tests/forseti_test.py | 31 ++++--------------- .../navitiacommon/navitiacommon/constants.py | 8 ++++- 4 files changed, 22 insertions(+), 28 deletions(-) diff --git a/source/jormungandr/jormungandr/instance.py b/source/jormungandr/jormungandr/instance.py index fa0d04e890..6d34f39b70 100644 --- a/source/jormungandr/jormungandr/instance.py +++ b/source/jormungandr/jormungandr/instance.py @@ -263,7 +263,9 @@ def __init__( if disable_database: self.bss_provider_manager = BssProviderManager(app.config['BSS_PROVIDER']) else: - self.bss_provider_manager = BssProviderManager(app.config['BSS_PROVIDER'], self.get_bss_stations_from_db) + self.bss_provider_manager = BssProviderManager( + app.config['BSS_PROVIDER'], self.get_bss_stations_from_db + ) self.external_service_provider_manager.init_external_services() self.instance_db = instance_db diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py index 490898b367..dd806149bc 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -38,7 +38,12 @@ from jormungandr.parking_space_availability.bss.common_bss_provider import CommonBssProvider, BssProxyError import six -DEFAULT_FORSETI_FEED_PUBLISHER = {'id': 'forseti', 'name': 'forseti', 'license': 'Private', 'url': 'www.forseti.fr'} +DEFAULT_FORSETI_FEED_PUBLISHER = { + 'id': 'forseti', + 'name': 'forseti', + 'license': 'Private', + 'url': 'www.forseti.fr', +} class ForsetiProvider(CommonBssProvider): diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py index b1b90ba1b7..10db54940d 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py @@ -43,10 +43,7 @@ { "id": "forseti_stations", "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", - "args": { - "service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", - "timeout": 20 - } + "args": {"service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", "timeout": 20}, } ] @@ -67,29 +64,13 @@ def parking_space_availability_forseti_get_informations_test(): { "id": "TAN:Station:18", "name": "018-VIARME", - "coord": { - "lat": 48.0981147, - "lon": -1.6552921 - }, - "vehicles": [ - { - "type": "bicycle", - "count": 9 - } - ], - "docks": { - "available": 4, - "total": 13 - }, - "status": "OPEN" + "coord": {"lat": 48.0981147, "lon": -1.6552921}, + "vehicles": [{"type": "bicycle", "count": 9}], + "docks": {"available": 4, "total": 13}, + "status": "OPEN", } ], - "pagination": { - "start_page": 0, - "items_on_page": 2, - "items_per_page": 25, - "total_result": 2 - } + "pagination": {"start_page": 0, "items_on_page": 2, "items_per_page": 25, "total_result": 2}, } provider = ForsetiProvider('http://forseti') diff --git a/source/navitiacommon/navitiacommon/constants.py b/source/navitiacommon/navitiacommon/constants.py index 1ffafd75e1..b4cd6c20be 100644 --- a/source/navitiacommon/navitiacommon/constants.py +++ b/source/navitiacommon/navitiacommon/constants.py @@ -34,4 +34,10 @@ ENUM_SHAPE_SCOPE = ('admin', 'street', 'addr', 'poi', 'stop') DEFAULT_SHAPE_SCOPE = ('admin', 'street', 'addr', 'poi') -ENUM_EXTERNAL_SERVICE = ('free_floatings', 'vehicle_occupancies', 'realtime_proxies', 'vehicle_positions', 'bss_stations') +ENUM_EXTERNAL_SERVICE = ( + 'free_floatings', + 'vehicle_occupancies', + 'realtime_proxies', + 'vehicle_positions', + 'bss_stations', +) From 4e946d2c782134908c16ae35a7a1000be94557fb Mon Sep 17 00:00:00 2001 From: kadhikari Date: Thu, 7 Sep 2023 09:56:21 +0200 Subject: [PATCH 03/11] Another precommit --- .../parking_space_availability/bss/tests/forseti_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py index 10db54940d..6c6f4ed60e 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py @@ -36,14 +36,14 @@ poi = { 'poi_type': {'name': 'station vls', 'id': 'poi_type:amenity:bicycle_rental'}, - 'coord': {'lat': '48.0981147', 'lon': '-1.6552921'} + 'coord': {'lat': '48.0981147', 'lon': '-1.6552921'}, } BSS_PROVIDER = [ { - "id": "forseti_stations", - "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", - "args": {"service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", "timeout": 20}, + "id": "forseti_stations", + "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", + "args": {"service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", "timeout": 20}, } ] From 3d2bac2c3dbb241d11ecc28f5f0c99685158226c Mon Sep 17 00:00:00 2001 From: kadhikari Date: Thu, 7 Sep 2023 14:59:37 +0200 Subject: [PATCH 04/11] Another formatting --- .../parking_space_availability/bss/tests/forseti_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py index 6c6f4ed60e..fb54b7aaf3 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py @@ -45,7 +45,7 @@ "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", "args": {"service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", "timeout": 20}, } - ] +] def parking_space_availability_forseti_support_poi_test(): From 2d4425b44218f30c1fa5ad2db29c8388a5aebdf3 Mon Sep 17 00:00:00 2001 From: kadhikari Date: Tue, 12 Sep 2023 13:57:23 +0200 Subject: [PATCH 05/11] Adding organizations and distance in the config --- source/jormungandr/jormungandr/instance.py | 4 ++-- .../bss/bss_provider_manager.py | 2 -- .../parking_space_availability/bss/forseti.py | 22 +++++++++++++++++-- .../bss/tests/forseti_test.py | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/source/jormungandr/jormungandr/instance.py b/source/jormungandr/jormungandr/instance.py index 6d34f39b70..f3fc0b3d0e 100644 --- a/source/jormungandr/jormungandr/instance.py +++ b/source/jormungandr/jormungandr/instance.py @@ -261,10 +261,10 @@ def __init__( # Init BSS provider manager from config from external services in bdd if disable_database: - self.bss_provider_manager = BssProviderManager(app.config['BSS_PROVIDER']) + self.bss_provider_manager = BssProviderManager(app.config[str('BSS_PROVIDER')]) else: self.bss_provider_manager = BssProviderManager( - app.config['BSS_PROVIDER'], self.get_bss_stations_from_db + app.config[str('BSS_PROVIDER')], self.get_bss_stations_from_db ) self.external_service_provider_manager.init_external_services() diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py b/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py index d2b3ec58d2..d079d71bb8 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py @@ -112,10 +112,8 @@ def _handle_poi(self, item): return provider return None - # TODO use public version everywhere def _get_providers(self): self.update_config() - # providers from the database have priority on legacies providers return list(self._bss_providers.values()) + self._bss_providers_legacy def get_providers(self): diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py index dd806149bc..1301032251 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -52,9 +52,18 @@ class managing calls to Forseti external service providing real-time BSS stands """ - def __init__(self, service_url, feed_publisher=DEFAULT_FORSETI_FEED_PUBLISHER, timeout=2, **kwargs): + def __init__( + self, + service_url, + distance=50, + organizations=[], + feed_publisher=DEFAULT_FORSETI_FEED_PUBLISHER, + timeout=2, + **kwargs, + ): self.logger = logging.getLogger(__name__) self.service_url = service_url + self.distance = distance self.timeout = timeout self.network = "Forseti" self.breaker = pybreaker.CircuitBreaker( @@ -65,6 +74,10 @@ def __init__(self, service_url, feed_publisher=DEFAULT_FORSETI_FEED_PUBLISHER, t ) self._feed_publisher = FeedPublisher(**feed_publisher) if feed_publisher else None + if not isinstance(organizations, list): + self.organizations = organizations.strip('[').strip(']').split(',') + else: + self.organizations = organizations def service_caller(self, method, url): try: @@ -111,7 +124,12 @@ def _get_informations(self, poi): if latitude is None or latitude is None: return Stands(0, 0, StandsStatus.unavailable) - arguments = 'coord={}%3B{}&distance=50'.format(longitude, latitude) + params_organizations = '' + for param in self.organizations: + params_organizations += '&organization[]={}'.format(param) + + # /stations?coord=lon%3Blat&distance=self.distance&organization[]=org1&organization[]=org2 ... + arguments = 'coord={}%3B{}&distance={}{}'.format(longitude, latitude, self.distance, params_organizations) data = self._call_webservice(arguments) if not data: diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py index fb54b7aaf3..1416d0e464 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py @@ -43,7 +43,7 @@ { "id": "forseti_stations", "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", - "args": {"service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", "timeout": 20}, + "args": {"service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", "timeout": 20, "distance": 20}, } ] From f5d0a5f117bb2bd7621e266876b010bc6096de78 Mon Sep 17 00:00:00 2001 From: kadhikari Date: Tue, 12 Sep 2023 17:25:51 +0200 Subject: [PATCH 06/11] Formatting --- .../jormungandr/parking_space_availability/bss/forseti.py | 4 +++- .../parking_space_availability/bss/tests/forseti_test.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py index 1301032251..b48f58a3d5 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -129,7 +129,9 @@ def _get_informations(self, poi): params_organizations += '&organization[]={}'.format(param) # /stations?coord=lon%3Blat&distance=self.distance&organization[]=org1&organization[]=org2 ... - arguments = 'coord={}%3B{}&distance={}{}'.format(longitude, latitude, self.distance, params_organizations) + arguments = 'coord={}%3B{}&distance={}{}'.format( + longitude, latitude, self.distance, params_organizations + ) data = self._call_webservice(arguments) if not data: diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py index 1416d0e464..4472e870c4 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py @@ -43,7 +43,11 @@ { "id": "forseti_stations", "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", - "args": {"service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", "timeout": 20, "distance": 20}, + "args": { + "service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", + "timeout": 20, + "distance": 20, + }, } ] From d5cdfc8d5d15432bd439fc8fbb983ac27ddc2601 Mon Sep 17 00:00:00 2001 From: kadhikari Date: Wed, 13 Sep 2023 09:11:23 +0200 Subject: [PATCH 07/11] Minor --- .../jormungandr/parking_space_availability/bss/forseti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py index b48f58a3d5..386e524f6f 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -59,7 +59,7 @@ def __init__( organizations=[], feed_publisher=DEFAULT_FORSETI_FEED_PUBLISHER, timeout=2, - **kwargs, + **kwargs ): self.logger = logging.getLogger(__name__) self.service_url = service_url From 8dedd432c3dfbf8264df2f2e11e4a5ffb481f751 Mon Sep 17 00:00:00 2001 From: kadhikari Date: Fri, 15 Sep 2023 17:05:17 +0200 Subject: [PATCH 08/11] Some modifications on remarks --- source/jormungandr/jormungandr/instance.py | 11 +++++------ source/jormungandr/jormungandr/instance_manager.py | 1 + .../jormungandr/interfaces/v1/serializer/status.py | 4 ++-- .../bss/bss_provider_manager.py | 4 ++-- .../parking_space_availability/bss/forseti.py | 2 +- .../bss/tests/forseti_test.py | 12 ------------ .../parking_places_manager.py | 8 ++++---- 7 files changed, 15 insertions(+), 27 deletions(-) diff --git a/source/jormungandr/jormungandr/instance.py b/source/jormungandr/jormungandr/instance.py index f3fc0b3d0e..70f1644887 100644 --- a/source/jormungandr/jormungandr/instance.py +++ b/source/jormungandr/jormungandr/instance.py @@ -170,6 +170,7 @@ def __init__( use_multi_reverse=False, resp_content_limit_bytes=None, resp_content_limit_endpoints_whitelist=None, + individual_bss_provider=[], ): super(Instance, self).__init__( name=name, @@ -261,11 +262,9 @@ def __init__( # Init BSS provider manager from config from external services in bdd if disable_database: - self.bss_provider_manager = BssProviderManager(app.config[str('BSS_PROVIDER')]) + self.bss_provider_manager = BssProviderManager(individual_bss_provider) else: - self.bss_provider_manager = BssProviderManager( - app.config[str('BSS_PROVIDER')], self.get_bss_stations_from_db - ) + self.bss_provider_manager = BssProviderManager(individual_bss_provider, self.get_bss_stations_services_from_db) self.external_service_provider_manager.init_external_services() self.instance_db = instance_db @@ -343,12 +342,12 @@ def get_realtime_proxies_from_db(self): result = models.external_services if models else None return [res for res in result if res.navitia_service == 'realtime_proxies'] - def get_bss_stations_from_db(self): + def get_bss_stations_services_from_db(self): """ :return: a callable query of external services associated to the current instance in db """ models = self._get_models() - result = models.external_services if models else None + result = models.external_services if models else [] return [res for res in result if res.navitia_service == 'bss_stations'] @property diff --git a/source/jormungandr/jormungandr/instance_manager.py b/source/jormungandr/jormungandr/instance_manager.py index 80ac5fe246..1e94e2e079 100644 --- a/source/jormungandr/jormungandr/instance_manager.py +++ b/source/jormungandr/jormungandr/instance_manager.py @@ -128,6 +128,7 @@ def register_instance(self, config): use_multi_reverse=config.get('use_multi_reverse', False), resp_content_limit_bytes=config.get('resp_content_limit_bytes', None), resp_content_limit_endpoints_whitelist=config.get('resp_content_limit_endpoints_whitelist', None), + individual_bss_provider=config.get('individual_bss_provider', []), ) self.instances[instance.name] = instance diff --git a/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py b/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py index b693982797..ca9745e0b3 100644 --- a/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py +++ b/source/jormungandr/jormungandr/interfaces/v1/serializer/status.py @@ -204,7 +204,7 @@ def get_arrival_radius(self, obj): return obj.get('arrival_radius') -class BSSStationsSerializer(OutsideServiceCommon): +class BSSStationsServiceSerializer(OutsideServiceCommon): id = Field(display_none=True) url = Field(display_none=True) class_ = Field(schema_type=str, label='class', attr='class') @@ -261,7 +261,7 @@ class CommonStatusSerializer(NullableDictSerializer): publication_date = Field(schema_type=str, display_none=False) street_networks = StreetNetworkSerializer(many=True, display_none=False) ridesharing_services = RidesharingServicesSerializer(many=True, display_none=False) - bss_providers = BSSStationsSerializer(many=True, display_none=False) + bss_providers = BSSStationsServiceSerializer(many=True, display_none=False) equipment_providers_services = EquipmentProvidersServicesSerializer(display_none=False) external_providers_services = ExternalServiceProvidersServicesSerializer(display_none=False) start_production_date = Field(schema_type=str, display_none=False) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py b/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py index d079d71bb8..014292ba90 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/bss_provider_manager.py @@ -67,7 +67,7 @@ def update_config(self): self._last_update = datetime.datetime.utcnow() try: - # BSS provider list form the database (table bss_provider) + # BSS provider list from the database (table bss_provider) providers = self._providers_getter() except Exception as e: logger.exception('No access to table bss_provider (error: {})'.format(e)) @@ -121,4 +121,4 @@ def get_providers(self): def exist_provider(self): self.update_config() - return any((self._bss_providers.values(), self._bss_providers_legacy)) + return any(self.get_providers()) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py index 386e524f6f..3f8a7fedfe 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -42,7 +42,7 @@ 'id': 'forseti', 'name': 'forseti', 'license': 'Private', - 'url': 'www.forseti.fr', + 'url': 'www.navitia.io', } diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py index 4472e870c4..21b65efab9 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/tests/forseti_test.py @@ -39,18 +39,6 @@ 'coord': {'lat': '48.0981147', 'lon': '-1.6552921'}, } -BSS_PROVIDER = [ - { - "id": "forseti_stations", - "class": "jormungandr.parking_space_availability.bss.forseti.ForsetiProvider", - "args": { - "service_url": "https://gbfs-station.forseti.sbx.aws.private/stations", - "timeout": 20, - "distance": 20, - }, - } -] - def parking_space_availability_forseti_support_poi_test(): """ diff --git a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py index 28386e77e4..8cd5661824 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py +++ b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py @@ -80,16 +80,16 @@ def wrapper(*args, **kwargs): bss_provider_manager, self.attribute, self.logger, - 'Error while handling BSS realtime availability', + 'Error while handling global BSS realtime availability', ) - if show_bss_stands and instance and instance.bss_provider_manager.exist_provider(): + if show_bss_stands and instance and instance.bss_provider_manager and instance.bss_provider_manager.exist_provider(): _handle( response, instance.bss_provider_manager, self.attribute, self.logger, - 'Error while handling BSS realtime availability', + 'Error while handling individual BSS realtime availability', ) if show_car_park and instance and instance.car_park_provider: @@ -98,7 +98,7 @@ def wrapper(*args, **kwargs): car_park_provider_manager, self.attribute, self.logger, - 'Error while handling car park realtime availability', + 'Error while handling global car park realtime availability', ) return response, status, h From 523366d42903c09353438ede856820f0f7ff4c17 Mon Sep 17 00:00:00 2001 From: kadhikari Date: Fri, 15 Sep 2023 17:17:41 +0200 Subject: [PATCH 09/11] Formatting --- source/jormungandr/jormungandr/instance.py | 4 +++- .../parking_space_availability/parking_places_manager.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/source/jormungandr/jormungandr/instance.py b/source/jormungandr/jormungandr/instance.py index 70f1644887..b19724b1f5 100644 --- a/source/jormungandr/jormungandr/instance.py +++ b/source/jormungandr/jormungandr/instance.py @@ -264,7 +264,9 @@ def __init__( if disable_database: self.bss_provider_manager = BssProviderManager(individual_bss_provider) else: - self.bss_provider_manager = BssProviderManager(individual_bss_provider, self.get_bss_stations_services_from_db) + self.bss_provider_manager = BssProviderManager( + individual_bss_provider, self.get_bss_stations_services_from_db + ) self.external_service_provider_manager.init_external_services() self.instance_db = instance_db diff --git a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py index 8cd5661824..06b04ecb4a 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py +++ b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py @@ -83,7 +83,12 @@ def wrapper(*args, **kwargs): 'Error while handling global BSS realtime availability', ) - if show_bss_stands and instance and instance.bss_provider_manager and instance.bss_provider_manager.exist_provider(): + if ( + show_bss_stands + and instance + and instance.bss_provider_manager + and instance.bss_provider_manager.exist_provider() + ): _handle( response, instance.bss_provider_manager, From 415c7ee0a8c51f69eee82079abcaa8f97b00c93a Mon Sep 17 00:00:00 2001 From: kadhikari Date: Mon, 18 Sep 2023 10:16:04 +0200 Subject: [PATCH 10/11] What a mistake, thanks sonarcloud --- .../jormungandr/parking_space_availability/bss/forseti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py index 3f8a7fedfe..b7d5c2783d 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -121,7 +121,7 @@ def feed_publisher(self): def _get_informations(self, poi): longitude = poi.get('coord', {}).get('lon', None) latitude = poi.get('coord', {}).get('lat', None) - if latitude is None or latitude is None: + if latitude is None or longitude is None: return Stands(0, 0, StandsStatus.unavailable) params_organizations = '' From c15d3e9a06e96991a261eda01711464c22c98169 Mon Sep 17 00:00:00 2001 From: krishna Date: Mon, 18 Sep 2023 17:05:48 +0200 Subject: [PATCH 11/11] Some modifications on remarks --- .../parking_space_availability/bss/forseti.py | 10 ++++------ .../parking_places_manager.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py index b7d5c2783d..52dff87235 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py +++ b/source/jormungandr/jormungandr/parking_space_availability/bss/forseti.py @@ -75,7 +75,8 @@ def __init__( self._feed_publisher = FeedPublisher(**feed_publisher) if feed_publisher else None if not isinstance(organizations, list): - self.organizations = organizations.strip('[').strip(']').split(',') + import json + self.organizations = json.loads(str(organizations)) else: self.organizations = organizations @@ -140,9 +141,6 @@ def _get_informations(self, poi): if not obj_stations: return Stands(0, 0, StandsStatus.unavailable) - vehicle_count = 0 - for v in obj_stations[0].get('vehicles'): - vehicle_count = vehicle_count + v.get('count', 0) - stand = Stands(obj_stations[0].get('docks', {}).get('available', 0), vehicle_count, StandsStatus.open) - return stand + vehicle_count = sum((v.get('count', 0) for v in obj_stations[0].get('vehicles', {}))) + return Stands(obj_stations[0].get('docks', {}).get('available', 0), vehicle_count, StandsStatus.open) diff --git a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py index 06b04ecb4a..578ac6fe13 100644 --- a/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py +++ b/source/jormungandr/jormungandr/parking_space_availability/parking_places_manager.py @@ -103,7 +103,7 @@ def wrapper(*args, **kwargs): car_park_provider_manager, self.attribute, self.logger, - 'Error while handling global car park realtime availability', + f'Error while handling global car park realtime availability with configuration for instance: {instance}', ) return response, status, h