diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cffcf34b..71e4a257d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,25 +9,41 @@ jobs: runs-on: ubuntu-latest + services: + redis: + image: redis:alpine + ports: + - "6379:6379" + + mongo: + image: mongo:4.4 + ports: + - "27017:27017" + + elastic: + image: elasticsearch:7.17.23 + ports: + - "9200:9200" + env: + discovery.type: single-node + defaults: run: working-directory: server steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: '3.8' - run: sudo apt-get update - run: sudo apt-get -y install libxmlsec1-dev - - run: docker compose -f ../.github/docker-compose.yml up -d - - run: | python -m pip install --upgrade pip wheel setuptools - pip install -r dev-requirements.txt + python -m pip install -r dev-requirements.txt - run: flake8 @@ -49,13 +65,15 @@ jobs: working-directory: client steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v4 with: - node-version: 14.x + node-version: 14 + cache: 'npm' + cache-dependency-path: client/package-lock.json - - run: npm install --no-audit + - run: npm ci - run: npm test diff --git a/server/features/steps/steps.py b/server/features/steps/steps.py index cc6967e9d..12e119091 100644 --- a/server/features/steps/steps.py +++ b/server/features/steps/steps.py @@ -9,6 +9,7 @@ # at https://www.sourcefabric.org/superdesk/license import os +import responses from copy import deepcopy from unittest import mock from datetime import timedelta @@ -43,8 +44,7 @@ def step_impl_when_fetch_from_ntb_events_api_ingest(context, mock_requests, prov :return: """ - with mock.patch.object(http_base_service.requests, 'get') as get: - feeds = [] + with responses.RequestsMock() as rsps: with context.app.test_request_context(context.app.config['URL_PREFIX']): ingest_provider_service = get_resource_service('ingest_providers') provider = ingest_provider_service.find_one(name=provider_name, req=None) @@ -55,24 +55,7 @@ def step_impl_when_fetch_from_ntb_events_api_ingest(context, mock_requests, prov filename ) with open(file_path, 'rb') as f: - feeds.append(f.read()) - - def side_effect(*args, **kwargs): - m = mock.MagicMock() - m.ok = True - - if kwargs['params']['search.offset'] == 0: - m.content = feeds[0] - elif kwargs['params']['search.offset'] == 2: - m.content = feeds[1] - elif kwargs['params']['search.offset'] == 4: - m.content = feeds[2] - elif kwargs['params']['search.offset'] == 6: - m.content = feeds[3] - - return m - - get.side_effect = side_effect + rsps.get(provider["config"]["url"], body=f.read(), status=200) with context.app.app_context(): with mock.patch.object(update_ingest, 'is_scheduled', return_value=True): diff --git a/server/ntb/tests/io/feeding_services/event_http_service_tests.py b/server/ntb/tests/io/feeding_services/event_http_service_tests.py index 808abe550..3b3e98c02 100644 --- a/server/ntb/tests/io/feeding_services/event_http_service_tests.py +++ b/server/ntb/tests/io/feeding_services/event_http_service_tests.py @@ -1,42 +1,32 @@ import os -from unittest import mock +import responses -from superdesk.io.feeding_services import http_base_service -from planning.feeding_services import event_http_service from planning.tests import TestCase - - -class MockResponse: - def __init__(self, status_code, content): - self.status_code = status_code - self.content = content - self.headers = mock.MagicMock() +from planning.feeding_services import event_http_service class EventHTTPFeedingServiceTestCase(TestCase): - - def setUp(self): - super().setUp() - - @mock.patch.object(http_base_service, 'requests') - def test_update_ntb(self, requests): + @responses.activate(assert_all_requests_are_fired=True) + def test_update_ntb(self): dirname = os.path.dirname(os.path.realpath(__file__)) - fixture = os.path.normpath(os.path.join(dirname, '../fixtures', 'ntb_event.xml')) + fixture = os.path.normpath( + os.path.join(dirname, "../fixtures", "ntb_event.xml") + ) + + with open(fixture, "rb") as f: + body = f.read() - with open(fixture, 'rb') as f: - response = MockResponse(status_code=200, content=f.read()) - response.ok = True - requests.get.return_value = response + responses.get("https://example.com/NTBEvent.xml", body=body, status=200) service = event_http_service.EventHTTPFeedingService() provider = { - 'feed_parser': 'ntb_event_xml', - 'config': { - 'url': 'https://example.com/NTBEvent.xml', + "feed_parser": "ntb_event_xml", + "config": { + "url": "https://example.com/NTBEvent.xml", }, - 'is_closed': False, - '_id': 'ntb_event_xml', - 'name': 'ntb_event_xml', + "is_closed": False, + "_id": "ntb_event_xml", + "name": "ntb_event_xml", } events = list(service.update(provider, {})) diff --git a/server/ntb/tests/io/feeding_services/ntb_event_api_test.py b/server/ntb/tests/io/feeding_services/ntb_event_api_test.py index ee4411914..cad185b72 100644 --- a/server/ntb/tests/io/feeding_services/ntb_event_api_test.py +++ b/server/ntb/tests/io/feeding_services/ntb_event_api_test.py @@ -1,131 +1,91 @@ import os -from unittest import mock +import ntb +import responses +from responses import matchers from superdesk.tests import TestCase -from apps.prepopulate.app_populate import AppPopulateCommand -import ntb -from superdesk.io.feeding_services import http_base_service from ntb.io.feeding_services import ntb_event_api from planning.events import init_app as init_events_app +from apps.prepopulate.app_populate import AppPopulateCommand PROVIDER = { - '_id': 'test_ntb_events_api_xml', - 'feed_parser': 'ntb_events_api_xml', - 'config': { - 'url': 'https://nyheter.ntb.no/ntbWeb/api/x1/search/full?' - 'search.service=newscalendar&' - 'search.calendarStart=2018-10-01&' - 'search.calendarStop=2018-10-31', - 'username': 'fake', - 'password': 'fake' + "_id": "test_ntb_events_api_xml", + "feed_parser": "ntb_events_api_xml", + "config": { + "url": "https://nyheter.ntb.no/ntbWeb/api/x1/search/full?" + "search.service=newscalendar&" + "search.calendarStart=2018-10-01&" + "search.calendarStop=2018-10-31", + "username": "fake", + "password": "fake", }, } class NTBEventsApiFeedingServiceTestCase(TestCase): - def setUp(self): super().setUp() with self.app.app_context(): # prepopulate vocabularies voc_file = os.path.join( - os.path.abspath(os.path.dirname(os.path.dirname(ntb.__file__))), 'data', 'vocabularies.json' + os.path.abspath(os.path.dirname(os.path.dirname(ntb.__file__))), + "data", + "vocabularies.json", ) AppPopulateCommand().run(voc_file) # by default events resource is not available init_events_app(self.app) # NTBEventsApiFeedingService does 4 request during 1 update, - # to mock returning of different results (requests.get) self._side_effect is used self.feeds = [] for i in range(4): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( - os.path.join(dirname, '../fixtures', 'ntb_events_api', '{}.xml'.format(i)) + os.path.join( + dirname, "../fixtures", "ntb_events_api", "{}.xml".format(i) + ) ) - with open(fixture, 'rb') as f: + with open(fixture, "rb") as f: self.feeds.append(f.read()) - def _side_effect(self, *args, **kwargs): - m = mock.MagicMock() - m.ok = True - - if kwargs['params']['search.offset'] == 0: - m.content = self.feeds[0] - elif kwargs['params']['search.offset'] == 2: - m.content = self.feeds[1] - elif kwargs['params']['search.offset'] == 4: - m.content = self.feeds[2] - elif kwargs['params']['search.offset'] == 6: - m.content = self.feeds[3] - - return m - - @mock.patch.object(http_base_service.requests, 'get') - def test_items_count(self, get): - get.side_effect = self._side_effect - feeding_service = ntb_event_api.NTBEventsApiFeedingService() - feeding_service.provider = PROVIDER - items = feeding_service._update(PROVIDER, {})[0] - - self.assertEqual(len(items), 8) - - @mock.patch.object(http_base_service.requests, 'get') - def test_items_count_ignore_duplicates(self, get): - feeding_service = ntb_event_api.NTBEventsApiFeedingService() - - get.return_value.ok = True - get.return_value.content = self.feeds[0] - feeding_service.provider = PROVIDER - items = feeding_service._update(PROVIDER, {})[0] - self.assertEqual(len(items), 2) - - @mock.patch.object(http_base_service.requests, 'get') - def test_requests_offset(self, get): - get.side_effect = self._side_effect - feeding_service = ntb_event_api.NTBEventsApiFeedingService() - feeding_service.provider = PROVIDER - feeding_service._update(PROVIDER, {}) - - self.assertEqual( - get.mock_calls[0], - mock.call( - PROVIDER['config']['url'], - auth=('fake', 'fake'), - params={'search.offset': 0, 'search.showNumResults': 25}, - timeout=20 - ) + @responses.activate(assert_all_requests_are_fired=True) + def test_requests_offset(self): + responses.get( + "https://nyheter.ntb.no/ntbWeb/api/x1/search/full", + match=( + matchers.query_param_matcher({"search.offset": 0}, strict_match=False), + ), + body=self.feeds[0], + status=200, ) - - self.assertEqual( - get.mock_calls[1], - mock.call( - PROVIDER['config']['url'], - auth=('fake', 'fake'), - params={'search.offset': 2, 'search.showNumResults': 25}, - timeout=20 - ) + responses.get( + "https://nyheter.ntb.no/ntbWeb/api/x1/search/full", + match=( + matchers.query_param_matcher({"search.offset": 2}, strict_match=False), + ), + body=self.feeds[1], + status=200, ) - - self.assertEqual( - get.mock_calls[2], - mock.call( - PROVIDER['config']['url'], - auth=('fake', 'fake'), - params={'search.offset': 4, 'search.showNumResults': 25}, - timeout=20 - ) + responses.get( + "https://nyheter.ntb.no/ntbWeb/api/x1/search/full", + match=( + matchers.query_param_matcher({"search.offset": 4}, strict_match=False), + ), + body=self.feeds[2], + status=200, ) - - self.assertEqual( - get.mock_calls[3], - mock.call( - PROVIDER['config']['url'], - auth=('fake', 'fake'), - params={'search.offset': 6, 'search.showNumResults': 25}, - timeout=20 - ) + responses.get( + "https://nyheter.ntb.no/ntbWeb/api/x1/search/full", + match=( + matchers.query_param_matcher({"search.offset": 6}, strict_match=False), + ), + body=self.feeds[3], + status=200, ) + feeding_service = ntb_event_api.NTBEventsApiFeedingService() + feeding_service.provider = PROVIDER + items = feeding_service._update(PROVIDER, {})[0] + assert 8 == len(items) diff --git a/server/setup.cfg b/server/setup.cfg index 7ca251911..cdfaf1c6a 100644 --- a/server/setup.cfg +++ b/server/setup.cfg @@ -5,7 +5,7 @@ ignore=F811,D200,D202,D205,D400,D401,D100,D101,D102,D103,D104,D105,D107,W503,W50 # W504, W605, F401, E261 and F841 are temporally ignored, due to recent changes in flake8 [mypy] -python_version = 3.6 +python_version = 3.8 allow_untyped_globals = True ignore_missing_imports = True exclude = src