diff --git a/client/package.json b/client/package.json index 8d95205e6..1ee3661db 100644 --- a/client/package.json +++ b/client/package.json @@ -2,7 +2,7 @@ "name": "superdesk", "license": "GPL-3.0", "dependencies": { - "superdesk-core": "superdesk/superdesk-client-core#b4ca7cce6", + "superdesk-core": "superdesk/superdesk-client-core#da74ed1665", "babel-core": "^6.10.4", "babel-loader": "^6.2.4", "babel-preset-es2015": "^6.9.0", diff --git a/server/aap/macros/ap_weather_format.py b/server/aap/macros/ap_weather_format.py new file mode 100644 index 000000000..cad8c47fa --- /dev/null +++ b/server/aap/macros/ap_weather_format.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8; -*- +# +# This file is part of Superdesk. +# +# Copyright 2013, 2014 Sourcefabric z.u. and contributors. +# +# For the full copyright and license information, please see the +# AUTHORS and LICENSE files distributed with this source code, or +# at https://www.sourcefabric.org/superdesk/license + +import re +from bs4 import BeautifulSoup +from io import StringIO +from flask import current_app as app +from apps.archive.common import format_dateline_to_locmmmddsrc +from superdesk.utc import get_date +from datetime import datetime +from pytz import timezone +from superdesk.errors import SuperdeskApiError +import superdesk + + +def ap_weather_format(item, **kwargs): + if not item.get('slugline', '').startswith('WEA--GlobalWeather-Ce') or not item.get('source', '') == 'AP': + raise SuperdeskApiError.badRequestError("Article should be an AP sourced weather table") + item['slugline'] = 'WORLD WEATHER' + + soup = BeautifulSoup(item['body_html'], 'html.parser') + lines = soup.get_text().splitlines() + if not lines[0] == 'BC-WEA--Global Weather-Celsius,<': + raise SuperdeskApiError.badRequestError("Table should be in Celsius only") + + # tabular column max lengths are extracted into this list + columns = [] + # map of the columns to extract and the substitutions to apply to the column + columnMap = ({'index': 0}, {'index': 1}, {'index': 2}, + {'index': 3, 'substitute': [('COND', 'CONDITIONS'), + ('pc', 'partly cloudy'), ('clr', 'clear'), + ('cdy', 'cloudy'), ('rn', 'rain'), + ('sn', 'snow')]}) + # story preamble + preamble = 'Temperatures and conditions in world centres:\r\n' + output = StringIO() + output.write(preamble) + + # story is always datelined News York + city = 'New York City' + cities = app.locators.find_cities() + located = [c for c in cities if c['city'].lower() == city.lower()] + if 'dateline' not in item: + item['dateline'] = {} + item['dateline']['located'] = located[0] if len(located) > 0 else {'city_code': city, 'city': city, + 'tz': 'UTC', 'dateline': 'city'} + item['dateline']['date'] = datetime.fromtimestamp(get_date(item['firstcreated']).timestamp(), + tz=timezone(item['dateline']['located']['tz'])) + item['dateline']['source'] = 'AP' + item['dateline']['text'] = format_dateline_to_locmmmddsrc(item['dateline']['located'], + get_date(item['firstcreated']), + source=item.get('original_source', 'AP')) + + item['headline'] = 'World Weather for ' + item['dateline']['date'].strftime('%b %-d') + + item['subject'] = [{"name": "weather", "qcode": "17000000"}] + locator_map = superdesk.get_resource_service('vocabularies').find_one(req=None, _id='locators') + item['place'] = [x for x in locator_map.get('items', []) if x['qcode'] == 'US'] + + if lines: + # scan all the lines in the file for potential collimated lines and calculate the length + # of the column + for line in lines: + row = re.split('[;\<]+', line) + # only consider it if there are more than two rows + if len(row) > 2: + index = 0 + for col in row: + # check if the column is mapped + map = [me for me in columnMap if me['index'] == index] + if len(map): + for sub in map[0].get('substitute', ()): + col = col.replace(sub[0], sub[1]) + # if it's a new column + if 0 <= index < len(columns): + # check the length + if len(col) > columns[index]: + columns[index] = len(col) + else: + columns.append(len(col)) + index += 1 + + for line in lines: + row = re.split('[;\<]+', line) + if len(row) > 2: + index = 0 + for col in row: + map = [me for me in columnMap if me['index'] == index] + if len(map) > 0: + for sub in map[0].get('substitute', ()): + col = col.replace(sub[0], sub[1]) + output.write( + '{}'.format(col.lstrip('\t').ljust(columns[map[0].get('index')] + 2)).rstrip('\r\n')) + index += 1 + output.write('\r\n') + + item['body_html'] = '
' + output.getvalue() + '
' + return item + + +name = 'Format AP Weather stories' +label = 'AP Weather' +callback = ap_weather_format +access_type = 'frontend' +action_type = 'direct' diff --git a/server/aap/macros/ap_weather_format_test.py b/server/aap/macros/ap_weather_format_test.py new file mode 100644 index 000000000..230317e9b --- /dev/null +++ b/server/aap/macros/ap_weather_format_test.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8; -*- +# +# This file is part of Superdesk. +# +# Copyright 2013, 2014 Sourcefabric z.u. and contributors. +# +# For the full copyright and license information, please see the +# AUTHORS and LICENSE files distributed with this source code, or +# at https://www.sourcefabric.org/superdesk/license + +from test_factory import SuperdeskTestCase +from .ap_weather_format import ap_weather_format + + +class APWeatherTestCase(SuperdeskTestCase): + vocab = [{'_id': 'locators', 'items': [{'qcode': 'US'}]}] + + def setUp(self): + super().setUp() + self.app.data.insert('vocabularies', self.vocab) + + def test_weather_story(self): + text = '
BC-WEA--Global Weather-Celsius,<\r\n' \
+               '\t   NEW YORK (AP) _ Minimum and maximum temperatures in Celsius, precipitation in' \
+               ' centimeters and weather conditions as recorded for the previous day and forecast for the' \
+               'current and following day in each city as of 1400 GMT:<\r\n' \
+               '\t   ;MIN;MAX;COND;PRECIP;MIN;MAX;COND;MIN;MAX;COND<\r\n' \
+               '\t   Amsterdam;15;21;rn;3.0;17;25;clr;18;31;clr<\r\n' \
+               '\t   Athens;21;34;clr;0.0;24;31;pc;23;31;pc<\r\n' \
+               '\t   Atlanta;21;31;clr;0.0;22;32;pc;21;31;pc<\r\n' \
+               '\t   Auckland;7;16;rn;0.0;12;14;rn;14;15;rn<\r\n' \
+               '\t   Basra;29;48;clr;0.0;28;46;clr;27;48;clr<\r\n' \
+               '\t   Bahrain;33;39;clr;0.0;33;38;pc;33;39;clr<\r\n' \
+               '\t   Bangkok;25;35;rn;0.0;27;35;rn;27;34;rn<\r\n' \
+               '\t   Barbados;26;31;pc;0.0;26;31;pc;26;29;rn<\r\n' \
+               '\t   Barcelona;22;29;rn;0.0;19;28;clr;21;29;clr<\r\n' \
+               '\t   Beijing;24;34;cdy;0.0;24;33;clr;17;28;pc<\r\n' \
+               '\t   Beirut;25;31;clr;0.0;27;30;clr;27;31;clr<\r\n' \
+               '\t   Belgrade;16;18;rn;7.0;16;24;cdy;15;27;clr<\r\n' \
+               '\t   Berlin;11;23;pc;0.0;14;25;pc;14;27;pc<\r\n' \
+               '      x - Indicates missing information.\r\n' \
+               '\t      clr - clear\r\n' \
+               '\t      pc - partly cloudy\r\n' \
+               '\t      cdy - cloudy\r\n' \
+               '\t      rn - rain\r\n' \
+               '\t      sn - snow\r\n' \
+               '\t      Source: Weather Underground<\r\n' \
+               'END<\r\n' \
+               '\t   
' + item = {'body_html': text, 'slugline': 'WEA--GlobalWeather-Ce', 'source': 'AP', + 'firstcreated': "2016-08-23T14:32:00.000Z"} + res = ap_weather_format(item) + self.assertTrue(' Barbados 26 31 partly cloudy' in res['body_html']) diff --git a/server/requirements.txt b/server/requirements.txt index f44a114f5..76a039543 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -4,5 +4,5 @@ pyodbc==3.0.10 unidecode==0.04.19 -git+git://github.com/superdesk/superdesk-core.git@a3965a579#egg=Superdesk-Core +git+git://github.com/superdesk/superdesk-core.git@efe93a81d#egg=Superdesk-Core