-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #108 from mdhaman/1.1
Updating cores.
- Loading branch information
Showing
4 changed files
with
167 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| # -*- coding: utf-8; -*- | ||
| # | ||
| # This file is part of Superdesk. | ||
| # | ||
| # Copyright 2013, 2014 Sourcefabric z.u. and contributors. | ||
| # | ||
| # For the full copyright and license information, please see the | ||
| # AUTHORS and LICENSE files distributed with this source code, or | ||
| # at https://www.sourcefabric.org/superdesk/license | ||
|
|
||
| import re | ||
| from bs4 import BeautifulSoup | ||
| from io import StringIO | ||
| from flask import current_app as app | ||
| from apps.archive.common import format_dateline_to_locmmmddsrc | ||
| from superdesk.utc import get_date | ||
| from datetime import datetime | ||
| from pytz import timezone | ||
| from superdesk.errors import SuperdeskApiError | ||
| import superdesk | ||
|
|
||
|
|
||
| def ap_weather_format(item, **kwargs): | ||
| if not item.get('slugline', '').startswith('WEA--GlobalWeather-Ce') or not item.get('source', '') == 'AP': | ||
| raise SuperdeskApiError.badRequestError("Article should be an AP sourced weather table") | ||
| item['slugline'] = 'WORLD WEATHER' | ||
|
|
||
| soup = BeautifulSoup(item['body_html'], 'html.parser') | ||
| lines = soup.get_text().splitlines() | ||
| if not lines[0] == 'BC-WEA--Global Weather-Celsius,<': | ||
| raise SuperdeskApiError.badRequestError("Table should be in Celsius only") | ||
|
|
||
| # tabular column max lengths are extracted into this list | ||
| columns = [] | ||
| # map of the columns to extract and the substitutions to apply to the column | ||
| columnMap = ({'index': 0}, {'index': 1}, {'index': 2}, | ||
| {'index': 3, 'substitute': [('COND', 'CONDITIONS'), | ||
| ('pc', 'partly cloudy'), ('clr', 'clear'), | ||
| ('cdy', 'cloudy'), ('rn', 'rain'), | ||
| ('sn', 'snow')]}) | ||
| # story preamble | ||
| preamble = 'Temperatures and conditions in world centres:\r\n' | ||
| output = StringIO() | ||
| output.write(preamble) | ||
|
|
||
| # story is always datelined News York | ||
| city = 'New York City' | ||
| cities = app.locators.find_cities() | ||
| located = [c for c in cities if c['city'].lower() == city.lower()] | ||
| if 'dateline' not in item: | ||
| item['dateline'] = {} | ||
| item['dateline']['located'] = located[0] if len(located) > 0 else {'city_code': city, 'city': city, | ||
| 'tz': 'UTC', 'dateline': 'city'} | ||
| item['dateline']['date'] = datetime.fromtimestamp(get_date(item['firstcreated']).timestamp(), | ||
| tz=timezone(item['dateline']['located']['tz'])) | ||
| item['dateline']['source'] = 'AP' | ||
| item['dateline']['text'] = format_dateline_to_locmmmddsrc(item['dateline']['located'], | ||
| get_date(item['firstcreated']), | ||
| source=item.get('original_source', 'AP')) | ||
|
|
||
| item['headline'] = 'World Weather for ' + item['dateline']['date'].strftime('%b %-d') | ||
|
|
||
| item['subject'] = [{"name": "weather", "qcode": "17000000"}] | ||
| locator_map = superdesk.get_resource_service('vocabularies').find_one(req=None, _id='locators') | ||
| item['place'] = [x for x in locator_map.get('items', []) if x['qcode'] == 'US'] | ||
|
|
||
| if lines: | ||
| # scan all the lines in the file for potential collimated lines and calculate the length | ||
| # of the column | ||
| for line in lines: | ||
| row = re.split('[;\<]+', line) | ||
| # only consider it if there are more than two rows | ||
| if len(row) > 2: | ||
| index = 0 | ||
| for col in row: | ||
| # check if the column is mapped | ||
| map = [me for me in columnMap if me['index'] == index] | ||
| if len(map): | ||
| for sub in map[0].get('substitute', ()): | ||
| col = col.replace(sub[0], sub[1]) | ||
| # if it's a new column | ||
| if 0 <= index < len(columns): | ||
| # check the length | ||
| if len(col) > columns[index]: | ||
| columns[index] = len(col) | ||
| else: | ||
| columns.append(len(col)) | ||
| index += 1 | ||
|
|
||
| for line in lines: | ||
| row = re.split('[;\<]+', line) | ||
| if len(row) > 2: | ||
| index = 0 | ||
| for col in row: | ||
| map = [me for me in columnMap if me['index'] == index] | ||
| if len(map) > 0: | ||
| for sub in map[0].get('substitute', ()): | ||
| col = col.replace(sub[0], sub[1]) | ||
| output.write( | ||
| '{}'.format(col.lstrip('\t').ljust(columns[map[0].get('index')] + 2)).rstrip('\r\n')) | ||
| index += 1 | ||
| output.write('\r\n') | ||
|
|
||
| item['body_html'] = '<pre>' + output.getvalue() + '</pre>' | ||
| return item | ||
|
|
||
|
|
||
| name = 'Format AP Weather stories' | ||
| label = 'AP Weather' | ||
| callback = ap_weather_format | ||
| access_type = 'frontend' | ||
| action_type = 'direct' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| # -*- coding: utf-8; -*- | ||
| # | ||
| # This file is part of Superdesk. | ||
| # | ||
| # Copyright 2013, 2014 Sourcefabric z.u. and contributors. | ||
| # | ||
| # For the full copyright and license information, please see the | ||
| # AUTHORS and LICENSE files distributed with this source code, or | ||
| # at https://www.sourcefabric.org/superdesk/license | ||
|
|
||
| from test_factory import SuperdeskTestCase | ||
| from .ap_weather_format import ap_weather_format | ||
|
|
||
|
|
||
| class APWeatherTestCase(SuperdeskTestCase): | ||
| vocab = [{'_id': 'locators', 'items': [{'qcode': 'US'}]}] | ||
|
|
||
| def setUp(self): | ||
| super().setUp() | ||
| self.app.data.insert('vocabularies', self.vocab) | ||
|
|
||
| def test_weather_story(self): | ||
| text = '<pre>BC-WEA--Global Weather-Celsius,<\r\n' \ | ||
| '\t NEW YORK (AP) _ Minimum and maximum temperatures in Celsius, precipitation in' \ | ||
| ' centimeters and weather conditions as recorded for the previous day and forecast for the' \ | ||
| 'current and following day in each city as of 1400 GMT:<\r\n' \ | ||
| '\t ;MIN;MAX;COND;PRECIP;MIN;MAX;COND;MIN;MAX;COND<\r\n' \ | ||
| '\t Amsterdam;15;21;rn;3.0;17;25;clr;18;31;clr<\r\n' \ | ||
| '\t Athens;21;34;clr;0.0;24;31;pc;23;31;pc<\r\n' \ | ||
| '\t Atlanta;21;31;clr;0.0;22;32;pc;21;31;pc<\r\n' \ | ||
| '\t Auckland;7;16;rn;0.0;12;14;rn;14;15;rn<\r\n' \ | ||
| '\t Basra;29;48;clr;0.0;28;46;clr;27;48;clr<\r\n' \ | ||
| '\t Bahrain;33;39;clr;0.0;33;38;pc;33;39;clr<\r\n' \ | ||
| '\t Bangkok;25;35;rn;0.0;27;35;rn;27;34;rn<\r\n' \ | ||
| '\t Barbados;26;31;pc;0.0;26;31;pc;26;29;rn<\r\n' \ | ||
| '\t Barcelona;22;29;rn;0.0;19;28;clr;21;29;clr<\r\n' \ | ||
| '\t Beijing;24;34;cdy;0.0;24;33;clr;17;28;pc<\r\n' \ | ||
| '\t Beirut;25;31;clr;0.0;27;30;clr;27;31;clr<\r\n' \ | ||
| '\t Belgrade;16;18;rn;7.0;16;24;cdy;15;27;clr<\r\n' \ | ||
| '\t Berlin;11;23;pc;0.0;14;25;pc;14;27;pc<\r\n' \ | ||
| ' x - Indicates missing information.\r\n' \ | ||
| '\t clr - clear\r\n' \ | ||
| '\t pc - partly cloudy\r\n' \ | ||
| '\t cdy - cloudy\r\n' \ | ||
| '\t rn - rain\r\n' \ | ||
| '\t sn - snow\r\n' \ | ||
| '\t Source: Weather Underground<\r\n' \ | ||
| 'END<\r\n' \ | ||
| '\t </pre>' | ||
| item = {'body_html': text, 'slugline': 'WEA--GlobalWeather-Ce', 'source': 'AP', | ||
| 'firstcreated': "2016-08-23T14:32:00.000Z"} | ||
| res = ap_weather_format(item) | ||
| self.assertTrue(' Barbados 26 31 partly cloudy' in res['body_html']) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters