Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Attempt to use json.load(fd) again #624

Merged
merged 3 commits into from
Dec 9, 2019
Merged

Attempt to use json.load(fd) again #624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c8e71ed
Attempt to use json.load(fdesc) again
dagwieers Dec 9, 2019
85d065a
Support json HTTP error responses
mediaminister Dec 9, 2019
a94bff0
Attempt to use json.load(fdesc) again
dagwieers Dec 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion resources/lib/addon.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from urllib import unquote_plus

from kodiutils import localize, log_access, notification, refresh_caches
from statichelper import from_unicode, to_unicode
from utils import from_unicode, to_unicode

plugin = Plugin() # pylint: disable=invalid-name

Expand Down
2 changes: 1 addition & 1 deletion resources/lib/addon_entry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
''' This is the actual VRT NU video plugin entry point '''

from __future__ import absolute_import, division, unicode_literals
import kodiutils
import xbmcaddon
import kodiutils

kodiutils.ADDON = xbmcaddon.Addon()

Expand Down
83 changes: 26 additions & 57 deletions resources/lib/apihelper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,20 @@
from __future__ import absolute_import, division, unicode_literals

try: # Python 3
from urllib.error import HTTPError
from urllib.parse import quote_plus, unquote
from urllib.request import build_opener, install_opener, ProxyHandler, Request, urlopen
from urllib.request import build_opener, install_opener, ProxyHandler, urlopen
except ImportError: # Python 2
from urllib import quote_plus
from urllib2 import build_opener, install_opener, ProxyHandler, Request, HTTPError, unquote, urlopen
from urllib2 import build_opener, install_opener, ProxyHandler, unquote, urlopen

from data import CHANNELS
from helperobjects import TitleItem
from kodiutils import (delete_cached_thumbnail, get_cache, get_global_setting, get_proxies, get_setting,
has_addon, localize, localize_from_data, log, log_error, ok_dialog, ttl, update_cache,
url_for)
from statichelper import (add_https_method, convert_html_to_kodilabel, find_entry, from_unicode, play_url_to_id,
program_to_url, realpage, to_unicode, strip_newlines, url_to_program)
from kodiutils import (delete_cached_thumbnail, get_cache, get_cached_url_json, get_global_setting,
get_proxies, get_setting, get_url_json, has_addon, localize, localize_from_data,
log, ttl, update_cache, url_for)
from metadata import Metadata
from utils import (add_https_proto, html_to_kodilabel, find_entry, from_unicode, play_url_to_id,
program_to_url, realpage, strip_newlines, url_to_program)


class ApiHelper:
Expand All @@ -39,7 +38,7 @@ def __init__(self, _favorites, _resumepoints):

def get_tvshows(self, category=None, channel=None, feature=None):
''' Get all TV shows for a given category, channel or feature, optionally filtered by favorites '''
params = dict()
params = {}

if category:
params['facets[categories]'] = category
Expand All @@ -57,16 +56,10 @@ def get_tvshows(self, category=None, channel=None, feature=None):
if not category and not channel and not feature:
params['facets[transcodingStatus]'] = 'AVAILABLE' # Required for getting results in Suggests API
cache_file = 'programs.json'
tvshows = get_cache(cache_file, ttl=ttl('indirect')) # Try the cache if it is fresh
if not tvshows:
from json import loads
querystring = '&'.join('{}={}'.format(key, value) for key, value in list(params.items()))
suggest_url = self._VRTNU_SUGGEST_URL + '?' + querystring
log(2, 'URL get: {url}', url=unquote(suggest_url))
tvshows = loads(to_unicode(urlopen(suggest_url).read()))
update_cache(cache_file, tvshows)

return tvshows
querystring = '&'.join('{}={}'.format(key, value) for key, value in list(params.items()))
suggest_url = self._VRTNU_SUGGEST_URL + '?' + querystring
return get_cached_url_json(url=suggest_url, cache=cache_file, ttl=ttl('indirect'), fail=[])

def list_tvshows(self, category=None, channel=None, feature=None, use_favorites=False):
''' List all TV shows for a given category, channel or feature, optionally filtered by favorites '''
Expand Down Expand Up @@ -151,7 +144,7 @@ def __map_episodes(self, episodes, titletype=None, season=None, use_favorites=Fa
highlight = episode.get('highlight')
if highlight:
for key in highlight:
episode[key] = convert_html_to_kodilabel(highlight.get(key)[0])
episode[key] = html_to_kodilabel(highlight.get(key)[0])

list_item, sort, ascending = self.episode_to_listitem(episode, program, cache_file, titletype)
episode_items.append(list_item)
Expand Down Expand Up @@ -268,7 +261,7 @@ def get_upnext(self, info):

# Get all episodes from current program and sort by program, seasonTitle and episodeNumber
episodes = sorted(self.get_episodes(keywords=program), key=lambda k: (k.get('program'), k.get('seasonTitle'), k.get('episodeNumber')))
upnext = dict()
upnext = {}
for episode in episodes:
if ep_id.get('whatson_id') == episode.get('whatsonId') or \
ep_id.get('video_id') == episode.get('videoId') or \
Expand Down Expand Up @@ -413,8 +406,7 @@ def get_episode_by_air_date(self, channel_name, start_date, end_date=None):
schedule_date = onairdate
schedule_datestr = schedule_date.isoformat().split('T')[0]
url = 'https://www.vrt.be/bin/epg/schedule.%s.json' % schedule_datestr
from json import loads
schedule_json = loads(to_unicode(urlopen(url).read()))
schedule_json = get_url_json(url, fail={})
episodes = schedule_json.get(channel.get('id'), [])
if not episodes:
return None
Expand Down Expand Up @@ -569,41 +561,17 @@ def get_episodes(self, program=None, season=None, episodes=None, category=None,
# Construct VRT NU Search API Url and get api data
querystring = '&'.join('{}={}'.format(key, value) for key, value in list(params.items()))
search_url = self._VRTNU_SEARCH_URL + '?' + querystring.replace(' ', '%20') # Only encode spaces to minimize url length

from json import loads
if cache_file:
# Get api data from cache if it is fresh
search_json = get_cache(cache_file, ttl=ttl('indirect'))
if not search_json:
log(2, 'URL get: {url}', url=unquote(search_url))
req = Request(search_url)
try:
search_json = loads(to_unicode(urlopen(req).read()))
except (TypeError, ValueError): # No JSON object could be decoded
return []
except HTTPError as exc:
url_length = len(req.get_selector())
if exc.code == 413 and url_length > 8192:
ok_dialog(heading='HTTP Error 413', message=localize(30967))
log_error('HTTP Error 413: Exceeded maximum url length: '
'VRT Search API url has a length of {length} characters.', length=url_length)
return []
if exc.code == 400 and 7600 <= url_length <= 8192:
ok_dialog(heading='HTTP Error 400', message=localize(30967))
log_error('HTTP Error 400: Probably exceeded maximum url length: '
'VRT Search API url has a length of {length} characters.', length=url_length)
return []
raise
update_cache(cache_file, search_json)
search_json = get_cached_url_json(url=search_url, cache=cache_file, ttl=ttl('indirect'), fail={})
else:
log(2, 'URL get: {url}', url=unquote(search_url))
search_json = loads(to_unicode(urlopen(search_url).read()))
search_json = get_url_json(url=search_url, fail={})

# Check for multiple seasons
seasons = None
seasons = []
if 'facets[seasonTitle]' not in unquote(search_url):
facets = search_json.get('facets', dict()).get('facets')
seasons = next((f.get('buckets', []) for f in facets if f.get('name') == 'seasons' and len(f.get('buckets', [])) > 1), None)
facets = search_json.get('facets', {}).get('facets')
if facets:
seasons = next((f.get('buckets', []) for f in facets if f.get('name') == 'seasons' and len(f.get('buckets', [])) > 1), None)

episodes = search_json.get('results', [{}])
show_seasons = bool(season != 'allseasons')
Expand All @@ -619,8 +587,9 @@ def get_episodes(self, program=None, season=None, episodes=None, category=None,
if all_items and total_results > api_page_size:
for api_page in range(1, api_pages):
api_page_url = search_url + '&from=' + str(api_page * api_page_size + 1)
api_page_json = loads(to_unicode(urlopen(api_page_url).read()))
episodes += api_page_json.get('results', [{}])
api_page_json = get_url_json(api_page_url)
if api_page_json is not None:
episodes += api_page_json.get('results', [{}])

# Return episodes
return episodes
Expand All @@ -642,7 +611,7 @@ def list_channels(self, channels=None, live=True):
continue

context_menu = []
art_dict = dict()
art_dict = {}

# Try to use the white icons for thumbnails (used for icons as well)
if has_addon('resource.images.studios.white'):
Expand Down Expand Up @@ -713,7 +682,7 @@ def list_youtube(channels=None):
continue

context_menu = []
art_dict = dict()
art_dict = {}

# Try to use the white icons for thumbnails (used for icons as well)
if has_addon('resource.images.studios.white'):
Expand Down Expand Up @@ -850,7 +819,7 @@ def get_category_thumbnail(element):
''' Return a category thumbnail, if available '''
if get_setting('showfanart', 'true') == 'true':
raw_thumbnail = element.find(class_='media').get('data-responsive-image', 'DefaultGenre.png')
return add_https_method(raw_thumbnail)
return add_https_proto(raw_thumbnail)
return 'DefaultGenre.png'

@staticmethod
Expand Down
28 changes: 10 additions & 18 deletions resources/lib/favorites.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@
except ImportError: # Python 2
from urllib2 import build_opener, install_opener, ProxyHandler, Request, unquote, urlopen

from kodiutils import (container_refresh, get_cache, get_proxies, get_setting, has_credentials,
input_down, invalidate_caches, localize, log, log_error, multiselect,
notification, ok_dialog, to_unicode, update_cache)
from kodiutils import (container_refresh, get_cache, get_proxies, get_setting, get_url_json,
has_credentials, input_down, invalidate_caches, localize, log, log_error,
multiselect, notification, ok_dialog, update_cache)


class Favorites:
Expand Down Expand Up @@ -43,17 +43,9 @@ def refresh(self, ttl=None):
'content-type': 'application/json',
'Referer': 'https://www.vrt.be/vrtnu',
}
req = Request('https://video-user-data.vrt.be/favorites', headers=headers)
log(2, 'URL get: https://video-user-data.vrt.be/favorites')
from json import loads
try:
favorites_json = loads(to_unicode(urlopen(req).read()))
except (TypeError, ValueError): # No JSON object could be decoded
# Force favorites from cache
favorites_json = get_cache('favorites.json', ttl=None)
else:
update_cache('favorites.json', favorites_json)
if favorites_json:
favorites_url = 'https://video-user-data.vrt.be/favorites'
favorites_json = get_url_json(url=favorites_url, cache='favorites.json', headers=headers)
if favorites_json is not None:
self._favorites = favorites_json

def update(self, program, title, value=True):
Expand All @@ -78,9 +70,9 @@ def update(self, program, title, value=True):
'Referer': 'https://www.vrt.be/vrtnu',
}

from statichelper import program_to_url
payload = dict(isFavorite=value, programUrl=program_to_url(program, 'short'), title=title)
from json import dumps
from utils import program_to_url
payload = dict(isFavorite=value, programUrl=program_to_url(program, 'short'), title=title)
data = dumps(payload).encode('utf-8')
program_id = self.program_to_id(program)
log(2, 'URL post: https://video-user-data.vrt.be/favorites/{program_id}', program_id=program_id)
Expand Down Expand Up @@ -132,12 +124,12 @@ def titles(self):

def programs(self):
''' Return all favorite programs '''
from statichelper import url_to_program
from utils import url_to_program
return [url_to_program(value.get('value').get('programUrl')) for value in list(self._favorites.values()) if value.get('value').get('isFavorite')]

def manage(self):
''' Allow the user to unselect favorites to be removed from the listing '''
from statichelper import url_to_program
from utils import url_to_program
self.refresh(ttl=0)
if not self._favorites:
ok_dialog(heading=localize(30418), message=localize(30419)) # No favorites found
Expand Down
Loading