From 0c952b105a5d0d419607910907222f0e215a8dec Mon Sep 17 00:00:00 2001
From: Dag Wieers <dag@wieers.com>
Date: Mon, 31 Aug 2020 21:42:20 +0200
Subject: [PATCH] Handle none-responses from open_url

This handles all none-responses from open_url().

I am not sure this is the best way to handle this, it is not very
elegant. Raising an exception helps in one case, but would not be
elegant in the other cases either.
---
 resources/lib/tokenresolver.py | 20 ++++++++++++++++----
 resources/lib/webscraper.py    |  7 ++++++-
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/resources/lib/tokenresolver.py b/resources/lib/tokenresolver.py
index 608989fac..bbad8fa7f 100644
--- a/resources/lib/tokenresolver.py
+++ b/resources/lib/tokenresolver.py
@@ -199,7 +199,10 @@ def _get_xvrttoken(self, login_json=None):
         )
         data = dumps(payload).encode()
         headers = {'Content-Type': 'application/json', 'Cookie': login_cookie}
-        setcookie_header = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers).info().get('Set-Cookie')
+        response = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers)
+        if reponse is None:
+            return None
+        setcookie_header = response.info().get('Set-Cookie')
         xvrttoken = TokenResolver._create_token_dictionary(setcookie_header)
         if xvrttoken is None:
             return None
@@ -213,13 +216,22 @@ def _get_roaming_xvrttoken(self):
             return None
         cookie_value = 'vrtlogin-at=' + vrtlogin_at
         headers = {'Cookie': cookie_value}
-        req_info = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False).info()
+        response = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False)
+        if response is None:
+            return None
+        req_info = response.info()
         cookie_value += '; state=' + req_info.get('Set-Cookie').split('state=')[1].split('; ')[0]
-        url = open_url(req_info.get('Location'), follow_redirects=False).info().get('Location')
+        response = open_url(req_info.get('Location'), follow_redirects=False)
+        if response is None:
+            return None
+        url = response.info().get('Location')
         headers = {'Cookie': cookie_value}
         if url is None:
             return None
-        setcookie_header = open_url(url, headers=headers, follow_redirects=False).info().get('Set-Cookie')
+        response = open_url(url, headers=headers, follow_redirects=False)
+        if response is None:
+            return None
+        setcookie_header = response.info().get('Set-Cookie')
         return TokenResolver._create_token_dictionary(setcookie_header)
 
     def get_token(self, name, variant=None, url=None, roaming=False):
diff --git a/resources/lib/webscraper.py b/resources/lib/webscraper.py
index 701f8a6b1..bddfa4c32 100644
--- a/resources/lib/webscraper.py
+++ b/resources/lib/webscraper.py
@@ -31,6 +31,8 @@ def get_categories():
     if not valid_categories(categories):
         from bs4 import BeautifulSoup, SoupStrainer
         response = open_url('https://www.vrt.be/vrtnu/categorieen/')
+        if response is None:
+            return categories
         tiles = SoupStrainer('nui-list--content')
         soup = BeautifulSoup(response.read(), 'html.parser', parse_only=tiles)
 
@@ -79,10 +81,13 @@ def get_video_attributes(vrtnu_url):
     # Scrape video attributes
     from bs4 import BeautifulSoup, SoupStrainer
     try:
-        html_page = open_url(vrtnu_url, raise_errors='all').read()
+        response = open_url(vrtnu_url, raise_errors='all')
     except HTTPError as exc:
         log_error('Web scraping video attributes failed: {error}', error=exc)
         return None
+    if response is None:
+        return None
+    html_page = response.read()
     strainer = SoupStrainer(['section', 'div'], {'class': ['video-player', 'livestream__inner']})
     soup = BeautifulSoup(html_page, 'html.parser', parse_only=strainer)
     item = None