From 6426699b1de0881f99ee39060ab940f5f4c569c2 Mon Sep 17 00:00:00 2001
From: Dag Wieers <dag@wieers.com>
Date: Mon, 31 Aug 2020 21:42:20 +0200
Subject: [PATCH] Handle none-responses from open_url

This handles all none-responses from open_url().

I am not sure this is the best way to handle this, it is not very
elegant. Raising an exception helps in one case, but would not be
elegant in the other cases either.
---
 resources/lib/tokenresolver.py | 20 ++++++++++++++++----
 resources/lib/webscraper.py    |  7 ++++++-
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/resources/lib/tokenresolver.py b/resources/lib/tokenresolver.py
index 608989fac..e1faf29e9 100644
--- a/resources/lib/tokenresolver.py
+++ b/resources/lib/tokenresolver.py
@@ -199,7 +199,10 @@ def _get_xvrttoken(self, login_json=None):
         )
         data = dumps(payload).encode()
         headers = {'Content-Type': 'application/json', 'Cookie': login_cookie}
-        setcookie_header = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers).info().get('Set-Cookie')
+        response = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers)
+        if reponse is None:
+            return None
+        setcookie_header = response.info().get('Set-Cookie')
         xvrttoken = TokenResolver._create_token_dictionary(setcookie_header)
         if xvrttoken is None:
             return None
@@ -213,13 +216,22 @@ def _get_roaming_xvrttoken(self):
             return None
         cookie_value = 'vrtlogin-at=' + vrtlogin_at
         headers = {'Cookie': cookie_value}
-        req_info = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False).info()
+        response = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False)
+        if response is None:
+            return None
+        req_info = response.info()
         cookie_value += '; state=' + req_info.get('Set-Cookie').split('state=')[1].split('; ')[0]
-        url = open_url(req_info.get('Location'), follow_redirects=False).info().get('Location')
+        response = open_url(req_info.get('Location'), follow_redirects=False)
+        if response is None:
+            return None
+        url = response.info().get('Location')
         headers = {'Cookie': cookie_value}
         if url is None:
             return None
-        setcookie_header = open_url(url, headers=headers, follow_redirects=False).info().get('Set-Cookie')
+        response = open_url(url, headers=headers, follow_redirects=False)
+        if response ise None:
+            return None
+        setcookie_header = response.info().get('Set-Cookie')
         return TokenResolver._create_token_dictionary(setcookie_header)
 
     def get_token(self, name, variant=None, url=None, roaming=False):
diff --git a/resources/lib/webscraper.py b/resources/lib/webscraper.py
index 701f8a6b1..bddfa4c32 100644
--- a/resources/lib/webscraper.py
+++ b/resources/lib/webscraper.py
@@ -31,6 +31,8 @@ def get_categories():
     if not valid_categories(categories):
         from bs4 import BeautifulSoup, SoupStrainer
         response = open_url('https://www.vrt.be/vrtnu/categorieen/')
+        if response is None:
+            return categories
         tiles = SoupStrainer('nui-list--content')
         soup = BeautifulSoup(response.read(), 'html.parser', parse_only=tiles)
 
@@ -79,10 +81,13 @@ def get_video_attributes(vrtnu_url):
     # Scrape video attributes
     from bs4 import BeautifulSoup, SoupStrainer
     try:
-        html_page = open_url(vrtnu_url, raise_errors='all').read()
+        response = open_url(vrtnu_url, raise_errors='all')
     except HTTPError as exc:
         log_error('Web scraping video attributes failed: {error}', error=exc)
         return None
+    if response is None:
+        return None
+    html_page = response.read()
     strainer = SoupStrainer(['section', 'div'], {'class': ['video-player', 'livestream__inner']})
     soup = BeautifulSoup(html_page, 'html.parser', parse_only=strainer)
     item = None