Add options to set HTTP timeouts

This patch adds the ability to set the cURL options `CONNECTTIMEOUT` and
`TIMEOUT` for HTTP requests.

`CONNECTTIMEOUT` is applied to all HTTP requests while `TIMEOUT` is only
applied to non-ingest requests. The defaults are set to 180s and 300s,
respectively.

etc/pyca.conf

@@ -278,3 +278,19 @@ password         = 'CHANGE_ME'
 # Log format configuration
 # Default: [%(name)s:%(lineno)s:%(funcName)s()] [%(levelname)s] %(message)s
 #format           = [%(name)s:%(lineno)s:%(funcName)s()] [%(levelname)s] %(message)s
+
+
+[http]
+
+# Number of seconds after which non-upload HTTP requests will time out.
+# Note that this is the total request duration including the establishment of the connection. Also see
+# connection_timeout to set a timeout for the latter. Setting this to 0 will disable a time out for HTTP requests.
+# HTTP requests that ingest files will not time out.
+# Default: 300
+#timeout          = 300
+
+# Number of seconds after which the connection establishment of HTTP requests will time out.
+# This applies to all HTTP requests pyCA will make including upload requests. The connection is considered established
+# after the TCP, TLS or QUIC handshakes.
+# Default: 180
+#connection_timeout = 180

pyca/config.py

@@ -64,6 +64,10 @@
 level            = option('debug', 'info', 'warning', 'error', default='info')
 format           = string(default='[%(name)s:%(lineno)s:%(funcName)s()] [%(levelname)s] %(message)s')
 
+[http]
+timeout          = integer(min=0, default=300)
+connection_timeout = integer(min=0, default=180)
+
 [services]
 '''  # noqa
 

pyca/ingest.py

@@ -59,7 +59,7 @@ def ingest(event):
 
     # create mediapackage
     logger.info('Creating new mediapackage')
-    mediapackage = http_request(service_url + '/createMediaPackage')
+    mediapackage = http_request(service_url + '/createMediaPackage', timeout=0)
 
     # extract workflow_def, workflow_config and add DC catalogs
     prop = 'org.opencastproject.capture.agent.properties'
@@ -78,7 +78,8 @@ def ingest(event):
             fields = [('mediaPackage', mediapackage),
                       ('flavor', 'dublincore/%s' % name),
                       ('dublinCore', data.encode('utf-8'))]
-            mediapackage = http_request(service_url + '/addDCCatalog', fields)
+            mediapackage = http_request(service_url + '/addDCCatalog', fields,
+                                        timeout=0)
 
         else:
             logger.info('Not uploading %s', attachment.get('x-apple-filename'))
@@ -90,7 +91,8 @@ def ingest(event):
         track = track.encode('ascii', 'ignore')
         fields = [('mediaPackage', mediapackage), ('flavor', flavor),
                   ('BODY1', (pycurl.FORM_FILE, track))]
-        mediapackage = http_request(service_url + '/addTrack', fields)
+        mediapackage = http_request(service_url + '/addTrack', fields,
+                                    timeout=0)
 
     # ingest
     logger.info('Ingest recording')
@@ -101,7 +103,7 @@ def ingest(event):
         fields.append(('workflowInstanceId',
                        event.uid.encode('ascii', 'ignore')))
     fields += workflow_config
-    mediapackage = http_request(service_url + '/ingest', fields)
+    mediapackage = http_request(service_url + '/ingest', fields, timeout=0)
 
     # Update status
     recording_state(event.uid, 'upload_finished')

pyca/ui/opencast_commands.py

@@ -53,7 +53,7 @@ def schedule(title='pyCA Recording', duration=60, creator=None):
 
     # create media package
     logger.info('Creating new media package')
-    mediapackage = http_request(service_url + '/createMediaPackage')
+    mediapackage = http_request(service_url + '/createMediaPackage', timeout=0)
 
     # add dublin core catalog
     start = datetime.utcnow() + timedelta(seconds=10)
@@ -68,12 +68,13 @@ def schedule(title='pyCA Recording', duration=60, creator=None):
     fields = [('mediaPackage', mediapackage),
               ('flavor', 'dublincore/episode'),
               ('dublinCore', dublincore)]
-    mediapackage = http_request(service_url + '/addDCCatalog', fields)
+    mediapackage = http_request(service_url + '/addDCCatalog', fields,
+                                timeout=0)
 
     # schedule event
     logger.info('Scheduling recording')
     fields = [('mediaPackage', mediapackage)]
-    mediapackage = http_request(service_url + '/schedule', fields)
+    mediapackage = http_request(service_url + '/schedule', fields, timeout=0)
 
     # Update status
     logger.info('Event successfully scheduled')

pyca/utils.py

@@ -25,7 +25,7 @@
 logger = logging.getLogger(__name__)
 
 
-def http_request(url, post_data=None):
+def http_request(url, post_data=None, timeout=None):
     '''Make an HTTP request to a given URL with optional parameters.
     '''
     logger.debug('Requesting URL: %s', url)
@@ -54,6 +54,12 @@ def http_request(url, post_data=None):
             curl.MAX_SEND_SPEED_LARGE,
             config('ingest', 'upload_rate'))
 
+    curl.setopt(curl.CONNECTTIMEOUT, config('http', 'connection_timeout'))
+    if timeout is not None:
+        curl.setopt(curl.TIMEOUT, timeout)
+    else:
+        curl.setopt(curl.TIMEOUT, config('http', 'timeout'))
+
     if post_data:
         curl.setopt(curl.HTTPPOST, post_data)
     curl.setopt(curl.WRITEFUNCTION, buf.write)

tests/test_agentstate.py

@@ -15,7 +15,7 @@
 class TestPycaAgentState(unittest.TestCase):
 
     def setUp(self):
-        utils.http_request = lambda x, y=False: b'xxx'
+        utils.http_request = lambda x, y=False, timeout=0: b'xxx'
         self.fd, self.dbfile = tempfile.mkstemp()
         config.config()['agent']['database'] = 'sqlite:///' + self.dbfile
         config.config()['service-capture.admin'] = ['']

tests/test_capture.py

@@ -17,7 +17,7 @@
 class TestPycaCapture(unittest.TestCase):
 
     def setUp(self):
-        utils.http_request = lambda x, y=False: b'xxx'
+        utils.http_request = lambda x, y=False, timeout=0: b'xxx'
         self.fd, self.dbfile = tempfile.mkstemp()
         self.cadir = tempfile.mkdtemp()
         preview = os.path.join(self.cadir, 'preview.png')

tests/test_ingest.py

@@ -19,7 +19,7 @@
 class TestPycaIngest(unittest.TestCase):
 
     def setUp(self):
-        ingest.http_request = lambda x, y=False: b'xxx'
+        ingest.http_request = lambda x, y=False, timeout=0: b'xxx'
         self.fd, self.dbfile = tempfile.mkstemp()
         self.cadir = tempfile.mkdtemp()
         config.config('agent')['database'] = 'sqlite:///' + self.dbfile

tests/test_schedule.py

@@ -29,7 +29,7 @@ class TestPycaCapture(unittest.TestCase):
         END:VCALENDAR''' % END).replace('\n        ', '\r\n').encode('utf-8')
 
     def setUp(self):
-        utils.http_request = lambda x, y=False: b'xxx'
+        utils.http_request = lambda x, y=False, timeout=0: b'xxx'
         self.fd, self.dbfile = tempfile.mkstemp()
         config.config()['agent']['database'] = 'sqlite:///' + self.dbfile
         config.config()['services']['org.opencastproject.scheduler'] = ['']

tests/test_ui_opencast_commands.py

@@ -12,7 +12,7 @@
 class TestPycaIngest(unittest.TestCase):
 
     def setUp(self):
-        opencast_commands.http_request = lambda x, y=False: b'xxx'
+        opencast_commands.http_request = lambda x, y=False, timeout=0: b'xxx'
         opencast_commands.service = lambda x, force_update=False: ['']
 
     def test_schedule_defaults(self):

tests/test_utils.py

@@ -44,7 +44,7 @@ def test_get_service(self):
                         "error_state_trigger":0,
                         "warning_state_trigger":0}}}'''.encode('utf-8')
         # Mock http_request method
-        utils.http_request = lambda x, y=False: res
+        utils.http_request = lambda x, y=False, timeout=0: res
         endpoint = u'https://octestallinone.virtuos.uos.de/capture-admin'
         self.assertEqual(utils.get_service(''), [endpoint])
 
@@ -75,23 +75,23 @@ def test_http_request_mocked_curl(self):
         reload(utils.pycurl)
 
     def test_register_ca(self):
-        utils.http_request = lambda x, y=False: b'xxx'
+        utils.http_request = lambda x, y=False, timeout=0: b'xxx'
         utils.register_ca()
         utils.http_request = should_fail
         utils.register_ca()
         config.config()['agent']['backup_mode'] = True
         utils.register_ca()
 
     def test_recording_state(self):
-        utils.http_request = lambda x, y=False: b''
+        utils.http_request = lambda x, y=False, timeout=0: b''
         utils.recording_state('123', 'recording')
         utils.http_request = should_fail
         utils.recording_state('123', 'recording')
         config.config()['agent']['backup_mode'] = True
         utils.recording_state('123', 'recording')
 
     def test_set_service_status_immediate(self):
-        utils.http_request = lambda x, y=False: b''
+        utils.http_request = lambda x, y=False, timeout=0: b''
         utils.set_service_status_immediate(db.Service.SCHEDULE,
                                            db.ServiceStatus.IDLE)
         utils.set_service_status_immediate(db.Service.INGEST,

tests/tools.py

@@ -50,6 +50,8 @@ class CurlMock():
     FAILONERROR = 11
     FOLLOWLOCATION = 12
     MAX_SEND_SPEED_LARGE = 13
+    CONNECTTIMEOUT = 14
+    TIMEOUT = 15
 
     def setopt(self, *args):
         pass