aborted migration from AWS to Google Cloud Run

works!...except HTTP response isn't streamed. :( now searching for other containerized serverless hosts that can stream responses.

https://cloud.google.com/run/docs/issues#grpc_websocket
https://stackoverflow.com/a/58980562/186123

.dockerignore

@@ -0,0 +1 @@
+.git*

.firebaserc

@@ -0,0 +1,5 @@
+{
+  "projects": {
+    "default": "huffduff-video"
+  }
+}

.gcloudignore

@@ -0,0 +1,42 @@
+# -*- conf -*-
+# Duplicated in other App Engine project repos: Bridgy, Bridgy Fed, granary,
+# oauth-dropins, etc. If you make a change here, change them too!
+#
+# https://cloud.google.com/appengine/docs/standard/python/config/appref#Python_app_yaml_Includes
+
+*.bak
+*.c
+*.cc
+*.cpp
+*.h
+*.o
+*.pyc
+*.pyo
+*.so
+.git*
+
+__pycache__/
+.coverage/
+.firebase/
+.git*/
+coverage/
+debian/
+doc/
+docs/
+example/
+examples/
+l/
+l3/
+local/
+local3/
+pydoc/
+pydocs/
+python3/
+RCS/
+ref/
+sample/
+samples/
+TAGS
+TAGS/
+test/
+tests/

.gitignore

@@ -2,3 +2,4 @@ aws_key_id
 aws_secret_key
 b2_app_key
 b2_key_id
+/.firebase

Dockerfile

@@ -0,0 +1,18 @@
+# Use the official lightweight Python image.
+# https://hub.docker.com/_/python
+FROM python:3.7-slim
+
+# Copy local code to the container image.
+ENV APP_HOME /app
+WORKDIR $APP_HOME
+COPY . ./
+
+# Install production dependencies.
+RUN pip install b2sdk webob youtube-dl gunicorn
+RUN apt-get update -y && apt-get install -y ffmpeg && apt-get clean
+
+# Run the web service on container startup. Here we use the gunicorn
+# webserver, with one worker process and 8 threads.
+# For environments with multiple CPU cores, increase the number of workers
+# to be equal to the cores available.
+CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 app:application

app.py

@@ -1,6 +1,6 @@
 """Extracts audio from Youtube (etc) videos to send to Huffduffer.
 
-Short test video: http://youtu.be/6dyWlM4ej3Q
+Short test video: https://youtu.be/6dyWlM4ej3Q
 """
 
 __author__ = ['Ryan Barrett <[email protected]>']
@@ -13,11 +13,11 @@
 import ssl
 from string import Template
 import sys
-import urllib
-import urlparse
+import urllib.parse
 
-from b2sdk.v1 import InMemoryAccountInfo, B2Api, AbstractProgressListener
-import boto.ec2.cloudwatch
+from b2sdk.account_info.in_memory import InMemoryAccountInfo
+from b2sdk.api import B2Api
+from b2sdk.progress import AbstractProgressListener
 import requests
 import webob
 import webob.exc
@@ -47,8 +47,6 @@ def read(filename):
   with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), filename)) as f:
     return f.read().strip()
 
-AWS_KEY_ID = read('aws_key_id')
-AWS_SECRET_KEY = read('aws_secret_key')
 B2_KEY_ID = read('b2_key_id')
 B2_APP_KEY = read('b2_app_key')
 B2_BUCKET = 'huffduff-video'
@@ -81,34 +79,15 @@ def application(environ, start_response):
     return webob.exc.HTTPBadRequest('Missing required parameter: url')(
       environ, start_response)
 
-  parsed = urlparse.urlparse(url)
+  parsed = urllib.parse.urlparse(url)
   if parsed.netloc in DOMAIN_BLACKLIST:
     return webob.exc.HTTPBadRequest(
       'Sorry, this content is not currently supported due to copyright.')(
         environ, start_response)
 
-  # check that our CPU credit balance isn't too low
-  try:
-    cloudwatch = boto.ec2.cloudwatch.connect_to_region(
-      'us-west-2', aws_access_key_id=AWS_KEY_ID,
-      aws_secret_access_key=AWS_SECRET_KEY)
-    for metric in cloudwatch.list_metrics(metric_name='CPUCreditBalance'):
-      if metric.name == 'CPUCreditBalance':
-        stats = metric.query(datetime.datetime.now() - datetime.timedelta(minutes=10),
-                             datetime.datetime.now(), ['Average'])
-        if stats:
-          credit = stats[-1].get('Average')
-          if credit and credit <= 30:
-            msg = "Sorry, we're too busy right now. Please try again later!"
-            exc = webob.exc.HTTPServiceUnavailable(msg)
-            exc.html_template_obj = Template(HTML_HEADER + msg + HTML_FOOTER)
-            return exc(environ, start_response)
-  except:
-    logging.exception("Couldn't fetch CPU credit balance from CloudWatch!")
-
   write_fn = start_response('200 OK', headers)
   def write(line):
-    write_fn(line.encode('utf-8'))
+    write_fn(line.encode())
 
   def run():
     """Generator that does all the work and yields the response body lines.
@@ -117,8 +96,8 @@ def run():
     an exception. Currently the log only gets the exception message. Wrapping
     the call at the bottom in try/except doesn't work since it's a generator. :/
     """
-    yield HTML_HEADER
-    yield ('<div id="progress">\nFetching %s ...<br />' % url).encode('utf-8')
+    yield HTML_HEADER.encode()
+    yield ('<div id="progress">\nFetching %s ...<br />' % url).encode()
 
     # function to print out status while downloading
     def download_progress_hook(progress):
@@ -186,7 +165,7 @@ def download_progress_hook(progress):
     uploaded_time = datetime.datetime.now()
     existing = requests.head(b2_url)
     if existing.ok:
-      yield 'Already downloaded! <br />\n'
+      yield 'Already downloaded! <br />\n'.encode()
       try:
         uploaded_time = datetime.datetime.utcfromtimestamp(
           int(existing.headers.get('X-Bz-Upload-Timestamp')) / 1000)  # ms
@@ -195,12 +174,12 @@ def download_progress_hook(progress):
         pass
     else:
       # download video and extract mp3
-      yield 'Downloading (this can take a while)...<br />\n'
+      yield 'Downloading (this can take a while)...<br />\n'.encode()
       with handle_errors(write):
         youtube_dl.YoutubeDL(options).download([url])
 
       # upload to B2
-      yield 'Uploading to B2...<br />\n'
+      yield 'Uploading to B2...<br />\n'.encode()
 
       class WriteProgress(AbstractProgressListener):
         def set_total_bytes(self, total):
@@ -238,17 +217,17 @@ def close(self):
     description += footer
 
     # open 'Huffduff it' page
-    yield """\n<br />Opening Huffduffer dialog...
+    yield ("""\n<br />Opening Huffduffer dialog...
 <script type="text/javascript">
 window.location = "https://huffduffer.com/add?popup=true&%s";
 </script>
-""" % urllib.urlencode([(k, v.encode('utf-8')) for k, v in
+""" % urllib.parse.urlencode([(k, v.encode()) for k, v in
       (('bookmark[url]', (b2_url)),
        ('bookmark[title]', info.get('title') or ''),
        ('bookmark[description]', description),
        ('bookmark[tags]', ','.join(info.get('categories') or [])),
-     )])
-    yield HTML_FOOTER
+     )])).encode()
+    yield HTML_FOOTER.encode()
 
     # alternative:
     # http://themindfulbit.com/blog/optimizing-your-podcast-site-for-huffduffer
@@ -261,7 +240,7 @@ def handle_errors(write):
   """Wraps youtube_dl calls in a try/except and handles errors."""
   try:
     yield
-  except Exception, e:
+  except Exception as e:
     write('<p>%s</p>\n' % e)
     if isinstance(e, (youtube_dl.DownloadError, youtube_dl.utils.ExtractorError)):
       write("""\

firebase.json

@@ -0,0 +1,17 @@
+{
+  "hosting": {
+    "public": "static",
+    "rewrites": [{
+      "source": "/get",
+      "run": {
+        "serviceId": "app",
+        "region": "us-central1"
+      }
+    }],
+    "ignore": [
+      "firebase.json",
+      "**/.*",
+      "**/node_modules/**"
+    ]
+  }
+}

static/404.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Page Not Found</title>
+
+    <style media="screen">
+      body { background: #ECEFF1; color: rgba(0,0,0,0.87); font-family: Roboto, Helvetica, Arial, sans-serif; margin: 0; padding: 0; }
+      #message { background: white; max-width: 360px; margin: 100px auto 16px; padding: 32px 24px 16px; border-radius: 3px; }
+      #message h3 { color: #888; font-weight: normal; font-size: 16px; margin: 16px 0 12px; }
+      #message h2 { color: #ffa100; font-weight: bold; font-size: 16px; margin: 0 0 8px; }
+      #message h1 { font-size: 22px; font-weight: 300; color: rgba(0,0,0,0.6); margin: 0 0 16px;}
+      #message p { line-height: 140%; margin: 16px 0 24px; font-size: 14px; }
+      #message a { display: block; text-align: center; background: #039be5; text-transform: uppercase; text-decoration: none; color: white; padding: 16px; border-radius: 4px; }
+      #message, #message a { box-shadow: 0 1px 3px rgba(0,0,0,0.12), 0 1px 2px rgba(0,0,0,0.24); }
+      #load { color: rgba(0,0,0,0.4); text-align: center; font-size: 13px; }
+      @media (max-width: 600px) {
+        body, #message { margin-top: 0; background: white; box-shadow: none; }
+        body { border-top: 16px solid #ffa100; }
+      }
+    </style>
+  </head>
+  <body>
+    <div id="message">
+      <h2>404</h2>
+      <h1>Page Not Found</h1>
+      <p>The specified file was not found on this website. Please check the URL for mistakes and try again.</p>
+    </div>
+  </body>
+</html>