feat: add Google Cloud Storage support (GCS)

openassessment/fileupload/backends/__init__.py

@@ -3,7 +3,7 @@
 
 from django.conf import settings
 
-from . import django_storage, filesystem, s3, swift
+from . import django_storage, filesystem, gcs, s3, swift
 
 
 def get_backend():
@@ -18,6 +18,8 @@ def get_backend():
  return filesystem.Backend()
  elif backend_setting == "swift":
  return swift.Backend()
+ elif backend_setting == "gcs":
+ return gcs.Backend()
  elif backend_setting == "django":
  return django_storage.Backend()
  else:

openassessment/fileupload/backends/gcs.py

@@ -0,0 +1,73 @@
+"""GCS Bucket File Upload Backend."""
+import functools
+import logging
+
+from ..exceptions import FileUploadInternalError
+from .base import BaseBackend
+
+log = logging.getLogger("openassessment.fileupload.api") # pylint: disable=invalid-name
+
+
+def catch_broad_exception(method):
+ """Decorator to catch broad exceptions, log them, and raise a FileUploadInternalError."""
+ @functools.wraps(method)
+ def wrapper(*args, **kwargs):
+ try:
+ return method(*args, **kwargs)
+ except Exception as ex: # pylint: disable=broad-except
+ log.exception(
+ f"Internal exception occurred while executing ora2 file-upload backend gcs.{method.__name__}: {str(ex)}"
+ )
+ raise FileUploadInternalError(ex) from ex
+ return wrapper
+
+
+class Backend(BaseBackend):
+ """ GCS Bucket File Upload Backend. """
+
+ @catch_broad_exception
+ def get_upload_url(self, key, content_type):
+ """Get a signed URL for uploading a file to GCS"""
+ bucket_name, key_name = self._retrieve_parameters(key)
+ blob = get_blob_object(bucket_name, key_name)
+
+ return blob.generate_signed_url(
+ version="v4",
+ expiration=self.UPLOAD_URL_TIMEOUT,
+ method="PUT",
+ content_type=content_type,
+ )
+
+ @catch_broad_exception
+ def get_download_url(self, key):
+ """Get a signed URL for downloading a file from GCS"""
+ bucket_name, key_name = self._retrieve_parameters(key)
+ blob = get_blob_object(bucket_name, key_name)
+ if not blob.exists():
+ return ""
+
+ return blob.generate_signed_url(
+ version="v4",
+ expiration=self.DOWNLOAD_URL_TIMEOUT,
+ method="GET",
+ )
+
+ @catch_broad_exception
+ def remove_file(self, key):
+ """Remove a file from GCS"""
+ bucket_name, key_name = self._retrieve_parameters(key)
+ blob = get_blob_object(bucket_name, key_name)
+ if blob.exists():
+ blob.delete()
+ return True
+
+ return False
+
+
+def get_blob_object(bucket_name, key_name):
+ """Get a blob object from GCS"""
+ # By default; avoid the need of google-cloud-storage library. It will be only needed if gcs backend is used.
+ from google.cloud import storage # pylint: disable=import-outside-toplevel
+
+ client = storage.Client()
+ return client.bucket(bucket_name).blob(key_name)

openassessment/fileupload/tests/test_api.py

@@ -19,6 +19,7 @@
 from openassessment.fileupload import api, exceptions, urls
 from openassessment.fileupload import views_filesystem as views
 from openassessment.fileupload.backends.base import Settings as FileUploadSettings
+from openassessment.fileupload.backends.gcs import get_blob_object
 from openassessment.fileupload.backends.filesystem import (
  get_cache as get_filesystem_cache,
 )
@@ -499,3 +500,140 @@ def test_remove(self, key):
  # File no longer exists
  download_url = self.backend.get_download_url(self.key)
  self.assertIsNone(download_url)
+
+
+@override_settings(
+ ORA2_FILEUPLOAD_BACKEND="gcs",
+ DEFAULT_FILE_STORAGE="storages.backends.gcloud.GoogleCloudStorage",
+ FILE_UPLOAD_STORAGE_PREFIX="submissions_on_gcp",
+ FILE_UPLOAD_STORAGE_BUCKET_NAME="testbucket",
+ LMS_ROOT_URL="http://foobar.example.com",
+)
+@ddt.ddt
+class TestFileUploadServiceWithGoogleStorageBackend(TestCase):
+ """
+ Test open assessment file upload using GCS storage backend.
+ """
+ class MockBlob:
+ def __init__(self):
+ self.exist_testing_flag = False
+ self.delete_called_during_test = False
+
+ def generate_signed_url(self, **kwargs):
+ return "http://foobar.example.com/submissions_on_gcs/signed_url_called.txt"
+
+ def delete(self):
+ self.delete_called_during_test = True
+
+ def exists(self):
+ return self.exist_testing_flag
+
+ def setUp(self):
+ super().setUp()
+ self.backend = api.backends.get_backend()
+ # self.username = "test_user"
+ # self.password = "password"
+ # self.user = get_user_model().objects.create_user(
+ # username=self.username, password=self.password
+ # )
+ #
+ # self.content = tempfile.TemporaryFile()
+ # self.content.write(b"foobar content")
+ # self.content.seek(0)
+
+ # self.key = "myfile.txt"
+ # self.content_type = "text/plain"
+ # self.base_url = "http://foobar.example.com"
+ self.patchers = {
+ "get_blob_object": patch(
+ "openassessment.fileupload.backends.gcs.get_blob_object",
+ return_value=self.MockBlob(),
+ ),
+ "log": patch("openassessment.fileupload.backends.gcs.log.exception"),
+ }
+
+ self.mock_get_blob_object = self.patchers["get_blob_object"].start()
+ self.mock_log = self.patchers["log"].start()
+
+ def tearDown(self):
+ """
+ Stop the patcher.
+ """
+ for patcher in self.patchers.values():
+ patcher.stop()
+
+ def test_get_backend(self):
+ """
+ Ensure the django storage backend is returned when ORA2_FILEUPLOAD_BACKEND="gcs".
+ """
+ self.assertTrue(isinstance(self.backend, api.backends.gcs.Backend))
+
+ @ddt.data(
+ ("get_upload_url", {'key': 'whatever', 'content_type': 'whatever'}),
+ ("get_download_url", {'key': 'whatever'}),
+ ("remove_file", {'key': 'whatever'}),
+ )
+ @ddt.unpack
+ def test_errors_raise_file_upload_internal_error(self, method_name, kwargs):
+ """
+ Ensure that exceptions are caught and raised as FileUploadInternalError.
+ """
+ self.mock_get_blob_object.side_effect = Exception("Some error!")
+ with raises(exceptions.FileUploadInternalError):
+ getattr(self.backend, method_name)(**kwargs)
+ self.mock_log.assert_called_once_with(
+ f"Internal exception occurred while executing ora2 file-upload backend gcs.{method_name}: Some error!"
+ )
+
+ def test_get_upload_url(self):
+ """
+ Verify the upload URL.
+ """
+ url = self.backend.get_upload_url("foo", "_text")
+ self.mock_get_blob_object.assert_called_once_with("testbucket", "submissions_on_gcp/foo")
+ self.assertEqual(url, "http://foobar.example.com/submissions_on_gcs/signed_url_called.txt")
+
+ def test_get_download_url(self):
+ """
+ Verify the download URL.
+ """
+ url = self.backend.get_download_url("foo")
+ self.mock_get_blob_object.assert_called_once_with("testbucket", "submissions_on_gcp/foo")
+ self.assertEqual(url, "")
+
+ self.mock_get_blob_object.return_value.exist_testing_flag = True
+ url = self.backend.get_download_url("foo")
+ self.assertEqual(url, "http://foobar.example.com/submissions_on_gcs/signed_url_called.txt")
+
+ def test_remove_file(self):
+ """
+ Verify the remove file method.
+ """
+ self.assertFalse(self.mock_get_blob_object.return_value.delete_called_during_test)
+ result = self.backend.remove_file("foo")
+ self.mock_get_blob_object.assert_called_once_with("testbucket", "submissions_on_gcp/foo")
+ self.assertFalse(result)
+
+ self.mock_get_blob_object.return_value.exist_testing_flag = True
+ self.assertFalse(self.mock_get_blob_object.return_value.delete_called_during_test)
+ result = self.backend.remove_file("foo")
+ self.assertTrue(result)
+ self.assertTrue(self.mock_get_blob_object.return_value.delete_called_during_test)
+
+ def test_get_blob_object(self):
+ """
+ Verify the get_blob_object method.
+ """
+ with patch("google.cloud.storage") as mock_storage:
+ blob = Mock(id="the_test_blob")
+
+ bucket = Mock()
+ bucket.blob.return_value = blob
+
+ client = Mock()
+ client.bucket.return_value = bucket
+
+ mock_storage.Client.return_value = client
+
+ result = get_blob_object("testbucket", "submissions_on_gcp/foo")
+ self.assertEqual(result.id, "the_test_blob")