Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Minio #1767

Merged
merged 16 commits into from
Feb 6, 2024
Merged

Minio #1767

Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 13 additions & 15 deletions caseworker/cases/views/main.py
Original file line number Diff line number Diff line change
@@ -13,14 +13,16 @@
from django.urls import reverse, reverse_lazy
from django.utils import timezone
from django.utils.functional import cached_property
from django.views.generic import FormView, TemplateView
from django.views.generic import FormView, TemplateView, View

from requests.exceptions import HTTPError

from core.auth.views import LoginRequiredMixin
from core.builtins.custom_tags import filter_advice_by_level
from core.decorators import expect_status
from core.file_handler import s3_client
from core.exceptions import APIError
from core.helpers import get_document_data
from core.file_handler import download_document_from_s3

from lite_content.lite_internal_frontend import cases
from lite_content.lite_internal_frontend.cases import (
@@ -523,37 +525,33 @@ def post(self, request, **kwargs):
file = files[0]
data.append(
{
"name": file.original_name,
"s3_key": file.name,
"size": int(file.size // 1024) if file.size else 0, # in kilobytes
"description": request.POST["description"],
**get_document_data(file),
}
)

# Send LITE API the file information
case_documents, _ = post_case_documents(request, case_id, data)

if "errors" in case_documents:
if settings.DEBUG:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

feels like a bad pattern to have settings.DEBUG in normal control flow.

Is there not any other way ?
not sure what though atm

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is your concern?

The reason behind this was that when I was testing this locally I would sometimes get a generic error that was hard for me to then understand what was going on so when running in debug mode it was better to just return the error that came from the API so it was clear what just happened.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just don't like seeing a different flow based on settings.DEBUG on production code.

no biggie so approving

raise APIError(case_documents["errors"])
return error_page(None, "We had an issue uploading your files. Try again later.")

return redirect(
reverse("cases:case", kwargs={"queue_pk": kwargs["queue_pk"], "pk": case_id, "tab": "documents"})
)


class Document(TemplateView):
class Document(View):
def get(self, request, **kwargs):
document, _ = get_document(request, pk=kwargs["file_pk"])
client = s3_client()
signed_url = client.generate_presigned_url(
"get_object",
Params={
"Bucket": settings.AWS_STORAGE_BUCKET_NAME,
"Key": document["document"]["s3_key"],
},
ExpiresIn=15,
document = document["document"]

return download_document_from_s3(
document["s3_key"],
document["name"],
)
return redirect(signed_url)


class CaseOfficer(SingleFormView):
1 change: 1 addition & 0 deletions conf/base.py
Original file line number Diff line number Diff line change
@@ -189,6 +189,7 @@
# AWS
VCAP_SERVICES = env.json("VCAP_SERVICES", {})

AWS_S3_ENDPOINT_URL = env.str("AWS_S3_ENDPOINT_URL", None)
AWS_ACCESS_KEY_ID = env.str("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = env.str("AWS_SECRET_ACCESS_KEY")
AWS_REGION = env.str("AWS_REGION")
4 changes: 4 additions & 0 deletions core/exceptions.py
Original file line number Diff line number Diff line change
@@ -9,3 +9,7 @@ def __init__(self, message, status_code, response, log_message, user_message):
self.response = response
self.log_message = log_message
self.user_message = user_message


class APIError(Exception):
pass
16 changes: 16 additions & 0 deletions core/file_handler.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@

from django.conf import settings
from django.core.files.uploadhandler import UploadFileException
from django.http import StreamingHttpResponse

from django_chunk_upload_handlers.s3 import S3FileUploadHandler

@@ -83,3 +84,18 @@ class UploadFailed(UploadFileException):

class UnacceptableMimeTypeError(UploadFailed):
pass


def generate_file(result):
for chunk in iter(lambda: result["Body"].read(settings.STREAMING_CHUNK_SIZE), b""):
yield chunk


def download_document_from_s3(s3_key, original_file_name):
s3_response = s3_client().get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=s3_key)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like a good place for us to flip over to proxying the new download API endpoint

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah. This is me initially picking up from a PR that I've had sat dormant for a long time but I definitely see this ending up being the function we switch out to read from the API, I think the only thing that this misses at the moment is the document id, which we'd want to also send to check for file permissions.

_kwargs = {}
if s3_response.get("ContentType"):
_kwargs["content_type"] = s3_response["ContentType"]
response = StreamingHttpResponse(generate_file(s3_response), **_kwargs)
response["Content-Disposition"] = f'attachment; filename="{original_file_name}"'
return response
8 changes: 4 additions & 4 deletions example.caseworker.env
Original file line number Diff line number Diff line change
@@ -26,10 +26,10 @@ EXPORTER_TEST_SSO_EMAIL=<<FROM_VAULT>>
EXPORTER_TEST_SSO_PASSWORD=<<FROM_VAULT>>

# AWS
AWS_ACCESS_KEY_ID=<<FROM_VAULT>>
AWS_SECRET_ACCESS_KEY=<<FROM_VAULT>>
AWS_STORAGE_BUCKET_NAME=<<FROM_VAULT>>
AWS_REGION=<<FROM_VAULT>>
AWS_S3_ENDPOINT_URL=http://s3:9000
AWS_ACCESS_KEY_ID=minio_username
AWS_SECRET_ACCESS_KEY=minio_password
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the expectation here that minio will be present on the API docker-compose only? Just wrapping my head around how this works

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is correct. The API needs to have access to the same bucket so we need to have one single instance of minio running and it makes sense for that to be in the docker-compose of the API.

We do the same for redis as well.

It's what

networks:
  lite:
    external: true

is for to bridge between the two docker-compose clusters (or whatever they're called).

AWS_STORAGE_BUCKET_NAME=uploads

LITE_INTERNAL_HAWK_KEY=LITE_INTERNAL_HAWK_KEY

8 changes: 4 additions & 4 deletions example.exporter.env
Original file line number Diff line number Diff line change
@@ -18,10 +18,10 @@ AUTHBROKER_CLIENT_ID=<<FROM_VAULT>>
AUTHBROKER_CLIENT_SECRET=<<FROM_VAULT>>
TOKEN_SESSION_KEY=<<FROM_VAULT>>

AWS_ACCESS_KEY_ID=<<FROM_VAULT>>
AWS_SECRET_ACCESS_KEY=<<FROM_VAULT>>
AWS_STORAGE_BUCKET_NAME=<<FROM_VAULT>>
AWS_REGION=<<FROM_VAULT>>
AWS_S3_ENDPOINT_URL=http://s3:9000
AWS_ACCESS_KEY_ID=minio_username
AWS_SECRET_ACCESS_KEY=minio_password
AWS_STORAGE_BUCKET_NAME=uploads

DJANGO_SECRET_KEY=DJANGO_SECRET_KEY

32 changes: 3 additions & 29 deletions exporter/applications/services.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
from http import HTTPStatus

from django.http import StreamingHttpResponse
from django.conf import settings

from core import client
from core.file_handler import s3_client
from core.helpers import get_document_data

from exporter.applications.helpers.date_fields import (
format_date_fields,
format_date,
@@ -468,37 +466,13 @@ def add_document_data(request):
if len(files) != 1:
return None, "Multiple files attached"
file = files[0]
try:
original_name = file.original_name
except Exception: # noqa
original_name = file.name

data = {
"name": original_name,
"s3_key": file.name,
"size": int(file.size // 1024) if file.size else 0, # in kilobytes
}
data = get_document_data(file)
if "description" in request.POST:
data["description"] = request.POST.get("description")

return data, None


def generate_file(result):
for chunk in iter(lambda: result["Body"].read(settings.STREAMING_CHUNK_SIZE), b""):
yield chunk


def download_document_from_s3(s3_key, original_file_name):
s3_response = s3_client().get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=s3_key)
_kwargs = {}
if s3_response.get("ContentType"):
_kwargs["content_type"] = s3_response["ContentType"]
response = StreamingHttpResponse(generate_file(s3_response), **_kwargs)
response["Content-Disposition"] = f'attachment; filename="{original_file_name}"'
return response


def get_goods_type(request, app_pk, good_pk):
data = client.get(request, f"/applications/{app_pk}/goodstype/{good_pk}/")
return data.json(), data.status_code
22 changes: 7 additions & 15 deletions exporter/applications/views/documents.py
Original file line number Diff line number Diff line change
@@ -2,19 +2,18 @@
from http import HTTPStatus
from inspect import signature

from django.conf import settings
from django.shortcuts import redirect
from django.urls import reverse, NoReverseMatch
from django.views.generic import TemplateView, View

from core.file_handler import download_document_from_s3

from caseworker.cases.services import get_document
from core.decorators import expect_status
from core.file_handler import s3_client
from exporter.applications.forms.documents import attach_document_form, delete_document_confirmation_form
from exporter.applications.helpers.check_your_answers import is_application_export_type_permanent
from exporter.applications.services import (
add_document_data,
download_document_from_s3,
get_application,
post_party_document,
get_party_document,
@@ -168,7 +167,7 @@ def post(self, request, **kwargs):
return get_homepage(request, draft_id)


class DownloadDocument(LoginRequiredMixin, TemplateView):
class DownloadDocument(LoginRequiredMixin, View):
def get(self, request, **kwargs):
draft_id = str(kwargs["pk"])
action = document_switch(request.path)["download"]
@@ -185,19 +184,12 @@ def get(self, request, **kwargs):
return error_page(request, strings.applications.AttachDocumentPage.DOWNLOAD_GENERIC_ERROR)


class DownloadGeneratedDocument(LoginRequiredMixin, TemplateView):
class DownloadGeneratedDocument(LoginRequiredMixin, View):
def get(self, request, case_pk, document_pk):
document, _ = get_document(request, pk=document_pk)
client = s3_client()
signed_url = client.generate_presigned_url(
"get_object",
Params={
"Bucket": settings.AWS_STORAGE_BUCKET_NAME,
"Key": document["document"]["s3_key"],
},
ExpiresIn=15,
)
return redirect(signed_url)
document = document["document"]

return download_document_from_s3(document["s3_key"], document["name"])


class DownloadAppealDocument(LoginRequiredMixin, View):
2 changes: 1 addition & 1 deletion exporter/applications/views/goods/common/actions.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
from http import HTTPStatus

from core.decorators import expect_status
from core.helpers import get_document_data

from exporter.applications.views.goods.common.constants import PRODUCT_DOCUMENT_UPLOAD
from exporter.core.helpers import get_document_data
from exporter.goods.services import post_good_documents


2 changes: 1 addition & 1 deletion exporter/applications/views/goods/common/edit.py
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@
from core.auth.views import LoginRequiredMixin
from core.decorators import expect_status

from exporter.core.helpers import get_document_data
from core.helpers import get_document_data
from core.wizard.conditionals import C
from core.wizard.views import BaseSessionWizardView
from exporter.goods.forms.common import (
2 changes: 1 addition & 1 deletion exporter/applications/views/goods/component/views/add.py
Original file line number Diff line number Diff line change
@@ -7,10 +7,10 @@

from core.auth.views import LoginRequiredMixin
from core.decorators import expect_status
from core.helpers import get_document_data

from exporter.applications.views.goods.common.actions import ProductDocumentAction
from core.wizard.views import BaseSessionWizardView
from exporter.core.helpers import get_document_data
from exporter.goods.forms.common import (
ProductControlListEntryForm,
ProductDocumentAvailabilityForm,
2 changes: 1 addition & 1 deletion exporter/applications/views/goods/firearm/views/actions.py
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
OrganisationDocumentType,
)
from core.decorators import expect_status
from core.helpers import get_document_data

from exporter.applications.services import (
delete_additional_document,
@@ -18,7 +19,6 @@
)
from exporter.core.forms import CurrentFile
from exporter.core.helpers import (
get_document_data,
get_organisation_firearm_act_document,
get_rfd_certificate,
has_organisation_firearm_act_document,
2 changes: 1 addition & 1 deletion exporter/applications/views/goods/firearm/views/add.py
Original file line number Diff line number Diff line change
@@ -13,14 +13,14 @@
OrganisationDocumentType,
)
from core.decorators import expect_status
from core.helpers import get_document_data

from exporter.applications.services import (
post_additional_document,
post_firearm_good_on_application,
)
from exporter.applications.views.goods.common.actions import ProductDocumentAction
from exporter.core.helpers import (
get_document_data,
get_rfd_certificate,
has_valid_rfd_certificate as has_valid_organisation_rfd_certificate,
)
20 changes: 9 additions & 11 deletions exporter/applications/views/goods/goods.py
Original file line number Diff line number Diff line change
@@ -17,19 +17,23 @@
OrganisationDocumentType,
ProductCategories,
)
from core.helpers import convert_dict_to_query_params
from core.file_handler import download_document_from_s3
from core.helpers import (
convert_dict_to_query_params,
get_document_data,
)
from core.summaries.summaries import (
get_summary_type_for_good_on_application,
NoSummaryForType,
SummaryTypes,
)

from exporter.applications.helpers.check_your_answers import get_total_goods_value
from exporter.applications.helpers.date_fields import format_date
from exporter.applications.services import (
add_document_data,
delete_application_document_data,
delete_application_preexisting_good,
download_document_from_s3,
get_application,
get_application_document,
get_application_documents,
@@ -219,14 +223,12 @@ def cache_rfd_certificate_details(self):
if not file:
return
self.request.session[self.SESSION_KEY_RFD_CERTIFICATE] = {
"name": getattr(file, "original_name", file.name),
"s3_key": file.name,
"size": int(file.size // 1024) if file.size else 0, # in kilobytes
"document_on_organisation": {
"expiry_date": format_date(self.request.POST, "expiry_date_"),
"reference_code": self.request.POST["reference_code"],
"document_type": OrganisationDocumentType.RFD_CERTIFICATE,
},
**get_document_data(file),
}

def post_success_step(self):
@@ -445,14 +447,12 @@ def done(self, form_list, **kwargs):

if cert_file:
rfd_cert = {
"name": getattr(cert_file, "original_name", cert_file.name),
"s3_key": cert_file.name,
"size": int(cert_file.size // 1024) if cert_file.size else 0, # in kilobytes
"document_on_organisation": {
"expiry_date": format_date(all_data, "expiry_date_"),
"reference_code": all_data["reference_code"],
"document_type": OrganisationDocumentType.RFD_CERTIFICATE,
},
**get_document_data(cert_file),
}

_, status_code = post_additional_document(
@@ -947,14 +947,12 @@ def done(self, form_list, **kwargs):

if cert_file:
rfd_cert = {
"name": getattr(cert_file, "original_name", cert_file.name),
"s3_key": cert_file.name,
"size": int(cert_file.size // 1024) if cert_file.size else 0, # in kilobytes
"document_on_organisation": {
"expiry_date": format_date(all_data, "expiry_date_"),
"reference_code": all_data["reference_code"],
"document_type": OrganisationDocumentType.RFD_CERTIFICATE,
},
**get_document_data(cert_file),
}

_, status_code = post_additional_document(request=self.request, pk=str(self.kwargs["pk"]), json=rfd_cert)
Loading
Loading