csvs: Allow uploading multiple CSVs

#15

Also change to using settings from Django settings,
which allows possibility that COVE app can override settings in
libcoveweb

cove_ofds/process.py

@@ -97,6 +97,55 @@ def get_context(self):
         return context
 
 
+class ConvertCSVsIntoJSON(ProcessDataTask):
+    """If User uploaded CSVs, convert to our primary format, JSON."""
+
+    def process(self, process_data: dict) -> dict:
+        if self.supplied_data.format != "csvs":
+            return process_data
+
+        # check already done
+        # TODO
+
+        output_dir = os.path.join(self.supplied_data.data_dir(), "unflatten")
+
+        os.makedirs(output_dir, exist_ok=True)
+
+        unflatten_kwargs = {
+            "output_name": os.path.join(output_dir, "unflattened.json"),
+            "root_list_path": "networks",
+            "input_format": "csv",
+        }
+
+        flattentool.unflatten(self.supplied_data.upload_dir(), **unflatten_kwargs)
+
+        process_data["json_data_filename"] = os.path.join(
+            self.supplied_data.data_dir(), "unflatten", "unflattened.json"
+        )
+
+        return process_data
+
+    def get_context(self):
+        context = {}
+        # original format
+        if self.supplied_data.format == "csvs":
+            context["original_format"] = "csvs"
+            # Download data
+            filename = os.path.join(
+                self.supplied_data.data_dir(), "unflatten", "unflattened.json"
+            )
+            if os.path.exists(filename):
+                context["can_download_json"] = True
+                context["download_json_url"] = os.path.join(
+                    self.supplied_data.data_url(), "unflatten", "unflattened.json"
+                )
+                context["download_json_size"] = os.stat(filename).st_size
+            else:
+                context["can_download_json"] = False
+        # Return
+        return context
+
+
 class ConvertGeoJSONIntoJSON(ProcessDataTask):
     """If User uploaded GeoJSON, convert to our primary format, JSON."""
 

cove_ofds/templates/cove_ofds/index.html

@@ -10,6 +10,9 @@
     <div>
         <a href="{% url 'new_spreadsheet' %}">Upload Spreadsheet</a>
     </div>
+    <div>
+        <a href="{% url 'new_csvs' %}">Upload CSVs</a>
+    </div>
     <div>
         <a href="{% url 'new_geojson' %}">Upload GeoJSON</a>
     </div>

cove_ofds/views.py

@@ -9,6 +9,7 @@
 from cove_ofds.forms import NewGeoJSONUploadForm
 from cove_ofds.process import (
     AdditionalFieldsChecksTask,
+    ConvertCSVsIntoJSON,
     ConvertGeoJSONIntoJSON,
     ConvertJSONIntoGeoJSON,
     ConvertJSONIntoSpreadsheets,
@@ -85,6 +86,7 @@ def explore_ofds(request, pk):
         # Make sure uploads are in primary format
         WasJSONUploaded(db_data),
         ConvertSpreadsheetIntoJSON(db_data),
+        ConvertCSVsIntoJSON(db_data),
         ConvertGeoJSONIntoJSON(db_data),
         # Convert into output formats
         ConvertJSONIntoGeoJSON(db_data),

cove_project/settings.py

@@ -158,6 +158,15 @@
 # https://github.com/OpenDataServices/cove/issues/1098
 FILE_UPLOAD_PERMISSIONS = 0o644
 
+ALLOWED_JSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES
+ALLOWED_JSON_EXTENSIONS = settings.ALLOWED_JSON_EXTENSIONS
+
+ALLOWED_SPREADSHEET_CONTENT_TYPES = settings.ALLOWED_SPREADSHEET_CONTENT_TYPES
+ALLOWED_SPREADSHEET_EXTENSIONS = settings.ALLOWED_SPREADSHEET_EXTENSIONS
+
+ALLOWED_CSV_CONTENT_TYPES = settings.ALLOWED_CSV_CONTENT_TYPES
+ALLOWED_CSV_EXTENSIONS = settings.ALLOWED_CSV_EXTENSIONS
+
 ALLOWED_GEOJSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES + [
     "application/geo+json"
 ]

libcoveweb2/forms.py

@@ -1,18 +1,15 @@
 from django import forms
-
-from libcoveweb2.settings import (
-    ALLOWED_JSON_CONTENT_TYPES,
-    ALLOWED_JSON_EXTENSIONS,
-    ALLOWED_SPREADSHEET_CONTENT_TYPES,
-    ALLOWED_SPREADSHEET_EXTENSIONS,
-)
+from django.conf import settings
 
 
 class NewJSONUploadForm(forms.Form):
     file_upload = forms.FileField(
         widget=forms.FileInput(
             attrs={
-                "accept": ",".join(ALLOWED_JSON_CONTENT_TYPES + ALLOWED_JSON_EXTENSIONS)
+                "accept": ",".join(
+                    settings.ALLOWED_JSON_CONTENT_TYPES
+                    + settings.ALLOWED_JSON_EXTENSIONS
+                )
             }
         )
     )
@@ -23,8 +20,114 @@ class NewSpreadsheetUploadForm(forms.Form):
         widget=forms.FileInput(
             attrs={
                 "accept": ",".join(
-                    ALLOWED_SPREADSHEET_CONTENT_TYPES + ALLOWED_SPREADSHEET_EXTENSIONS
+                    settings.ALLOWED_SPREADSHEET_CONTENT_TYPES
+                    + settings.ALLOWED_SPREADSHEET_EXTENSIONS
                 )
             }
         )
     )
+
+
+class NewCSVsUploadForm(forms.Form):
+    # I know it's hacky to copy and paste code like this but as this needs to be replaced by
+    # something that allows any number of uploads with no limits this will do for now
+    file_field_names = ["file_upload" + str(i) for i in range(0, 10)]
+    file_upload0 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        )
+    )
+    file_upload1 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload2 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload3 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload4 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload5 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload6 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload7 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload8 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )
+    file_upload9 = forms.FileField(
+        widget=forms.FileInput(
+            attrs={
+                "accept": ",".join(
+                    settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS
+                )
+            }
+        ),
+        required=False,
+    )

libcoveweb2/settings.py

@@ -182,3 +182,6 @@
     "application/vnd.oasis.opendocument.spreadsheet",
 ]
 ALLOWED_SPREADSHEET_EXTENSIONS = [".ods", ".xlsx"]
+
+ALLOWED_CSV_CONTENT_TYPES = ["text/csv"]
+ALLOWED_CSV_EXTENSIONS = [".csv"]

libcoveweb2/templates/libcoveweb2/new_csvs.html

@@ -0,0 +1,19 @@
+{% extends request.current_app_base_template %}
+{% load i18n %}
+{% load bootstrap3 %}
+
+{% block content %}
+
+    <div class="panel panel-default">
+        <form method="POST" action="{% url 'new_csvs' %}" enctype="multipart/form-data">
+            {% csrf_token %}
+            {% bootstrap_form forms.upload_form %}
+            {% buttons %}
+                <button type="submit" class="btn btn-primary">
+                    {% trans 'Submit' %}
+                </button>
+            {% endbuttons %}
+        </form>
+    </div>
+
+{% endblock %}

libcoveweb2/urls.py

@@ -6,6 +6,7 @@
 
 urlpatterns = [
     re_path(r"^new_json$", libcoveweb2.views.new_json, name="new_json"),
+    re_path(r"^new_csvs$", libcoveweb2.views.new_csvs, name="new_csvs"),
     re_path(
         r"^new_spreadsheet$", libcoveweb2.views.new_spreadsheet, name="new_spreadsheet"
     ),

libcoveweb2/views.py

@@ -1,16 +1,15 @@
+from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.http import HttpResponseRedirect
 from django.shortcuts import render
 from django.utils.translation import gettext_lazy as _
 
-from libcoveweb2.forms import NewJSONUploadForm, NewSpreadsheetUploadForm
-from libcoveweb2.models import SuppliedData, SuppliedDataFile
-from libcoveweb2.settings import (
-    ALLOWED_JSON_CONTENT_TYPES,
-    ALLOWED_JSON_EXTENSIONS,
-    ALLOWED_SPREADSHEET_CONTENT_TYPES,
-    ALLOWED_SPREADSHEET_EXTENSIONS,
+from libcoveweb2.forms import (
+    NewCSVsUploadForm,
+    NewJSONUploadForm,
+    NewSpreadsheetUploadForm,
 )
+from libcoveweb2.models import SuppliedData, SuppliedDataFile
 
 
 def new_json(request):
@@ -23,11 +22,14 @@ def new_json(request):
     form = forms["upload_form"]
     if form.is_valid():
         # Extra Validation
-        if not request.FILES["file_upload"].content_type in ALLOWED_JSON_CONTENT_TYPES:
+        if (
+            not request.FILES["file_upload"].content_type
+            in settings.ALLOWED_JSON_CONTENT_TYPES
+        ):
             form.add_error("file_upload", "This does not appear to be a JSON file")
         if not [
             e
-            for e in ALLOWED_JSON_EXTENSIONS
+            for e in settings.ALLOWED_JSON_EXTENSIONS
             if str(request.FILES["file_upload"].name).lower().endswith(e)
         ]:
             form.add_error("file_upload", "This does not appear to be a JSON file")
@@ -45,6 +47,45 @@ def new_json(request):
     return render(request, "libcoveweb2/new_json.html", {"forms": forms})
 
 
+def new_csvs(request):
+
+    forms = {
+        "upload_form": NewCSVsUploadForm(request.POST, request.FILES)
+        if request.POST
+        else NewCSVsUploadForm()
+    }
+    form = forms["upload_form"]
+    if form.is_valid():
+        # Extra Validation
+        for field in form.file_field_names:
+            if request.FILES.get(field):
+                if (
+                    not request.FILES[field].content_type
+                    in settings.ALLOWED_CSV_CONTENT_TYPES
+                ):
+                    form.add_error(field, "This does not appear to be a CSV file")
+                if not [
+                    e
+                    for e in settings.ALLOWED_CSV_EXTENSIONS
+                    if str(request.FILES[field].name).lower().endswith(e)
+                ]:
+                    form.add_error(field, "This does not appear to be a CSV file")
+
+        # Process
+        if form.is_valid():
+            supplied_data = SuppliedData()
+            supplied_data.format = "csvs"
+            supplied_data.save()
+
+            for field in form.file_field_names:
+                if request.FILES.get(field):
+                    supplied_data.save_file(request.FILES[field])
+
+            return HttpResponseRedirect(supplied_data.get_absolute_url())
+
+    return render(request, "libcoveweb2/new_csvs.html", {"forms": forms})
+
+
 def new_spreadsheet(request):
 
     forms = {
@@ -57,12 +98,12 @@ def new_spreadsheet(request):
         # Extra Validation
         if (
             not request.FILES["file_upload"].content_type
-            in ALLOWED_SPREADSHEET_CONTENT_TYPES
+            in settings.ALLOWED_SPREADSHEET_CONTENT_TYPES
         ):
             form.add_error("file_upload", "This does not appear to be a spreadsheet")
         if not [
             e
-            for e in ALLOWED_SPREADSHEET_EXTENSIONS
+            for e in settings.ALLOWED_SPREADSHEET_EXTENSIONS
             if str(request.FILES["file_upload"].name).lower().endswith(e)
         ]:
             form.add_error("file_upload", "This does not appear to be a spreadsheet")