Allow downloading only one resource in the Open Data page (decidim#13500

)

Co-authored-by: Alexandru Emil Lupu <[email protected]>

decidim-accountability/spec/system/download_open_data_files_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "decidim/core/test/shared_examples/download_open_data_shared_context"
+require "decidim/core/test/shared_examples/download_open_data_shared_examples"
+
+describe "Download Open Data files", download: true do
+  let(:organization) { create(:organization) }
+
+  include_context "when downloading open data files"
+
+  it "lets the users download open data files" do
+    download_open_data_file
+
+    expect(File.basename(download_path)).to include("open-data.zip")
+    Zip::File.open(download_path) do |zipfile|
+      expect(zipfile.glob("*open-data-results.csv").length).to eq(1)
+    end
+  end
+
+  describe "results" do
+    let(:file_name) { "open-data-results.csv" }
+
+    context "when there is none" do
+      it "returns an empty file" do
+        download_open_data_file
+        content = extract_content_from_zip(download_path, file_name)
+        expect(content).to eq("\n")
+      end
+    end
+
+    context "when the result's component is unpublished" do
+      let!(:result) { create(:result, component:) }
+      let(:resource_title) { translated_attribute(result.title).gsub('"', '""') }
+      let(:component) { create(:accountability_component, :unpublished, organization:) }
+
+      it_behaves_like "does not include it in the open data ZIP file"
+    end
+
+    context "when the result's component is published" do
+      let!(:result) { create(:result, component:) }
+      let(:resource_title) { translated_attribute(result.title).gsub('"', '""') }
+      let(:component) { create(:accountability_component, organization:) }
+
+      it_behaves_like "includes it in the open data ZIP file"
+    end
+  end
+
+  describe "open data page" do
+    let(:resource_type) { "results" }
+    let!(:result) { create(:result, component:) }
+    let(:component) { create(:accountability_component, organization:) }
+    let(:resource_title) { translated_attribute(result.title).gsub('"', '""') }
+
+    it_behaves_like "includes it in the open data CSV file"
+  end
+end

decidim-admin/config/locales/pt-BR.yml

@@ -202,11 +202,11 @@ pt-BR:
         verify: Verificar
       admin_terms_of_service:
         accept:
-          error: Houve um erro ao aceitar os termos de serviço de administrador. 
+          error: Houve um erro ao aceitar os termos de serviço de administrador.
           success: Ótimo! Você aceitou os termos de uso do administrador.
         actions:
           accept: Concordo com estes termos
-          are_you_sure: Tem certeza que deseja negar os termos de Serviço para administrador? 
+          are_you_sure: Tem certeza que deseja negar os termos de Serviço para administrador?
           refuse: Recusar os termos de administrador
           title: "Concordo com os termos de serviço\n"
         required_review:

decidim-assemblies/spec/system/download_open_data_files_spec.rb

@@ -31,30 +31,38 @@
 
     context "when the assembly is unpublished" do
       let!(:assembly) { create(:assembly, :unpublished, organization:) }
-      let(:participatory_space_title) { translated_attribute(assembly.title).gsub('"', '""') }
+      let(:resource_title) { translated_attribute(assembly.title).gsub('"', '""') }
 
-      it_behaves_like "does not include it in the open data file"
+      it_behaves_like "does not include it in the open data ZIP file"
     end
 
     context "when the assembly is published and not private" do
       let!(:assembly) { create(:assembly, :published, organization:, private_space: false) }
-      let(:participatory_space_title) { translated_attribute(assembly.title).gsub('"', '""') }
+      let(:resource_title) { translated_attribute(assembly.title).gsub('"', '""') }
 
-      it_behaves_like "includes it in the open data file"
+      it_behaves_like "includes it in the open data ZIP file"
     end
 
     context "when the assembly is published, private and transparent" do
       let!(:assembly) { create(:assembly, :published, organization:, private_space: true, is_transparent: true) }
-      let(:participatory_space_title) { translated_attribute(assembly.title).gsub('"', '""') }
+      let(:resource_title) { translated_attribute(assembly.title).gsub('"', '""') }
 
-      it_behaves_like "includes it in the open data file"
+      it_behaves_like "includes it in the open data ZIP file"
     end
 
     context "when the assembly is published, private and not transparent" do
       let!(:assembly) { create(:assembly, :published, organization:, private_space: true, is_transparent: false) }
-      let(:participatory_space_title) { translated_attribute(assembly.title).gsub('"', '""') }
+      let(:resource_title) { translated_attribute(assembly.title).gsub('"', '""') }
 
-      it_behaves_like "does not include it in the open data file"
+      it_behaves_like "does not include it in the open data ZIP file"
     end
   end
+
+  describe "open data page" do
+    let(:resource_type) { "assemblies" }
+    let!(:assembly) { create(:assembly, :published, organization:, private_space: false) }
+    let(:resource_title) { translated_attribute(assembly.title).gsub('"', '""') }
+
+    it_behaves_like "includes it in the open data CSV file"
+  end
 end

decidim-budgets/spec/serializers/project_serializer_spec.rb

@@ -90,6 +90,21 @@ module Decidim::Budgets
     end
 
     context "when subscribed to the serialize event" do
+      before do
+        I18n.backend.store_translations(
+          :en,
+          decidim: {
+            open_data: {
+              help: {
+                projects: {
+                  test_field: "Test field for projects serializer subscription"
+                }
+              }
+            }
+          }
+        )
+      end
+
       ActiveSupport::Notifications.subscribe("decidim.serialize.budgets.project_serializer") do |_event_name, data|
         data[:serialized_data][:test_field] = "Resource class: #{data[:resource].class}"
       end

decidim-budgets/spec/system/download_open_data_files_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "decidim/core/test/shared_examples/download_open_data_shared_context"
+require "decidim/core/test/shared_examples/download_open_data_shared_examples"
+
+describe "Download Open Data files", download: true do
+  let(:organization) { create(:organization) }
+
+  include_context "when downloading open data files"
+
+  it "lets the users download open data files" do
+    download_open_data_file
+
+    expect(File.basename(download_path)).to include("open-data.zip")
+    Zip::File.open(download_path) do |zipfile|
+      expect(zipfile.glob("*open-data-projects.csv").length).to eq(1)
+    end
+  end
+
+  describe "projects" do
+    let(:file_name) { "open-data-projects.csv" }
+
+    context "when there is none" do
+      it "returns an empty file" do
+        download_open_data_file
+        content = extract_content_from_zip(download_path, file_name)
+        expect(content).to eq("\n")
+      end
+    end
+
+    context "when the project's component is unpublished" do
+      let!(:project) { create(:project, component:) }
+      let(:resource_title) { translated_attribute(project.title).gsub('"', '""') }
+      let(:component) { create(:budgets_component, :unpublished, organization:) }
+
+      it_behaves_like "does not include it in the open data ZIP file"
+    end
+
+    context "when the project's component is published" do
+      let!(:project) { create(:project, component:) }
+      let(:resource_title) { translated_attribute(project.title).gsub('"', '""') }
+      let(:component) { create(:budgets_component, organization:) }
+
+      it_behaves_like "includes it in the open data ZIP file"
+    end
+  end
+
+  describe "open data page" do
+    let(:resource_type) { "projects" }
+    let!(:project) { create(:project, component:) }
+    let(:component) { create(:budgets_component, organization:) }
+    let(:resource_title) { translated_attribute(project.title).gsub('"', '""') }
+
+    it_behaves_like "includes it in the open data CSV file"
+  end
+end

decidim-conferences/spec/system/download_open_data_files_spec.rb

@@ -31,16 +31,24 @@
 
     context "when the conference is unpublished" do
       let!(:conference) { create(:conference, :unpublished, organization:) }
-      let(:participatory_space_title) { translated_attribute(conference.title).gsub('"', '""') }
+      let(:resource_title) { translated_attribute(conference.title).gsub('"', '""') }
 
-      it_behaves_like "does not include it in the open data file"
+      it_behaves_like "does not include it in the open data ZIP file"
     end
 
     context "when the conference is published" do
       let!(:conference) { create(:conference, :published, organization:) }
-      let(:participatory_space_title) { translated_attribute(conference.title).gsub('"', '""') }
+      let(:resource_title) { translated_attribute(conference.title).gsub('"', '""') }
 
-      it_behaves_like "includes it in the open data file"
+      it_behaves_like "includes it in the open data ZIP file"
     end
   end
+
+  describe "open data page" do
+    let(:resource_type) { "conferences" }
+    let!(:conference) { create(:conference, :published, organization:) }
+    let(:resource_title) { translated_attribute(conference.title).gsub('"', '""') }
+
+    it_behaves_like "includes it in the open data CSV file"
+  end
 end

decidim-core/app/controllers/decidim/open_data_controller.rb

@@ -2,26 +2,45 @@
 
 module Decidim
   class OpenDataController < Decidim::ApplicationController
+    helper_method :open_data_component_manifests, :open_data_participatory_space_manifests
+
     def index; end
 
     def download
-      if uploader.attached?
-        redirect_to uploader.path
+      resource = params[:resource] || nil
+
+      if open_data_file_for_resource(resource)
+        file = open_data_file_for_resource(resource)
+        send_data file.download, filename: file.blob.filename.to_s, type: file.blob.content_type
       else
-        schedule_open_data_generation
+        schedule_open_data_generation(resource)
         flash[:alert] = t("decidim.open_data.not_available_yet")
-        redirect_back fallback_location: root_path
+        redirect_back fallback_location: open_data_path
       end
     end
 
     private
 
-    def uploader
-      @uploader ||= Decidim::ApplicationUploader.new(current_organization, :open_data_file)
+    def open_data_component_manifests
+      @open_data_component_manifests ||= Decidim.component_manifests
+                                                .flat_map(&:export_manifests)
+                                                .select(&:include_in_open_data?)
+    end
+
+    def open_data_participatory_space_manifests
+      @open_data_participatory_space_manifests ||= Decidim.participatory_space_manifests
+                                                          .flat_map(&:export_manifests)
+                                                          .select(&:include_in_open_data?)
+    end
+
+    def open_data_file_for_resource(resource)
+      current_organization.open_data_files.all.find do |file|
+        file.blob.filename == current_organization.open_data_file_path(resource)
+      end
     end
 
-    def schedule_open_data_generation
-      OpenDataJob.perform_later(current_organization)
+    def schedule_open_data_generation(resource = nil)
+      OpenDataJob.perform_later(current_organization, resource)
     end
   end
 end

decidim-core/app/jobs/decidim/open_data_job.rb

@@ -4,13 +4,13 @@ module Decidim
   class OpenDataJob < ApplicationJob
     queue_as :exports
 
-    def perform(organization)
-      path = Rails.root.join("tmp/#{organization.open_data_file_path}")
+    def perform(organization, resource = nil)
+      path = Rails.root.join("tmp/#{organization.open_data_file_path(resource)}")
 
-      exporter = OpenDataExporter.new(organization, path)
+      exporter = OpenDataExporter.new(organization, path, resource)
       raise "Could not generate Open Data export" unless exporter.export.positive?
 
-      organization.open_data_file.attach(io: File.open(path, "rb"), filename: organization.open_data_file_path)
+      organization.open_data_files.attach(io: File.open(path, "rb"), filename: organization.open_data_file_path(resource))
       # Deletes the temporary file file
       File.delete(path)
     end

decidim-core/app/models/decidim/organization.rb

@@ -60,7 +60,7 @@ class Organization < ApplicationRecord
     has_one_attached :highlighted_content_banner_image
     validates_upload :highlighted_content_banner_image, uploader: Decidim::ImageUploader
 
-    has_one_attached :open_data_file
+    has_many_attached :open_data_files
 
     validate :unique_name
 
@@ -150,8 +150,10 @@ def sign_in_enabled?
       !users_registration_mode_disabled?
     end
 
-    def open_data_file_path
-      "#{host}-open-data.zip"
+    def open_data_file_path(resource = nil)
+      return "#{host}-open-data.zip" if resource.nil?
+
+      "#{host}-open-data-#{resource}.csv"
     end
 
     def enabled_omniauth_providers

decidim-core/app/services/decidim/open_data_exporter.rb

@@ -9,31 +9,35 @@ module Decidim
   class OpenDataExporter
     FILE_NAME_PATTERN = "%{host}-open-data-%{entity}.csv"
 
-    include Decidim::TranslatableAttributes
+    attr_reader :organization, :path, :resource, :help_definition
 
-    attr_reader :organization, :path
+    include Decidim::TranslatableAttributes
 
     # Public: Initializes the class.
     #
     # organization - The Organization to export the data from.
     # path         - The String path where to write the zip file.
-    def initialize(organization, path)
+    # resource     - The String of the component or participatory space to export. If nil, it will export all.
+    def initialize(organization, path, resource = nil)
       @organization = organization
       @path = File.expand_path path
+      @resource = resource
       @help_definition = {}
     end
 
     def export
       dirname = File.dirname(path)
       FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
-      File.binwrite(path, data)
+      if resource.nil?
+        File.binwrite(path, data_for_all_resources)
+      else
+        File.write(path, data_for_resource(resource))
+      end
     end
 
     private
 
-    attr_reader :help_definition
-
-    def data
+    def data_for_all_resources
       buffer = Zip::OutputStream.write_buffer do |out|
         open_data_component_manifests.each do |manifest|
           add_file_to_output(out, format(FILE_NAME_PATTERN, { host: organization.host, entity: manifest.name }), data_for_component(manifest).read)
@@ -47,6 +51,18 @@ def data
       buffer.string
     end
 
+    def data_for_resource(resource)
+      export_manifest = (open_data_component_manifests + open_data_participatory_space_manifests)
+                        .select { |manifest| manifest.name == resource.to_sym }.first
+
+      case export_manifest.manifest
+      when Decidim::ComponentManifest
+        data_for_component(export_manifest).read
+      when Decidim::ParticipatorySpaceManifest
+        data_for_participatory_space(export_manifest).read
+      end
+    end
+
     def data_for_component(export_manifest, col_sep = Decidim.default_csv_col_sep)
       headers = []
       collection = []