From 100784fc6a5ffd8e34ab58f8d769292f766936a4 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Thu, 1 Aug 2024 11:17:58 -0300 Subject: [PATCH 01/46] create SurveyFilesManager actor to handle generation of files --- lib/ask.ex | 3 +- lib/ask/runtime/survey_files_manager.ex | 169 ++++++++++++++++++ .../controllers/respondent_controller.ex | 95 +--------- 3 files changed, 177 insertions(+), 90 deletions(-) create mode 100644 lib/ask/runtime/survey_files_manager.ex diff --git a/lib/ask.ex b/lib/ask.ex index b9a6141de..36e7de1bb 100644 --- a/lib/ask.ex +++ b/lib/ask.ex @@ -42,7 +42,8 @@ defmodule Ask do worker(Ask.JsonSchema, []), worker(Ask.Runtime.ChannelStatusServer, []), worker(Ask.Config, []), - worker(Ask.Runtime.QuestionnaireSimulatorStore, []) + worker(Ask.Runtime.QuestionnaireSimulatorStore, []), + worker(Ask.Runtime.SurveyFilesManager, []) | children ] ++ [ # SurveyCancellerSupervisor depends on Ask.Repo, so must be started (and declared!) after it diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex new file mode 100644 index 000000000..24d3997ed --- /dev/null +++ b/lib/ask/runtime/survey_files_manager.ex @@ -0,0 +1,169 @@ + +## This GenServer uses :hibernate since its mailbox might be most of the time empty +## +## Hibernating a GenServer causes garbage collection and leaves a continuous +## heap that minimises the memory used by the process. +## +## When a process is hibernated it will continue the loop once a message is in its +## message queue. If when returning a handle_* call there is already a message +## in the message queue, the process will continue the loop immediately. +defmodule Ask.Runtime.SurveyFilesManager do + use GenServer + alias Ask.{Logger, Repo, Survey, SurveyLogEntry} + import Ecto.Query + + @db_chunk_limit 10_000 + + @server_ref {:global, __MODULE__} + def server_ref, do: @server_ref + + def start_link do + GenServer.start_link(__MODULE__, %{}, name: @server_ref) + end + + @impl true + def init(state) do + Logger.info("SurveyFilesManager started") + {:ok, state} + end + + @impl true + def handle_cast({:interactions, survey_id}, state) do + survey = Repo.get!(Survey, survey_id) + + channels = survey_log_entry_channel_names(survey) + + Logger.info("Starting to build interaction file (survey_id: #{survey_id})") + + log_entries = + Stream.resource( + fn -> {"", 0} end, + fn {last_hash, last_id} -> + results = + from(e in SurveyLogEntry, + where: + e.survey_id == ^survey.id and + ((e.respondent_hashed_number == ^last_hash and e.id > ^last_id) or + e.respondent_hashed_number > ^last_hash), + order_by: [e.respondent_hashed_number, e.id], + limit: @db_chunk_limit + ) + |> Repo.all() + + case List.last(results) do + nil -> {:halt, {last_hash, last_id}} + last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id}} + end + end, + fn _ -> [] end + ) + + tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) + tz_offset = Survey.timezone_offset(survey) + + csv_rows = + log_entries + |> Stream.map(fn e -> + [ + Integer.to_string(e.id), + e.respondent_hashed_number, + interactions_mode_label(e.mode), + Map.get(channels, e.channel_id, ""), + disposition_label(e.disposition), + action_type_label(e.action_type), + e.action_data, + csv_datetime(e.timestamp, tz_offset_in_seconds, tz_offset) + ] + end) + + header = [ + "ID", + "Respondent ID", + "Mode", + "Channel", + "Disposition", + "Action Type", + "Action Data", + "Timestamp" + ] + + rows = Stream.concat([[header], csv_rows]) + + filename = csv_filename(survey, "respondents_interactions") + file = File.open!(filename, [:write, :utf8]) + initial_datetime = Timex.now() + rows + |> CSV.encode() + |> Enum.each(&IO.write(file, &1)) + + seconds_to_process_file = Timex.diff(Timex.now(), initial_datetime, :seconds) + Logger.info("Generation of interaction files survey (id #{survey_id}) took #{seconds_to_process_file} seconds") + + {:noreply, state, :hibernate} + end + + @impl true + def handle_cast(message, state) do + Logger.warn("Ignoring message #{message}") + {:noreply, state, :hibernate} + end + + defp survey_log_entry_channel_names(survey) do + respondent_groups = Repo.preload(survey, respondent_groups: [:channels]).respondent_groups + respondent_groups + |> Enum.flat_map(fn resp_group -> resp_group.channels end) + |> Enum.map( fn channel -> {channel.id, channel.name} end) + |> MapSet.new # convert to set to remove duplicates + |> Enum.into(%{}) + end + + defp interactions_mode_label(mode) do + case mode do + "mobileweb" -> "Mobile Web" + _ -> String.upcase(mode) + end + end + + defp action_type_label(action) do + case action do + nil -> nil + "contact" -> "Contact attempt" + _ -> String.capitalize(action) + end + end + + defp disposition_label(disposition) do + case disposition do + nil -> nil + _ -> String.capitalize(disposition) + end + end + + # FIXME: duplicated from respondent_controller + defp csv_filename(survey, prefix) do + name = survey.name || "survey_id_#{survey.id}" + name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") + prefix = "#{name}-#{prefix}" + Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) + end + + # FIXME: duplicated from respondent_controller + defp csv_datetime(nil, _, _), do: "" + + defp csv_datetime(dt, tz_offset_in_seconds, tz_offset) when is_binary(dt) do + {:ok, datetime, _offset} = DateTime.from_iso8601(dt) + csv_datetime(datetime, tz_offset_in_seconds, tz_offset) + end + + defp csv_datetime(dt, tz_offset_in_seconds, tz_offset) do + Ask.TimeUtil.format(dt, tz_offset_in_seconds, tz_offset) + end + + + ## Public API + + def generate_interactions_file(survey_id) do + Logger.info("Enqueueing generation of survey (id: #{survey_id}) interaction file") + GenServer.cast(server_ref(), {:interactions, survey_id}) + end +end diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index ad5c3df47..85a64122b 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -11,10 +11,11 @@ defmodule AskWeb.RespondentController do RespondentDispositionHistory, Stats, Survey, - SurveyLogEntry, RespondentsFilter } + alias Ask.Runtime.SurveyFilesManager + @db_chunk_limit 10_000 def index(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do @@ -1131,74 +1132,12 @@ defmodule AskWeb.RespondentController do project = load_project_for_owner(conn, project_id) survey = load_survey(project, survey_id) - channels = survey_log_entry_channel_names(survey) - - log_entries = - Stream.resource( - fn -> {"", 0} end, - fn {last_hash, last_id} -> - results = - from(e in SurveyLogEntry, - where: - e.survey_id == ^survey.id and - ((e.respondent_hashed_number == ^last_hash and e.id > ^last_id) or - e.respondent_hashed_number > ^last_hash), - order_by: [e.respondent_hashed_number, e.id], - limit: @db_chunk_limit - ) - |> Repo.all() - - case List.last(results) do - nil -> {:halt, {last_hash, last_id}} - last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id}} - end - end, - fn _ -> [] end - ) - - tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) - tz_offset = Survey.timezone_offset(survey) - - csv_rows = - log_entries - |> Stream.map(fn e -> - [ - Integer.to_string(e.id), - e.respondent_hashed_number, - interactions_mode_label(e.mode), - Map.get(channels, e.channel_id, ""), - disposition_label(e.disposition), - action_type_label(e.action_type), - e.action_data, - csv_datetime(e.timestamp, tz_offset_in_seconds, tz_offset) - ] - end) - - header = [ - "ID", - "Respondent ID", - "Mode", - "Channel", - "Disposition", - "Action Type", - "Action Data", - "Timestamp" - ] - - rows = Stream.concat([[header], csv_rows]) - - filename = csv_filename(survey, "respondents_interactions") + # TODO: We just change this for "trigger generation" + # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "interactions") |> Repo.insert() - conn |> csv_stream(rows, filename) - end - defp survey_log_entry_channel_names(survey) do - respondent_groups = Repo.preload(survey, respondent_groups: [:channels]).respondent_groups - respondent_groups - |> Enum.flat_map(fn resp_group -> resp_group.channels end) - |> Enum.map( fn channel -> {channel.id, channel.name} end) - |> MapSet.new # convert to set to remove duplicates - |> Enum.into(%{}) + SurveyFilesManager.generate_interactions_file(survey_id) + conn |> send_resp(200, "OK") end defp mask_phone_numbers(respondent) do @@ -1228,28 +1167,6 @@ defmodule AskWeb.RespondentController do end end - defp interactions_mode_label(mode) do - case mode do - "mobileweb" -> "Mobile Web" - _ -> String.upcase(mode) - end - end - - defp action_type_label(action) do - case action do - nil -> nil - "contact" -> "Contact attempt" - _ -> String.capitalize(action) - end - end - - defp disposition_label(disposition) do - case disposition do - nil -> nil - _ -> String.capitalize(disposition) - end - end - defp csv_filename(survey, prefix) do name = survey.name || "survey_id_#{survey.id}" name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") From 1539c010c004eb6e6cd030e65e26d126fd56485b Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Thu, 1 Aug 2024 14:02:05 -0300 Subject: [PATCH 02/46] Generate interactions file into a new dir --- .gitignore | 2 ++ lib/ask/runtime/survey_files_manager.ex | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 9ab4b18f7..91367f044 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ yarn-error.log /integration_tests/cypress/screenshots /integration_tests/cypress/videos /integration_tests/output + +/generated_files diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex index 24d3997ed..b3d15e310 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/runtime/survey_files_manager.ex @@ -13,6 +13,7 @@ defmodule Ask.Runtime.SurveyFilesManager do import Ecto.Query @db_chunk_limit 10_000 + @target_dir "generated_files" @server_ref {:global, __MODULE__} def server_ref, do: @server_ref @@ -90,7 +91,8 @@ defmodule Ask.Runtime.SurveyFilesManager do rows = Stream.concat([[header], csv_rows]) filename = csv_filename(survey, "respondents_interactions") - file = File.open!(filename, [:write, :utf8]) + File.mkdir_p!(@target_dir) + file = File.open!("#{@target_dir}/#{filename}", [:write, :utf8]) initial_datetime = Timex.now() rows |> CSV.encode() @@ -143,7 +145,7 @@ defmodule Ask.Runtime.SurveyFilesManager do defp csv_filename(survey, prefix) do name = survey.name || "survey_id_#{survey.id}" name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") - prefix = "#{name}-#{prefix}" + prefix = "#{name}_#{survey.state}-#{prefix}" Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) end @@ -161,7 +163,6 @@ defmodule Ask.Runtime.SurveyFilesManager do ## Public API - def generate_interactions_file(survey_id) do Logger.info("Enqueueing generation of survey (id: #{survey_id}) interaction file") GenServer.cast(server_ref(), {:interactions, survey_id}) From aa163c81b0ad855648c7731f123f41cc94394bc6 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Thu, 1 Aug 2024 15:47:07 -0300 Subject: [PATCH 03/46] add basic decision for generating file or not --- lib/ask/runtime/survey_files_manager.ex | 126 +++++++++++++++--------- 1 file changed, 78 insertions(+), 48 deletions(-) diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex index b3d15e310..6a31a2ba8 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/runtime/survey_files_manager.ex @@ -1,8 +1,7 @@ - ## This GenServer uses :hibernate since its mailbox might be most of the time empty ## ## Hibernating a GenServer causes garbage collection and leaves a continuous -## heap that minimises the memory used by the process. +## heap that minimizes the memory used by the process. ## ## When a process is hibernated it will continue the loop once a message is in its ## message queue. If when returning a handle_* call there is already a message @@ -19,53 +18,50 @@ defmodule Ask.Runtime.SurveyFilesManager do def server_ref, do: @server_ref def start_link do - GenServer.start_link(__MODULE__, %{}, name: @server_ref) + GenServer.start_link(__MODULE__, %{}, name: @server_ref) end @impl true def init(state) do - Logger.info("SurveyFilesManager started") - {:ok, state} + Logger.info("SurveyFilesManager started") + {:ok, state} end - @impl true - def handle_cast({:interactions, survey_id}, state) do - survey = Repo.get!(Survey, survey_id) - - channels = survey_log_entry_channel_names(survey) + defp do_generate_interactions_file(survey) do + channels = survey_log_entry_channel_names(survey) - Logger.info("Starting to build interaction file (survey_id: #{survey_id})") + Logger.info("Starting to build interaction file (survey_id: #{survey.id})") - log_entries = + log_entries = Stream.resource( - fn -> {"", 0} end, - fn {last_hash, last_id} -> + fn -> {"", 0} end, + fn {last_hash, last_id} -> results = - from(e in SurveyLogEntry, + from(e in SurveyLogEntry, where: - e.survey_id == ^survey.id and + e.survey_id == ^survey.id and ((e.respondent_hashed_number == ^last_hash and e.id > ^last_id) or - e.respondent_hashed_number > ^last_hash), + e.respondent_hashed_number > ^last_hash), order_by: [e.respondent_hashed_number, e.id], limit: @db_chunk_limit - ) - |> Repo.all() + ) + |> Repo.all() case List.last(results) do - nil -> {:halt, {last_hash, last_id}} - last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id}} + nil -> {:halt, {last_hash, last_id}} + last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id}} end - end, - fn _ -> [] end + end, + fn _ -> [] end ) - tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) - tz_offset = Survey.timezone_offset(survey) + tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) + tz_offset = Survey.timezone_offset(survey) - csv_rows = + csv_rows = log_entries |> Stream.map(fn e -> - [ + [ Integer.to_string(e.id), e.respondent_hashed_number, interactions_mode_label(e.mode), @@ -74,10 +70,10 @@ defmodule Ask.Runtime.SurveyFilesManager do action_type_label(e.action_type), e.action_data, csv_datetime(e.timestamp, tz_offset_in_seconds, tz_offset) - ] + ] end) - header = [ + header = [ "ID", "Respondent ID", "Mode", @@ -86,32 +82,63 @@ defmodule Ask.Runtime.SurveyFilesManager do "Action Type", "Action Data", "Timestamp" - ] + ] - rows = Stream.concat([[header], csv_rows]) + rows = Stream.concat([[header], csv_rows]) - filename = csv_filename(survey, "respondents_interactions") - File.mkdir_p!(@target_dir) - file = File.open!("#{@target_dir}/#{filename}", [:write, :utf8]) - initial_datetime = Timex.now() - rows - |> CSV.encode() - |> Enum.each(&IO.write(file, &1)) + filename = csv_filename(survey, file_prefix(:interactions)) + File.mkdir_p!(@target_dir) + file = File.open!("#{@target_dir}/#{filename}", [:write, :utf8]) + initial_datetime = Timex.now() - seconds_to_process_file = Timex.diff(Timex.now(), initial_datetime, :seconds) - Logger.info("Generation of interaction files survey (id #{survey_id}) took #{seconds_to_process_file} seconds") + rows + |> CSV.encode() + |> Enum.each(&IO.write(file, &1)) - {:noreply, state, :hibernate} + seconds_to_process_file = Timex.diff(Timex.now(), initial_datetime, :seconds) + + Logger.info( + "Generation of interaction files survey (id #{survey.id}) took #{seconds_to_process_file} seconds" + ) + end + + defp file_prefix(:interactions), do: "respondents_interactions" + defp file_prefix(_), do: "" + + defp should_generate_file(:interactions, survey) do + # TODO: when do we want to skip the re-generation of the file? + existing_files = File.ls!(@target_dir) + + exists_file = + existing_files + |> Enum.any?(fn file -> + file |> String.starts_with?(survey_filename_prefix(survey, file_prefix(:interactions))) + end) + + !exists_file + end + + @impl true + def handle_cast({:interactions, survey_id}, state) do + survey = Repo.get!(Survey, survey_id) + + if should_generate_file(:interactions, survey) do + do_generate_interactions_file(survey) + else + Logger.info("Ignoring generation of :interaction file") + end + {:noreply, state, :hibernate} end @impl true def handle_cast(message, state) do - Logger.warn("Ignoring message #{message}") - {:noreply, state, :hibernate} + Logger.warn("Ignoring message #{message}") + {:noreply, state, :hibernate} end - + defp survey_log_entry_channel_names(survey) do respondent_groups = Repo.preload(survey, respondent_groups: [:channels]).respondent_groups + respondent_groups |> Enum.flat_map(fn resp_group -> resp_group.channels end) |> Enum.map( fn channel -> {channel.id, channel.name} end) @@ -143,12 +170,16 @@ defmodule Ask.Runtime.SurveyFilesManager do # FIXME: duplicated from respondent_controller defp csv_filename(survey, prefix) do + prefix = survey_filename_prefix(survey, prefix) + Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) + end + + defp survey_filename_prefix(survey, prefix) do name = survey.name || "survey_id_#{survey.id}" name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") - prefix = "#{name}_#{survey.state}-#{prefix}" - Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) + "#{name}_#{survey.state}-#{prefix}" end - + # FIXME: duplicated from respondent_controller defp csv_datetime(nil, _, _), do: "" @@ -161,7 +192,6 @@ defmodule Ask.Runtime.SurveyFilesManager do Ask.TimeUtil.format(dt, tz_offset_in_seconds, tz_offset) end - ## Public API def generate_interactions_file(survey_id) do Logger.info("Enqueueing generation of survey (id: #{survey_id}) interaction file") From d39129ebdd01a2033b9532eff8440db169741116 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Fri, 2 Aug 2024 12:39:00 -0300 Subject: [PATCH 04/46] Add incentives file to SurveyFilesManager --- lib/ask/runtime/survey_files_manager.ex | 137 +++++++++++++++--- .../controllers/respondent_controller.ex | 48 +----- 2 files changed, 118 insertions(+), 67 deletions(-) diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex index 6a31a2ba8..ecdd29a7e 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/runtime/survey_files_manager.ex @@ -8,7 +8,7 @@ ## in the message queue, the process will continue the loop immediately. defmodule Ask.Runtime.SurveyFilesManager do use GenServer - alias Ask.{Logger, Repo, Survey, SurveyLogEntry} + alias Ask.{Logger, Questionnaire, Repo, Respondent, Survey, SurveyLogEntry} import Ecto.Query @db_chunk_limit 10_000 @@ -27,7 +27,26 @@ defmodule Ask.Runtime.SurveyFilesManager do {:ok, state} end - defp do_generate_interactions_file(survey) do + @impl true + def handle_cast({file_type, survey_id}, state) do + survey = Repo.get!(Survey, survey_id) + + if should_generate_file(file_type, survey) do + do_generate_file(file_type, survey) + else + Logger.info("Ignoring generation of #{file_type} file (survey_id: #{survey_id})") + end + + {:noreply, state, :hibernate} + end + + @impl true + def handle_cast(message, state) do + Logger.warn("Ignoring message #{message}") + {:noreply, state, :hibernate} + end + + defp do_generate_file(:interactions, survey) do channels = survey_log_entry_channel_names(survey) Logger.info("Starting to build interaction file (survey_id: #{survey.id})") @@ -86,7 +105,46 @@ defmodule Ask.Runtime.SurveyFilesManager do rows = Stream.concat([[header], csv_rows]) - filename = csv_filename(survey, file_prefix(:interactions)) + write_to_file(:interactions, survey, rows) + end + + defp do_generate_file(:incentives, survey) do + questionnaires = survey_respondent_questionnaires(survey) + + tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) + tz_offset = Survey.timezone_offset(survey) + + Repo.transaction(fn -> + csv_rows = + from(r in Respondent, + where: + r.survey_id == ^survey.id and r.disposition == :completed and + not is_nil(r.questionnaire_id), + order_by: r.id + ) + |> Repo.stream() + |> Stream.map(fn r -> + questionnaire = Enum.find(questionnaires, fn q -> q.id == r.questionnaire_id end) + + [ + r.phone_number, + experiment_name(questionnaire, r.mode), + csv_datetime(r.completed_at, tz_offset_in_seconds, tz_offset) + ] + end) + + header = ["Telephone number", "Questionnaire-Mode", "Completion date"] + rows = Stream.concat([[header], csv_rows]) + + write_to_file(:incentives, survey, rows) + end) + end + + defp do_generate_file(file_type, survey), + do: Logger.warn("No function for generating #{file_type} files") + + defp write_to_file(file_type, survey, rows) do + filename = csv_filename(survey, file_prefix(file_type)) File.mkdir_p!(@target_dir) file = File.open!("#{@target_dir}/#{filename}", [:write, :utf8]) initial_datetime = Timex.now() @@ -98,11 +156,12 @@ defmodule Ask.Runtime.SurveyFilesManager do seconds_to_process_file = Timex.diff(Timex.now(), initial_datetime, :seconds) Logger.info( - "Generation of interaction files survey (id #{survey.id}) took #{seconds_to_process_file} seconds" + "Generation of #{file_type} file (survey_id: #{survey.id}) took #{seconds_to_process_file} seconds" ) end defp file_prefix(:interactions), do: "respondents_interactions" + defp file_prefix(:incentives), do: "respondents_incentives" defp file_prefix(_), do: "" defp should_generate_file(:interactions, survey) do @@ -118,31 +177,16 @@ defmodule Ask.Runtime.SurveyFilesManager do !exists_file end - @impl true - def handle_cast({:interactions, survey_id}, state) do - survey = Repo.get!(Survey, survey_id) - - if should_generate_file(:interactions, survey) do - do_generate_interactions_file(survey) - else - Logger.info("Ignoring generation of :interaction file") - end - {:noreply, state, :hibernate} - end - - @impl true - def handle_cast(message, state) do - Logger.warn("Ignoring message #{message}") - {:noreply, state, :hibernate} - end + defp should_generate_file(_type, _survey), do: true defp survey_log_entry_channel_names(survey) do respondent_groups = Repo.preload(survey, respondent_groups: [:channels]).respondent_groups - respondent_groups + respondent_groups |> Enum.flat_map(fn resp_group -> resp_group.channels end) - |> Enum.map( fn channel -> {channel.id, channel.name} end) - |> MapSet.new # convert to set to remove duplicates + |> Enum.map(fn channel -> {channel.id, channel.name} end) + # convert to set to remove duplicates + |> MapSet.new() |> Enum.into(%{}) end @@ -192,9 +236,54 @@ defmodule Ask.Runtime.SurveyFilesManager do Ask.TimeUtil.format(dt, tz_offset_in_seconds, tz_offset) end + # FIXME: duplicated from respondent_controller + defp experiment_name(quiz, mode) do + "#{questionnaire_name(quiz)} - #{mode_label(mode)}" + end + + # FIXME: duplicated from respondent_controller + defp mode_label(mode) do + case mode do + ["sms"] -> "SMS" + ["sms", "ivr"] -> "SMS with phone call fallback" + ["sms", "mobileweb"] -> "SMS with Mobile Web fallback" + ["ivr"] -> "Phone call" + ["ivr", "sms"] -> "Phone call with SMS fallback" + ["ivr", "mobileweb"] -> "Phone call with Mobile Web fallback" + ["mobileweb"] -> "Mobile Web" + ["mobileweb", "sms"] -> "Mobile Web with SMS fallback" + ["mobileweb", "ivr"] -> "Mobile Web with phone call fallback" + _ -> "Unknown mode" + end + end + + # FIXME: duplicated from respondent_controller + defp questionnaire_name(quiz) do + quiz.name || "Untitled questionnaire" + end + + defp survey_respondent_questionnaires(survey) do + from(q in Questionnaire, + where: + q.id in subquery( + from(r in Respondent, + distinct: true, + select: r.questionnaire_id, + where: r.survey_id == ^survey.id + ) + ) + ) + |> Repo.all() + end + ## Public API def generate_interactions_file(survey_id) do Logger.info("Enqueueing generation of survey (id: #{survey_id}) interaction file") GenServer.cast(server_ref(), {:interactions, survey_id}) end + + def generate_incentives_file(survey_id) do + Logger.info("Enqueueing generation of survey (id: #{survey_id}) incentives file") + GenServer.cast(server_ref(), {:incentives, survey_id}) + end end diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 85a64122b..ae91aa877 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -1082,50 +1082,12 @@ defmodule AskWeb.RespondentController do |> where([s], s.incentives_enabled) |> Repo.get!(survey_id) - questionnaires = survey_respondent_questionnaires(survey) - - tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) - tz_offset = Survey.timezone_offset(survey) - - csv_rows = - from(r in Respondent, - where: - r.survey_id == ^survey.id and r.disposition == :completed and - not is_nil(r.questionnaire_id), - order_by: r.id - ) - |> Repo.stream() - |> Stream.map(fn r -> - questionnaire = Enum.find(questionnaires, fn q -> q.id == r.questionnaire_id end) - - [ - r.phone_number, - experiment_name(questionnaire, r.mode), - csv_datetime(r.completed_at, tz_offset_in_seconds, tz_offset) - ] - end) - - header = ["Telephone number", "Questionnaire-Mode", "Completion date"] - rows = Stream.concat([[header], csv_rows]) - - filename = csv_filename(survey, "respondents_incentives") + # TODO: We just change this for "trigger generation" + # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() - {:ok, conn} = Repo.transaction(fn -> conn |> csv_stream(rows, filename) end) - conn - end - - defp survey_respondent_questionnaires(survey) do - from(q in Questionnaire, - where: - q.id in subquery( - from(r in Respondent, - distinct: true, - select: r.questionnaire_id, - where: r.survey_id == ^survey.id - ) - ) - ) - |> Repo.all() + + SurveyFilesManager.generate_incentives_file(survey_id) + conn |> send_resp(200, "OK") end def interactions(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do From c4c22effb1eaa6c409699cb31cc1d0335e7c915d Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Fri, 2 Aug 2024 12:55:20 -0300 Subject: [PATCH 05/46] add disposition_history files to SurveyFilesManager --- lib/ask/runtime/survey_files_manager.ex | 61 ++++++++++++++++++- .../controllers/respondent_controller.ex | 56 ++--------------- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex index ecdd29a7e..7a4cdf2dd 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/runtime/survey_files_manager.ex @@ -8,7 +8,17 @@ ## in the message queue, the process will continue the loop immediately. defmodule Ask.Runtime.SurveyFilesManager do use GenServer - alias Ask.{Logger, Questionnaire, Repo, Respondent, Survey, SurveyLogEntry} + + alias Ask.{ + Logger, + Questionnaire, + Repo, + Respondent, + RespondentDispositionHistory, + Survey, + SurveyLogEntry + } + import Ecto.Query @db_chunk_limit 10_000 @@ -140,7 +150,47 @@ defmodule Ask.Runtime.SurveyFilesManager do end) end - defp do_generate_file(file_type, survey), + defp do_generate_file(:disposition_history, survey) do + history = + Stream.resource( + fn -> 0 end, + fn last_id -> + results = + from(h in RespondentDispositionHistory, + where: h.survey_id == ^survey.id and h.id > ^last_id, + order_by: h.id, + limit: @db_chunk_limit + ) + |> Repo.all() + + case List.last(results) do + nil -> {:halt, last_id} + last_entry -> {results, last_entry.id} + end + end, + fn _ -> [] end + ) + + tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) + tz_offset = Survey.timezone_offset(survey) + + csv_rows = + history + |> Stream.map(fn history -> + [ + history.respondent_hashed_number, + history.disposition, + mode_label([history.mode]), + csv_datetime(history.inserted_at, tz_offset_in_seconds, tz_offset) + ] + end) + + header = ["Respondent ID", "Disposition", "Mode", "Timestamp"] + rows = Stream.concat([[header], csv_rows]) + write_to_file(:disposition_history, survey, rows) + end + + defp do_generate_file(file_type, _), do: Logger.warn("No function for generating #{file_type} files") defp write_to_file(file_type, survey, rows) do @@ -162,6 +212,7 @@ defmodule Ask.Runtime.SurveyFilesManager do defp file_prefix(:interactions), do: "respondents_interactions" defp file_prefix(:incentives), do: "respondents_incentives" + defp file_prefix(:disposition_history), do: "disposition_history" defp file_prefix(_), do: "" defp should_generate_file(:interactions, survey) do @@ -224,7 +275,6 @@ defmodule Ask.Runtime.SurveyFilesManager do "#{name}_#{survey.state}-#{prefix}" end - # FIXME: duplicated from respondent_controller defp csv_datetime(nil, _, _), do: "" defp csv_datetime(dt, tz_offset_in_seconds, tz_offset) when is_binary(dt) do @@ -286,4 +336,9 @@ defmodule Ask.Runtime.SurveyFilesManager do Logger.info("Enqueueing generation of survey (id: #{survey_id}) incentives file") GenServer.cast(server_ref(), {:incentives, survey_id}) end + + def generate_disposition_history_file(survey_id) do + Logger.info("Enqueueing generation of survey (id: #{survey_id}) disposition_history file") + GenServer.cast(server_ref(), {:disposition_history, survey_id}) + end end diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index ae91aa877..fe8a72816 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -8,7 +8,6 @@ defmodule AskWeb.RespondentController do Logger, Questionnaire, Respondent, - RespondentDispositionHistory, Stats, Survey, RespondentsFilter @@ -1029,46 +1028,12 @@ defmodule AskWeb.RespondentController do project = load_project(conn, project_id) survey = load_survey(project, survey_id) - history = - Stream.resource( - fn -> 0 end, - fn last_id -> - results = - from(h in RespondentDispositionHistory, - where: h.survey_id == ^survey.id and h.id > ^last_id, - order_by: h.id, - limit: @db_chunk_limit - ) - |> Repo.all() - - case List.last(results) do - nil -> {:halt, last_id} - last_entry -> {results, last_entry.id} - end - end, - fn _ -> [] end - ) - - tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) - tz_offset = Survey.timezone_offset(survey) - - csv_rows = - history - |> Stream.map(fn history -> - [ - history.respondent_hashed_number, - history.disposition, - mode_label([history.mode]), - csv_datetime(history.inserted_at, tz_offset_in_seconds, tz_offset) - ] - end) - - header = ["Respondent ID", "Disposition", "Mode", "Timestamp"] - rows = Stream.concat([[header], csv_rows]) - - filename = csv_filename(survey, "respondents_disposition_history") + # TODO: We just change this for "trigger generation" + # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "disposition_history") |> Repo.insert() - conn |> csv_stream(rows, filename) + + SurveyFilesManager.generate_disposition_history_file(survey_id) + conn |> send_resp(200, "OK") end def incentives(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do @@ -1136,17 +1101,6 @@ defmodule AskWeb.RespondentController do Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) end - defp csv_datetime(nil, _, _), do: "" - - defp csv_datetime(dt, tz_offset_in_seconds, tz_offset) when is_binary(dt) do - {:ok, datetime, _offset} = DateTime.from_iso8601(dt) - csv_datetime(datetime, tz_offset_in_seconds, tz_offset) - end - - defp csv_datetime(dt, tz_offset_in_seconds, tz_offset) do - Ask.TimeUtil.format(dt, tz_offset_in_seconds, tz_offset) - end - defp load_survey(project, survey_id) do project |> assoc(:surveys) From 3a3d08494a722f2d679a9fb6376975c626cb4e83 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Fri, 2 Aug 2024 22:07:12 -0300 Subject: [PATCH 06/46] Convert csv trigger request into POST requests with no csv format --- assets/js/api.js | 11 +++++++++ .../respondents/RespondentIndex.jsx | 15 ++++++------ assets/js/routes.jsx | 10 -------- lib/ask_web/router.ex | 23 ++++++------------- 4 files changed, 26 insertions(+), 33 deletions(-) diff --git a/assets/js/api.js b/assets/js/api.js index 25994da3e..195886d1d 100644 --- a/assets/js/api.js +++ b/assets/js/api.js @@ -691,6 +691,17 @@ export const refreshDispositionHistoryLink = (projectId, surveyId) => { ) } +export const triggerRespondentsResultFile = (projectId, surveyId, q) => + apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/results?${ + (q && `&q=${encodeURIComponent(q)}`) || "" + }`, null, null) +export const triggerRespondentsDispositionHistoryCSV = (projectId, surveyId) => + apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/disposition_history`, null, null) +export const triggerRespondentsIncentivesCSV = (projectId, surveyId) => + apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/incentives`, null, null) +export const triggerRespondentsInteractionsCSV = (projectId, surveyId) => + apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/interactions`, null, null) + export const startSimulation = (projectId, questionnaireId, mode) => { return apiPutOrPostJSONWithCallback( `projects/${projectId}/questionnaires/${questionnaireId}/simulation`, diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index c9e0b4aae..ebc16cb5d 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -2,6 +2,7 @@ import React, { Component } from "react" import { bindActionCreators } from "redux" import { connect } from "react-redux" +import * as api from "../../api" import * as actions from "../../actions/respondents" import { fieldUniqueKey, isFieldSelected } from "../../reducers/respondents" import * as surveyActions from "../../actions/survey" @@ -123,22 +124,22 @@ class RespondentIndex extends Component { downloadCSV(applyUserFilter = false) { const { projectId, surveyId, filter } = this.props const q = (applyUserFilter && filter) || null - window.location = routes.respondentsResultsCSV(projectId, surveyId, q) + api.triggerRespondentsResultFile(projectId, surveyId, q) } - + downloadDispositionHistoryCSV() { const { projectId, surveyId } = this.props - window.location = routes.respondentsDispositionHistoryCSV(projectId, surveyId) + api.triggerRespondentsDispositionHistoryCSV(projectId, surveyId) } - + downloadIncentivesCSV() { const { projectId, surveyId } = this.props - window.location = routes.respondentsIncentivesCSV(projectId, surveyId) + api.triggerRespondentsIncentivesCSV(projectId, surveyId) } - + downloadInteractionsCSV() { const { projectId, surveyId } = this.props - window.location = routes.respondentsInteractionsCSV(projectId, surveyId) + api.triggerRespondentsInteractionsCSV(projectId, surveyId) } sortBy(name) { diff --git a/assets/js/routes.jsx b/assets/js/routes.jsx index d706b7389..56de3e89b 100644 --- a/assets/js/routes.jsx +++ b/assets/js/routes.jsx @@ -132,16 +132,6 @@ export const surveyRespondents = (projectId, surveyId, q) => export const surveySettings = (projectId, surveyId) => `${survey(projectId, surveyId)}/settings` export const surveyIntegrations = (projectId, surveyId) => `${survey(projectId, surveyId)}/integrations` -export const respondentsResultsCSV = (projectId, surveyId, q) => - `/api/v1${surveyRespondents(projectId, surveyId)}/results?_format=csv${ - (q && `&q=${encodeURIComponent(q)}`) || "" - }` -export const respondentsDispositionHistoryCSV = (projectId, surveyId) => - `/api/v1${surveyRespondents(projectId, surveyId)}/disposition_history?_format=csv` -export const respondentsIncentivesCSV = (projectId, surveyId) => - `/api/v1${surveyRespondents(projectId, surveyId)}/incentives?_format=csv` -export const respondentsInteractionsCSV = (projectId, surveyId) => - `/api/v1${surveyRespondents(projectId, surveyId)}/interactions?_format=csv` export const surveyEdit = (projectId, surveyId) => `${survey(projectId, surveyId)}/edit` export const surveyFolderNew = (projectId, surveyId) => `${survey(projectId, surveyId)}/folders/new` export const questionnaireIndex = (projectId) => `${project(projectId)}/questionnaires` diff --git a/lib/ask_web/router.ex b/lib/ask_web/router.ex index c41f3c49a..f2f115746 100644 --- a/lib/ask_web/router.ex +++ b/lib/ask_web/router.ex @@ -42,14 +42,6 @@ defmodule AskWeb.Router do plug Coherence.Authentication.Session, db_model: Ask.User end - pipeline :csv_api do - plug :accepts, ["csv"] - plug :fetch_session - plug :fetch_flash - - plug Coherence.Authentication.Session, db_model: Ask.User - end - pipeline :mp3_api do plug TrailingFormatPlug plug :accepts, ["mp3"] @@ -198,19 +190,18 @@ defmodule AskWeb.Router do resources "/projects", ProjectController, only: [] do resources "/surveys", SurveyController, only: [] do scope "/respondents" do - pipe_through :csv_json_api - get "/results", RespondentController, :results, as: :respondents_results - end + pipe_through :api - scope "/respondents" do - pipe_through :csv_api + get "/results", RespondentController, :results, as: :get_respondents_results + + post "/results", RespondentController, :trigger_results, as: :respondents_results - get "/disposition_history", RespondentController, :disposition_history, + post "/disposition_history", RespondentController, :disposition_history, as: :respondents_disposition_history - get "/incentives", RespondentController, :incentives, as: :respondents_incentives + post "/incentives", RespondentController, :incentives, as: :respondents_incentives - get "/interactions", RespondentController, :interactions, + post "/interactions", RespondentController, :interactions, as: :respondents_interactions end end From d1f861baa72d38cbaa096c82b7411d1f2cbd9e52 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Fri, 2 Aug 2024 22:12:25 -0300 Subject: [PATCH 07/46] add respondents_result file to SurveyFilesManager --- lib/ask/runtime/survey_files_manager.ex | 238 +++++++++++-- lib/ask/survey.ex | 84 +++++ .../controllers/respondent_controller.ex | 316 ++---------------- .../controllers/survey_link_controller.ex | 1 + 4 files changed, 338 insertions(+), 301 deletions(-) diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex index 7a4cdf2dd..f31b38821 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/runtime/survey_files_manager.ex @@ -8,6 +8,7 @@ ## in the message queue, the process will continue the loop immediately. defmodule Ask.Runtime.SurveyFilesManager do use GenServer + require Ask.RespondentStats alias Ask.{ Logger, @@ -15,6 +16,7 @@ defmodule Ask.Runtime.SurveyFilesManager do Repo, Respondent, RespondentDispositionHistory, + Stats, Survey, SurveyLogEntry } @@ -38,11 +40,11 @@ defmodule Ask.Runtime.SurveyFilesManager do end @impl true - def handle_cast({file_type, survey_id}, state) do + def handle_cast({file_type, survey_id, args}, state) do survey = Repo.get!(Survey, survey_id) if should_generate_file(file_type, survey) do - do_generate_file(file_type, survey) + do_generate_file(file_type, survey, args) else Logger.info("Ignoring generation of #{file_type} file (survey_id: #{survey_id})") end @@ -56,7 +58,7 @@ defmodule Ask.Runtime.SurveyFilesManager do {:noreply, state, :hibernate} end - defp do_generate_file(:interactions, survey) do + defp do_generate_file(:interactions, survey, _) do channels = survey_log_entry_channel_names(survey) Logger.info("Starting to build interaction file (survey_id: #{survey.id})") @@ -118,7 +120,7 @@ defmodule Ask.Runtime.SurveyFilesManager do write_to_file(:interactions, survey, rows) end - defp do_generate_file(:incentives, survey) do + defp do_generate_file(:incentives, survey, _) do questionnaires = survey_respondent_questionnaires(survey) tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) @@ -150,7 +152,7 @@ defmodule Ask.Runtime.SurveyFilesManager do end) end - defp do_generate_file(:disposition_history, survey) do + defp do_generate_file(:disposition_history, survey, _) do history = Stream.resource( fn -> 0 end, @@ -190,6 +192,171 @@ defmodule Ask.Runtime.SurveyFilesManager do write_to_file(:disposition_history, survey, rows) end + defp do_generate_file(:respondent_result, survey, filter) do + + tz_offset = Survey.timezone_offset(survey) + + questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires + all_fields = all_questionnaires_fields(questionnaires, true) + has_comparisons = length(survey.comparisons) > 0 + + respondents = Ask.Survey.respondents_where(survey, filter) + + stats = + survey.mode + |> Enum.flat_map(fn modes -> + modes + |> Enum.flat_map(fn mode -> + case mode do + "sms" -> [:total_sent_sms, :total_received_sms, :sms_attempts] + "mobileweb" -> [:total_sent_sms, :total_received_sms, :mobileweb_attempts] + "ivr" -> [:total_call_time, :ivr_attempts] + _ -> [] + end + end) + end) + |> Enum.uniq() + + tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) + partial_relevant_enabled = Survey.partial_relevant_enabled?(survey, true) + respondents_count = Ask.RespondentStats.respondent_count(survey_id: ^survey.id) + + # Now traverse each respondent and create a row for it + csv_rows = + respondents + |> Stream.map(fn respondent -> + row = [respondent.hashed_number] + responses = respondent.responses + + row = row ++ [Respondent.show_disposition(respondent.disposition)] + + date = + case responses do + [] -> + nil + + _ -> + responses + |> Enum.map(fn r -> r.updated_at end) + |> Enum.max() + end + + row = + if date do + row ++ [Ask.TimeUtil.format2(date, tz_offset_in_seconds, tz_offset)] + else + row ++ ["-"] + end + + modes = + (respondent.effective_modes || []) + |> Enum.map(fn mode -> mode_label([mode]) end) + |> Enum.join(", ") + + row = row ++ [modes] + + row = row ++ [respondent.user_stopped] + + row = + row ++ + Enum.map(stats, fn stat -> + respondent |> respondent_stat(stat) + end) + + row = row ++ [Respondent.show_section_order(respondent, questionnaires)] + + respondent_group = respondent.respondent_group.name + + row = row ++ [respondent_group] + + questionnaire_id = respondent.questionnaire_id + questionnaire = questionnaires |> Enum.find(fn q -> q.id == questionnaire_id end) + mode = respondent.mode + + row = + if has_comparisons do + variant = + if questionnaire && mode do + experiment_name(questionnaire, mode) + else + "-" + end + + row ++ [variant] + else + row + end + + row = + if partial_relevant_enabled do + respondent_with_questionnaire = %{respondent | questionnaire: questionnaire} + + row ++ + [Respondent.partial_relevant_answered_count(respondent_with_questionnaire, false)] + else + row + end + + # We traverse all fields and see if there's a response for this respondent + row = + all_fields + |> Enum.reduce(row, fn field_name, acc -> + response = + responses + |> Enum.filter(fn response -> + response.field_name |> sanitize_variable_name == field_name + end) + + case response do + [resp] -> + value = resp.value + + # For the 'language' variable we convert the code to the native name + value = + if resp.field_name == "language" do + LanguageNames.for(value) || value + else + value + end + + acc ++ [value] + + _ -> + acc ++ [""] + end + end) + + row + end) + + append_if = fn list, elems, condition -> if condition, do: list ++ elems, else: list end + + # Add header to csv_rows + header = ["respondent_id", "disposition", "date", "modes", "user_stopped"] + + header = + header ++ + Enum.map(stats, fn stat -> + case stat do + :total_sent_sms -> "total_sent_sms" + :total_received_sms -> "total_received_sms" + :total_call_time -> "total_call_time" + :sms_attempts -> "sms_attempts" + :ivr_attempts -> "ivr_attempts" + :mobileweb_attempts -> "mobileweb_attempts" + end + end) + + header = header ++ ["section_order", "sample_file"] + header = append_if.(header, ["variant"], has_comparisons) + header = append_if.(header, ["p_relevants"], partial_relevant_enabled) + header = header ++ all_fields + + rows = Stream.concat([[header], csv_rows]) + + write_to_file(:respondent_result, survey, rows) + end + defp do_generate_file(file_type, _), do: Logger.warn("No function for generating #{file_type} files") @@ -213,6 +380,7 @@ defmodule Ask.Runtime.SurveyFilesManager do defp file_prefix(:interactions), do: "respondents_interactions" defp file_prefix(:incentives), do: "respondents_incentives" defp file_prefix(:disposition_history), do: "disposition_history" + defp file_prefix(:respondent_result), do: "respondents" defp file_prefix(_), do: "" defp should_generate_file(:interactions, survey) do @@ -311,34 +479,66 @@ defmodule Ask.Runtime.SurveyFilesManager do defp questionnaire_name(quiz) do quiz.name || "Untitled questionnaire" end - + defp survey_respondent_questionnaires(survey) do from(q in Questionnaire, - where: - q.id in subquery( - from(r in Respondent, - distinct: true, - select: r.questionnaire_id, - where: r.survey_id == ^survey.id - ) - ) - ) - |> Repo.all() + where: + q.id in subquery( + from(r in Respondent, + distinct: true, + select: r.questionnaire_id, + where: r.survey_id == ^survey.id + ) + ) + ) + |> Repo.all() + end + + # FIXME: duplicated from respondent_controller + defp respondent_stat(respondent, :sms_attempts), do: respondent.stats |> Stats.attempts(:sms) + defp respondent_stat(respondent, :ivr_attempts), do: respondent.stats |> Stats.attempts(:ivr) + + defp respondent_stat(respondent, :mobileweb_attempts), + do: respondent.stats |> Stats.attempts(:mobileweb) + + defp respondent_stat(respondent, key), do: apply(Stats, key, [respondent.stats]) + + # FIXME: duplicated from respondent_controller + defp all_questionnaires_fields(questionnaires, sanitize \\ false) do + fields = + questionnaires + |> Enum.flat_map(&Questionnaire.variables/1) + |> Enum.uniq() + |> Enum.reject(fn s -> String.length(s) == 0 end) + + if sanitize, do: sanitize_fields(fields), else: fields end + + # FIXME: duplicated from respondent_controller + def sanitize_variable_name(s), do: s |> String.trim() |> String.replace(" ", "_") + + # FIXME: duplicated from respondent_controller + defp sanitize_fields(fields), + do: Enum.map(fields, fn field -> sanitize_variable_name(field) end) ## Public API def generate_interactions_file(survey_id) do Logger.info("Enqueueing generation of survey (id: #{survey_id}) interaction file") - GenServer.cast(server_ref(), {:interactions, survey_id}) + GenServer.cast(server_ref(), {:interactions, survey_id, nil}) end def generate_incentives_file(survey_id) do Logger.info("Enqueueing generation of survey (id: #{survey_id}) incentives file") - GenServer.cast(server_ref(), {:incentives, survey_id}) + GenServer.cast(server_ref(), {:incentives, survey_id, nil}) end def generate_disposition_history_file(survey_id) do Logger.info("Enqueueing generation of survey (id: #{survey_id}) disposition_history file") - GenServer.cast(server_ref(), {:disposition_history, survey_id}) + GenServer.cast(server_ref(), {:disposition_history, survey_id, nil}) + end + + def generate_respondent_result_file(survey_id, filters) do + Logger.info("Enqueueing generation of survey (id: #{survey_id}) disposition_history file") + GenServer.cast(server_ref(), {:respondent_result, survey_id, filters}) end end diff --git a/lib/ask/survey.ex b/lib/ask/survey.ex index bdf2101bf..1d8451376 100644 --- a/lib/ask/survey.ex +++ b/lib/ask/survey.ex @@ -18,6 +18,7 @@ defmodule Ask.Survey do FloipEndpoint, Folder, RespondentStats, + RespondentsFilter, ConfigHelper, SystemTime, PanelSurvey @@ -28,6 +29,7 @@ defmodule Ask.Survey do @max_int 2_147_483_647 @default_fallback_delay 120 + @db_chunk_limit 10_000 schema "surveys" do field :name, :string @@ -836,4 +838,86 @@ defmodule Ask.Survey do survey = Repo.preload(survey, panel_survey: :waves) Enum.count(survey.panel_survey.waves) == 1 end + + # FIXME: remove from survey + defp experiment_name(quiz, mode) do + "#{questionnaire_name(quiz)} - #{mode_label(mode)}" + end + + # FIXME: remove from survey + defp mode_label(mode) do + case mode do + ["sms"] -> "SMS" + ["sms", "ivr"] -> "SMS with phone call fallback" + ["sms", "mobileweb"] -> "SMS with Mobile Web fallback" + ["ivr"] -> "Phone call" + ["ivr", "sms"] -> "Phone call with SMS fallback" + ["ivr", "mobileweb"] -> "Phone call with Mobile Web fallback" + ["mobileweb"] -> "Mobile Web" + ["mobileweb", "sms"] -> "Mobile Web with SMS fallback" + ["mobileweb", "ivr"] -> "Mobile Web with phone call fallback" + _ -> "Unknown mode" + end + end + + # FIXME: remove from survey + defp questionnaire_name(quiz) do + quiz.name || "Untitled questionnaire" + end + + # FIXME: remove from survey + def respondents_where(survey, filter) do + filter_where = RespondentsFilter.filter_where(filter, optimized: true) + + respondents = + Stream.resource( + fn -> 0 end, + fn last_seen_id -> + results = + from(r1 in Respondent, + join: r2 in Respondent, + on: r1.id == r2.id, + where: r2.survey_id == ^survey.id and r2.id > ^last_seen_id, + where: ^filter_where, + order_by: r2.id, + limit: @db_chunk_limit, + preload: [:responses, :respondent_group], + select: r1 + ) + |> Repo.all() + + case List.last(results) do + %{id: last_id} -> {results, last_id} + nil -> {:halt, last_seen_id} + end + end, + fn _ -> [] end + ) + + survey_has_comparisons = length(survey.comparisons) > 0 + questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires + + if survey_has_comparisons do + respondents + |> Stream.map(fn respondent -> + experiment_name = + if respondent.questionnaire_id && respondent.mode do + questionnaire = + questionnaires |> Enum.find(fn q -> q.id == respondent.questionnaire_id end) + + if questionnaire do + experiment_name(questionnaire, respondent.mode) + else + "-" + end + else + "-" + end + + %{respondent | experiment_name: experiment_name} + end) + else + respondents + end + end end diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index fe8a72816..dbe8ae0d1 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -15,8 +15,6 @@ defmodule AskWeb.RespondentController do alias Ask.Runtime.SurveyFilesManager - @db_chunk_limit 10_000 - def index(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do limit = Map.get(params, "limit", "") page = Map.get(params, "page", "") @@ -724,47 +722,41 @@ defmodule AskWeb.RespondentController do # ?param1=value is more specific than ?q=param1:value filter = add_params_to_filter(filter, params) - filter_where = RespondentsFilter.filter_where(filter, optimized: true) - - respondents = - Stream.resource( - fn -> 0 end, - fn last_seen_id -> - results = - from(r1 in Respondent, - join: r2 in Respondent, - on: r1.id == r2.id, - where: r2.survey_id == ^survey.id and r2.id > ^last_seen_id, - where: ^filter_where, - order_by: r2.id, - limit: @db_chunk_limit, - preload: [:responses, :respondent_group], - select: r1 - ) - |> Repo.all() - - case List.last(results) do - %{id: last_id} -> {results, last_id} - nil -> {:halt, last_seen_id} - end - end, - fn _ -> [] end - ) + # filter_where = RespondentsFilter.filter_where(filter, optimized: true) + respondents = Ask.Survey.respondents_where(survey, filter) partial_relevant_enabled = Survey.partial_relevant_enabled?(survey, true) - render_results( - conn, - get_format(conn), - project, - survey, - tz_offset, - questionnaires, - has_comparisons, - all_fields, - respondents, - partial_relevant_enabled - ) + respondents_count = Ask.RespondentStats.respondent_count(survey_id: ^survey.id) + + {:ok, conn} = + Repo.transaction(fn -> + render(conn, "index.json", + respondents: respondents, + respondents_count: respondents_count, + partial_relevant_enabled: partial_relevant_enabled + ) + end) + + conn + end + + def trigger_results(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do + project = load_project(conn, project_id) + survey = load_survey(project, survey_id) + + # The new filters, shared by the index and the downloaded CSV file + filter = RespondentsFilter.parse(Map.get(params, "q", "")) + # The old filters are being received by its own specific url params + # If the same filter is received twice, the old filter is priorized over new one because + # ?param1=value is more specific than ?q=param1:value + filter = add_params_to_filter(filter, params) + + SurveyFilesManager.generate_respondent_result_file(survey_id, filter) + + ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() + + conn |> send_resp(200, "OK") end defp all_questionnaires_fields(questionnaires, sanitize \\ false) do @@ -799,223 +791,6 @@ defmodule AskWeb.RespondentController do filter end - defp render_results( - conn, - "json", - _project, - survey, - _tz_offset, - questionnaires, - has_comparisons, - _all_fields, - respondents, - partial_relevant_enabled - ) do - respondents_count = Ask.RespondentStats.respondent_count(survey_id: ^survey.id) - - respondents = - if has_comparisons do - respondents - |> Stream.map(fn respondent -> - experiment_name = - if respondent.questionnaire_id && respondent.mode do - questionnaire = - questionnaires |> Enum.find(fn q -> q.id == respondent.questionnaire_id end) - - if questionnaire do - experiment_name(questionnaire, respondent.mode) - else - "-" - end - else - "-" - end - - %{respondent | experiment_name: experiment_name} - end) - else - respondents - end - - {:ok, conn} = - Repo.transaction(fn -> - render(conn, "index.json", - respondents: respondents, - respondents_count: respondents_count, - partial_relevant_enabled: partial_relevant_enabled - ) - end) - - conn - end - - defp render_results( - conn, - "csv", - project, - survey, - tz_offset, - questionnaires, - has_comparisons, - all_fields, - respondents, - partial_relevant_enabled - ) do - stats = - survey.mode - |> Enum.flat_map(fn modes -> - modes - |> Enum.flat_map(fn mode -> - case mode do - "sms" -> [:total_sent_sms, :total_received_sms, :sms_attempts] - "mobileweb" -> [:total_sent_sms, :total_received_sms, :mobileweb_attempts] - "ivr" -> [:total_call_time, :ivr_attempts] - _ -> [] - end - end) - end) - |> Enum.uniq() - - tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) - - # Now traverse each respondent and create a row for it - csv_rows = - respondents - |> Stream.map(fn respondent -> - row = [respondent.hashed_number] - responses = respondent.responses - - row = row ++ [Respondent.show_disposition(respondent.disposition)] - - date = - case responses do - [] -> - nil - - _ -> - responses - |> Enum.map(fn r -> r.updated_at end) - |> Enum.max() - end - - row = - if date do - row ++ [Ask.TimeUtil.format2(date, tz_offset_in_seconds, tz_offset)] - else - row ++ ["-"] - end - - modes = - (respondent.effective_modes || []) - |> Enum.map(fn mode -> mode_label([mode]) end) - |> Enum.join(", ") - - row = row ++ [modes] - - row = row ++ [respondent.user_stopped] - - row = - row ++ - Enum.map(stats, fn stat -> - respondent |> respondent_stat(stat) - end) - - row = row ++ [Respondent.show_section_order(respondent, questionnaires)] - - respondent_group = respondent.respondent_group.name - - row = row ++ [respondent_group] - - questionnaire_id = respondent.questionnaire_id - questionnaire = questionnaires |> Enum.find(fn q -> q.id == questionnaire_id end) - mode = respondent.mode - - row = - if has_comparisons do - variant = - if questionnaire && mode do - experiment_name(questionnaire, mode) - else - "-" - end - - row ++ [variant] - else - row - end - - row = - if partial_relevant_enabled do - respondent_with_questionnaire = %{respondent | questionnaire: questionnaire} - - row ++ - [Respondent.partial_relevant_answered_count(respondent_with_questionnaire, false)] - else - row - end - - # We traverse all fields and see if there's a response for this respondent - row = - all_fields - |> Enum.reduce(row, fn field_name, acc -> - response = - responses - |> Enum.filter(fn response -> - response.field_name |> sanitize_variable_name == field_name - end) - - case response do - [resp] -> - value = resp.value - - # For the 'language' variable we convert the code to the native name - value = - if resp.field_name == "language" do - LanguageNames.for(value) || value - else - value - end - - acc ++ [value] - - _ -> - acc ++ [""] - end - end) - - row - end) - - append_if = fn list, elems, condition -> if condition, do: list ++ elems, else: list end - - # Add header to csv_rows - header = ["respondent_id", "disposition", "date", "modes", "user_stopped"] - - header = - header ++ - Enum.map(stats, fn stat -> - case stat do - :total_sent_sms -> "total_sent_sms" - :total_received_sms -> "total_received_sms" - :total_call_time -> "total_call_time" - :sms_attempts -> "sms_attempts" - :ivr_attempts -> "ivr_attempts" - :mobileweb_attempts -> "mobileweb_attempts" - end - end) - - header = header ++ ["section_order", "sample_file"] - header = append_if.(header, ["variant"], has_comparisons) - header = append_if.(header, ["p_relevants"], partial_relevant_enabled) - header = header ++ all_fields - - rows = Stream.concat([[header], csv_rows]) - - filename = csv_filename(survey, "respondents") - ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() - conn |> csv_stream(rows, filename) - end - defp respondent_stat(respondent, :sms_attempts), do: respondent.stats |> Stats.attempts(:sms) defp respondent_stat(respondent, :ivr_attempts), do: respondent.stats |> Stats.attempts(:ivr) @@ -1031,7 +806,7 @@ defmodule AskWeb.RespondentController do # TODO: We just change this for "trigger generation" # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "disposition_history") |> Repo.insert() - + SurveyFilesManager.generate_disposition_history_file(survey_id) conn |> send_resp(200, "OK") end @@ -1050,7 +825,7 @@ defmodule AskWeb.RespondentController do # TODO: We just change this for "trigger generation" # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() - + SurveyFilesManager.generate_incentives_file(survey_id) conn |> send_resp(200, "OK") end @@ -1071,29 +846,6 @@ defmodule AskWeb.RespondentController do %{respondent | phone_number: Respondent.mask_phone_number(respondent.phone_number)} end - defp experiment_name(quiz, mode) do - "#{questionnaire_name(quiz)} - #{mode_label(mode)}" - end - - defp questionnaire_name(quiz) do - quiz.name || "Untitled questionnaire" - end - - defp mode_label(mode) do - case mode do - ["sms"] -> "SMS" - ["sms", "ivr"] -> "SMS with phone call fallback" - ["sms", "mobileweb"] -> "SMS with Mobile Web fallback" - ["ivr"] -> "Phone call" - ["ivr", "sms"] -> "Phone call with SMS fallback" - ["ivr", "mobileweb"] -> "Phone call with Mobile Web fallback" - ["mobileweb"] -> "Mobile Web" - ["mobileweb", "sms"] -> "Mobile Web with SMS fallback" - ["mobileweb", "ivr"] -> "Mobile Web with phone call fallback" - _ -> "Unknown mode" - end - end - defp csv_filename(survey, prefix) do name = survey.name || "survey_id_#{survey.id}" name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") diff --git a/lib/ask_web/controllers/survey_link_controller.ex b/lib/ask_web/controllers/survey_link_controller.ex index 26ed9fda5..6961390b5 100644 --- a/lib/ask_web/controllers/survey_link_controller.ex +++ b/lib/ask_web/controllers/survey_link_controller.ex @@ -16,6 +16,7 @@ defmodule AskWeb.SurveyLinkController do {name, target} = case target_name do + # FIXME: revisit this. We should remove _format "results" -> { Survey.link_name(survey, :results), From 400135db9cc1aff61d567cea739bcccd12d84780 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Mon, 5 Aug 2024 12:25:13 -0300 Subject: [PATCH 08/46] Cleanup --- lib/ask/questionnaire.ex | 2 ++ lib/ask/runtime/survey_files_manager.ex | 17 +++-------- .../controllers/respondent_controller.ex | 28 +------------------ 3 files changed, 7 insertions(+), 40 deletions(-) diff --git a/lib/ask/questionnaire.ex b/lib/ask/questionnaire.ex index 1ffb1f1e4..d38944b24 100644 --- a/lib/ask/questionnaire.ex +++ b/lib/ask/questionnaire.ex @@ -146,6 +146,8 @@ defmodule Ask.Questionnaire do nil end + def sanitize_variable_name(s), do: s |> String.trim() |> String.replace(" ", "_") + def all_steps(%Questionnaire{steps: steps, quota_completed_steps: nil}) do get_steps(steps) end diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex index f31b38821..3aea4308f 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/runtime/survey_files_manager.ex @@ -219,7 +219,6 @@ defmodule Ask.Runtime.SurveyFilesManager do tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) partial_relevant_enabled = Survey.partial_relevant_enabled?(survey, true) - respondents_count = Ask.RespondentStats.respondent_count(survey_id: ^survey.id) # Now traverse each respondent and create a row for it csv_rows = @@ -304,7 +303,7 @@ defmodule Ask.Runtime.SurveyFilesManager do response = responses |> Enum.filter(fn response -> - response.field_name |> sanitize_variable_name == field_name + response.field_name |> Questionnaire.sanitize_variable_name == field_name end) case response do @@ -357,7 +356,7 @@ defmodule Ask.Runtime.SurveyFilesManager do write_to_file(:respondent_result, survey, rows) end - defp do_generate_file(file_type, _), + defp do_generate_file(file_type, _, _), do: Logger.warn("No function for generating #{file_type} files") defp write_to_file(file_type, survey, rows) do @@ -431,7 +430,6 @@ defmodule Ask.Runtime.SurveyFilesManager do end end - # FIXME: duplicated from respondent_controller defp csv_filename(survey, prefix) do prefix = survey_filename_prefix(survey, prefix) Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) @@ -454,12 +452,10 @@ defmodule Ask.Runtime.SurveyFilesManager do Ask.TimeUtil.format(dt, tz_offset_in_seconds, tz_offset) end - # FIXME: duplicated from respondent_controller defp experiment_name(quiz, mode) do "#{questionnaire_name(quiz)} - #{mode_label(mode)}" end - # FIXME: duplicated from respondent_controller defp mode_label(mode) do case mode do ["sms"] -> "SMS" @@ -475,7 +471,6 @@ defmodule Ask.Runtime.SurveyFilesManager do end end - # FIXME: duplicated from respondent_controller defp questionnaire_name(quiz) do quiz.name || "Untitled questionnaire" end @@ -494,7 +489,6 @@ defmodule Ask.Runtime.SurveyFilesManager do |> Repo.all() end - # FIXME: duplicated from respondent_controller defp respondent_stat(respondent, :sms_attempts), do: respondent.stats |> Stats.attempts(:sms) defp respondent_stat(respondent, :ivr_attempts), do: respondent.stats |> Stats.attempts(:ivr) @@ -504,7 +498,7 @@ defmodule Ask.Runtime.SurveyFilesManager do defp respondent_stat(respondent, key), do: apply(Stats, key, [respondent.stats]) # FIXME: duplicated from respondent_controller - defp all_questionnaires_fields(questionnaires, sanitize \\ false) do + defp all_questionnaires_fields(questionnaires, sanitize) do fields = questionnaires |> Enum.flat_map(&Questionnaire.variables/1) @@ -514,12 +508,9 @@ defmodule Ask.Runtime.SurveyFilesManager do if sanitize, do: sanitize_fields(fields), else: fields end - # FIXME: duplicated from respondent_controller - def sanitize_variable_name(s), do: s |> String.trim() |> String.replace(" ", "_") - # FIXME: duplicated from respondent_controller defp sanitize_fields(fields), - do: Enum.map(fields, fn field -> sanitize_variable_name(field) end) + do: Enum.map(fields, fn field -> Questionnaire.sanitize_variable_name(field) end) ## Public API def generate_interactions_file(survey_id) do diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index dbe8ae0d1..07d7246db 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -8,7 +8,6 @@ defmodule AskWeb.RespondentController do Logger, Questionnaire, Respondent, - Stats, Survey, RespondentsFilter } @@ -701,20 +700,10 @@ defmodule AskWeb.RespondentController do |> Enum.into(%{}) end - def sanitize_variable_name(s), do: s |> String.trim() |> String.replace(" ", "_") - def results(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do project = load_project(conn, project_id) survey = load_survey(project, survey_id) - tz_offset = Survey.timezone_offset(survey) - - questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires - has_comparisons = length(survey.comparisons) > 0 - - # We first need to get all unique field names in all questionnaires - all_fields = all_questionnaires_fields(questionnaires, true) - # The new filters, shared by the index and the downloaded CSV file filter = RespondentsFilter.parse(Map.get(params, "q", "")) # The old filters are being received by its own specific url params @@ -770,7 +759,7 @@ defmodule AskWeb.RespondentController do end defp sanitize_fields(fields), - do: Enum.map(fields, fn field -> sanitize_variable_name(field) end) + do: Enum.map(fields, fn field -> Questionnaire.sanitize_variable_name(field) end) defp add_params_to_filter(filter, params) do filter = @@ -791,14 +780,6 @@ defmodule AskWeb.RespondentController do filter end - defp respondent_stat(respondent, :sms_attempts), do: respondent.stats |> Stats.attempts(:sms) - defp respondent_stat(respondent, :ivr_attempts), do: respondent.stats |> Stats.attempts(:ivr) - - defp respondent_stat(respondent, :mobileweb_attempts), - do: respondent.stats |> Stats.attempts(:mobileweb) - - defp respondent_stat(respondent, key), do: apply(Stats, key, [respondent.stats]) - def disposition_history(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do project = load_project(conn, project_id) survey = load_survey(project, survey_id) @@ -846,13 +827,6 @@ defmodule AskWeb.RespondentController do %{respondent | phone_number: Respondent.mask_phone_number(respondent.phone_number)} end - defp csv_filename(survey, prefix) do - name = survey.name || "survey_id_#{survey.id}" - name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") - prefix = "#{name}-#{prefix}" - Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) - end - defp load_survey(project, survey_id) do project |> assoc(:surveys) From 45b11812ed0396cece4a52ba49916cf02361e731 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Mon, 5 Aug 2024 12:56:10 -0300 Subject: [PATCH 09/46] Fix router path to point to new get_respondents_result path --- lib/ask_web/controllers/survey_link_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ask_web/controllers/survey_link_controller.ex b/lib/ask_web/controllers/survey_link_controller.ex index 6961390b5..d98a55a95 100644 --- a/lib/ask_web/controllers/survey_link_controller.ex +++ b/lib/ask_web/controllers/survey_link_controller.ex @@ -20,7 +20,7 @@ defmodule AskWeb.SurveyLinkController do "results" -> { Survey.link_name(survey, :results), - project_survey_respondents_results_path(conn, :results, project, survey, %{ + project_survey_get_respondents_results_path(conn, :results, project, survey, %{ "_format" => "csv" }) } From 59f4893ad5dcceeea384702e2301d47842f47240 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Mon, 5 Aug 2024 13:44:09 -0300 Subject: [PATCH 10/46] Remove respondents_where from Survey model --- lib/ask/questionnaire.ex | 10 ++ lib/ask/runtime/survey_files_manager.ex | 144 ++++++++++++++---- lib/ask/survey.ex | 83 ---------- .../controllers/respondent_controller.ex | 14 +- 4 files changed, 123 insertions(+), 128 deletions(-) diff --git a/lib/ask/questionnaire.ex b/lib/ask/questionnaire.ex index d38944b24..e921d1890 100644 --- a/lib/ask/questionnaire.ex +++ b/lib/ask/questionnaire.ex @@ -156,6 +156,16 @@ defmodule Ask.Questionnaire do get_steps(steps) ++ quota_completed_steps end + def all_questionnaires_fields(questionnaires, sanitize) do + fields = + questionnaires + |> Enum.flat_map(&Questionnaire.variables/1) + |> Enum.uniq() + |> Enum.reject(fn s -> String.length(s) == 0 end) + + if sanitize, do: sanitize_fields(fields), else: fields + end + defp get_steps(steps) do result = steps diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/runtime/survey_files_manager.ex index 3aea4308f..b96fede06 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/runtime/survey_files_manager.ex @@ -193,14 +193,13 @@ defmodule Ask.Runtime.SurveyFilesManager do end defp do_generate_file(:respondent_result, survey, filter) do - tz_offset = Survey.timezone_offset(survey) - + questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires - all_fields = all_questionnaires_fields(questionnaires, true) + all_fields = Questionnaire.all_questionnaires_fields(questionnaires, true) has_comparisons = length(survey.comparisons) > 0 - respondents = Ask.Survey.respondents_where(survey, filter) + respondents = survey_respondents_where(survey, filter) stats = survey.mode @@ -303,7 +302,7 @@ defmodule Ask.Runtime.SurveyFilesManager do response = responses |> Enum.filter(fn response -> - response.field_name |> Questionnaire.sanitize_variable_name == field_name + response.field_name |> Questionnaire.sanitize_variable_name() == field_name end) case response do @@ -474,45 +473,68 @@ defmodule Ask.Runtime.SurveyFilesManager do defp questionnaire_name(quiz) do quiz.name || "Untitled questionnaire" end - + defp survey_respondent_questionnaires(survey) do from(q in Questionnaire, - where: - q.id in subquery( - from(r in Respondent, - distinct: true, - select: r.questionnaire_id, - where: r.survey_id == ^survey.id - ) - ) - ) - |> Repo.all() - end - + where: + q.id in subquery( + from(r in Respondent, + distinct: true, + select: r.questionnaire_id, + where: r.survey_id == ^survey.id + ) + ) + ) + |> Repo.all() + end + defp respondent_stat(respondent, :sms_attempts), do: respondent.stats |> Stats.attempts(:sms) defp respondent_stat(respondent, :ivr_attempts), do: respondent.stats |> Stats.attempts(:ivr) - + defp respondent_stat(respondent, :mobileweb_attempts), - do: respondent.stats |> Stats.attempts(:mobileweb) - + do: respondent.stats |> Stats.attempts(:mobileweb) + defp respondent_stat(respondent, key), do: apply(Stats, key, [respondent.stats]) - + # FIXME: duplicated from respondent_controller - defp all_questionnaires_fields(questionnaires, sanitize) do - fields = - questionnaires - |> Enum.flat_map(&Questionnaire.variables/1) - |> Enum.uniq() - |> Enum.reject(fn s -> String.length(s) == 0 end) - - if sanitize, do: sanitize_fields(fields), else: fields - end - + # defp all_questionnaires_fields(questionnaires, sanitize) do + # fields = + # questionnaires + # |> Enum.flat_map(&Questionnaire.variables/1) + # |> Enum.uniq() + # |> Enum.reject(fn s -> String.length(s) == 0 end) + + # if sanitize, do: sanitize_fields(fields), else: fields + # end + # FIXME: duplicated from respondent_controller defp sanitize_fields(fields), do: Enum.map(fields, fn field -> Questionnaire.sanitize_variable_name(field) end) - ## Public API + defp experiment_name(quiz, mode) do + "#{questionnaire_name(quiz)} - #{mode_label(mode)}" + end + + defp mode_label(mode) do + case mode do + ["sms"] -> "SMS" + ["sms", "ivr"] -> "SMS with phone call fallback" + ["sms", "mobileweb"] -> "SMS with Mobile Web fallback" + ["ivr"] -> "Phone call" + ["ivr", "sms"] -> "Phone call with SMS fallback" + ["ivr", "mobileweb"] -> "Phone call with Mobile Web fallback" + ["mobileweb"] -> "Mobile Web" + ["mobileweb", "sms"] -> "Mobile Web with SMS fallback" + ["mobileweb", "ivr"] -> "Mobile Web with phone call fallback" + _ -> "Unknown mode" + end + end + + defp questionnaire_name(quiz) do + quiz.name || "Untitled questionnaire" + end + + ## Public GenServer API def generate_interactions_file(survey_id) do Logger.info("Enqueueing generation of survey (id: #{survey_id}) interaction file") GenServer.cast(server_ref(), {:interactions, survey_id, nil}) @@ -532,4 +554,60 @@ defmodule Ask.Runtime.SurveyFilesManager do Logger.info("Enqueueing generation of survey (id: #{survey_id}) disposition_history file") GenServer.cast(server_ref(), {:respondent_result, survey_id, filters}) end + + ## Public Module + def survey_respondents_where(survey, filter) do + filter_where = RespondentsFilter.filter_where(filter, optimized: true) + + respondents = + Stream.resource( + fn -> 0 end, + fn last_seen_id -> + results = + from(r1 in Respondent, + join: r2 in Respondent, + on: r1.id == r2.id, + where: r2.survey_id == ^survey.id and r2.id > ^last_seen_id, + where: ^filter_where, + order_by: r2.id, + limit: @db_chunk_limit, + preload: [:responses, :respondent_group], + select: r1 + ) + |> Repo.all() + + case List.last(results) do + %{id: last_id} -> {results, last_id} + nil -> {:halt, last_seen_id} + end + end, + fn _ -> [] end + ) + + survey_has_comparisons = length(survey.comparisons) > 0 + questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires + + if survey_has_comparisons do + respondents + |> Stream.map(fn respondent -> + experiment_name = + if respondent.questionnaire_id && respondent.mode do + questionnaire = + questionnaires |> Enum.find(fn q -> q.id == respondent.questionnaire_id end) + + if questionnaire do + experiment_name(questionnaire, respondent.mode) + else + "-" + end + else + "-" + end + + %{respondent | experiment_name: experiment_name} + end) + else + respondents + end + end end diff --git a/lib/ask/survey.ex b/lib/ask/survey.ex index 1d8451376..0d0e15546 100644 --- a/lib/ask/survey.ex +++ b/lib/ask/survey.ex @@ -29,7 +29,6 @@ defmodule Ask.Survey do @max_int 2_147_483_647 @default_fallback_delay 120 - @db_chunk_limit 10_000 schema "surveys" do field :name, :string @@ -838,86 +837,4 @@ defmodule Ask.Survey do survey = Repo.preload(survey, panel_survey: :waves) Enum.count(survey.panel_survey.waves) == 1 end - - # FIXME: remove from survey - defp experiment_name(quiz, mode) do - "#{questionnaire_name(quiz)} - #{mode_label(mode)}" - end - - # FIXME: remove from survey - defp mode_label(mode) do - case mode do - ["sms"] -> "SMS" - ["sms", "ivr"] -> "SMS with phone call fallback" - ["sms", "mobileweb"] -> "SMS with Mobile Web fallback" - ["ivr"] -> "Phone call" - ["ivr", "sms"] -> "Phone call with SMS fallback" - ["ivr", "mobileweb"] -> "Phone call with Mobile Web fallback" - ["mobileweb"] -> "Mobile Web" - ["mobileweb", "sms"] -> "Mobile Web with SMS fallback" - ["mobileweb", "ivr"] -> "Mobile Web with phone call fallback" - _ -> "Unknown mode" - end - end - - # FIXME: remove from survey - defp questionnaire_name(quiz) do - quiz.name || "Untitled questionnaire" - end - - # FIXME: remove from survey - def respondents_where(survey, filter) do - filter_where = RespondentsFilter.filter_where(filter, optimized: true) - - respondents = - Stream.resource( - fn -> 0 end, - fn last_seen_id -> - results = - from(r1 in Respondent, - join: r2 in Respondent, - on: r1.id == r2.id, - where: r2.survey_id == ^survey.id and r2.id > ^last_seen_id, - where: ^filter_where, - order_by: r2.id, - limit: @db_chunk_limit, - preload: [:responses, :respondent_group], - select: r1 - ) - |> Repo.all() - - case List.last(results) do - %{id: last_id} -> {results, last_id} - nil -> {:halt, last_seen_id} - end - end, - fn _ -> [] end - ) - - survey_has_comparisons = length(survey.comparisons) > 0 - questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires - - if survey_has_comparisons do - respondents - |> Stream.map(fn respondent -> - experiment_name = - if respondent.questionnaire_id && respondent.mode do - questionnaire = - questionnaires |> Enum.find(fn q -> q.id == respondent.questionnaire_id end) - - if questionnaire do - experiment_name(questionnaire, respondent.mode) - else - "-" - end - else - "-" - end - - %{respondent | experiment_name: experiment_name} - end) - else - respondents - end - end end diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 07d7246db..c9636b30f 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -88,7 +88,7 @@ defmodule AskWeb.RespondentController do defp index_fields_for_render("response" = field_type, questionnaires) do order_alphabetically = &(String.downcase(&1) < String.downcase(&2)) - all_questionnaires_fields(questionnaires) + Questionnaire.all_questionnaires_fields(questionnaires) |> Enum.sort(&order_alphabetically.(&1, &2)) |> map_fields_with_type(field_type) end @@ -712,7 +712,7 @@ defmodule AskWeb.RespondentController do filter = add_params_to_filter(filter, params) # filter_where = RespondentsFilter.filter_where(filter, optimized: true) - respondents = Ask.Survey.respondents_where(survey, filter) + respondents = SurveyFilesManager.survey_respondents_where(survey, filter) partial_relevant_enabled = Survey.partial_relevant_enabled?(survey, true) @@ -748,16 +748,6 @@ defmodule AskWeb.RespondentController do conn |> send_resp(200, "OK") end - defp all_questionnaires_fields(questionnaires, sanitize \\ false) do - fields = - questionnaires - |> Enum.flat_map(&Questionnaire.variables/1) - |> Enum.uniq() - |> Enum.reject(fn s -> String.length(s) == 0 end) - - if sanitize, do: sanitize_fields(fields), else: fields - end - defp sanitize_fields(fields), do: Enum.map(fields, fn field -> Questionnaire.sanitize_variable_name(field) end) From bba89be2f5c0db473508cd17c46e56ae2458e6ba Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Mon, 5 Aug 2024 13:50:49 -0300 Subject: [PATCH 11/46] Rename Ask.Runtime.SurveyFilesManager to Ask.SurveyResults --- lib/ask.ex | 2 +- .../survey_files_manager.ex => survey_results.ex} | 4 ++-- lib/ask_web/controllers/respondent_controller.ex | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) rename lib/ask/{runtime/survey_files_manager.ex => survey_results.ex} (99%) diff --git a/lib/ask.ex b/lib/ask.ex index 36e7de1bb..12756f070 100644 --- a/lib/ask.ex +++ b/lib/ask.ex @@ -43,7 +43,7 @@ defmodule Ask do worker(Ask.Runtime.ChannelStatusServer, []), worker(Ask.Config, []), worker(Ask.Runtime.QuestionnaireSimulatorStore, []), - worker(Ask.Runtime.SurveyFilesManager, []) + worker(Ask.SurveyResults, []) | children ] ++ [ # SurveyCancellerSupervisor depends on Ask.Repo, so must be started (and declared!) after it diff --git a/lib/ask/runtime/survey_files_manager.ex b/lib/ask/survey_results.ex similarity index 99% rename from lib/ask/runtime/survey_files_manager.ex rename to lib/ask/survey_results.ex index b96fede06..8ce9d5157 100644 --- a/lib/ask/runtime/survey_files_manager.ex +++ b/lib/ask/survey_results.ex @@ -6,7 +6,7 @@ ## When a process is hibernated it will continue the loop once a message is in its ## message queue. If when returning a handle_* call there is already a message ## in the message queue, the process will continue the loop immediately. -defmodule Ask.Runtime.SurveyFilesManager do +defmodule Ask.SurveyResults do use GenServer require Ask.RespondentStats @@ -35,7 +35,7 @@ defmodule Ask.Runtime.SurveyFilesManager do @impl true def init(state) do - Logger.info("SurveyFilesManager started") + Logger.info("SurveyResults started") {:ok, state} end diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index c9636b30f..217b24057 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -12,7 +12,7 @@ defmodule AskWeb.RespondentController do RespondentsFilter } - alias Ask.Runtime.SurveyFilesManager + alias Ask.SurveyResults def index(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do limit = Map.get(params, "limit", "") @@ -712,7 +712,7 @@ defmodule AskWeb.RespondentController do filter = add_params_to_filter(filter, params) # filter_where = RespondentsFilter.filter_where(filter, optimized: true) - respondents = SurveyFilesManager.survey_respondents_where(survey, filter) + respondents = SurveyResults.survey_respondents_where(survey, filter) partial_relevant_enabled = Survey.partial_relevant_enabled?(survey, true) @@ -741,7 +741,7 @@ defmodule AskWeb.RespondentController do # ?param1=value is more specific than ?q=param1:value filter = add_params_to_filter(filter, params) - SurveyFilesManager.generate_respondent_result_file(survey_id, filter) + SurveyResults.generate_respondent_result_file(survey_id, filter) ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() @@ -778,7 +778,7 @@ defmodule AskWeb.RespondentController do # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "disposition_history") |> Repo.insert() - SurveyFilesManager.generate_disposition_history_file(survey_id) + SurveyResults.generate_disposition_history_file(survey_id) conn |> send_resp(200, "OK") end @@ -797,7 +797,7 @@ defmodule AskWeb.RespondentController do # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() - SurveyFilesManager.generate_incentives_file(survey_id) + SurveyResults.generate_incentives_file(survey_id) conn |> send_resp(200, "OK") end @@ -809,7 +809,7 @@ defmodule AskWeb.RespondentController do # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "interactions") |> Repo.insert() - SurveyFilesManager.generate_interactions_file(survey_id) + SurveyResults.generate_interactions_file(survey_id) conn |> send_resp(200, "OK") end From 65e6f12e2ca1872ac36755a572a5dea58e3246c3 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Mon, 5 Aug 2024 14:20:21 -0300 Subject: [PATCH 12/46] Move all_questionnaires_fields to SurveyResults instead of Questionnaire --- lib/ask/questionnaire.ex | 12 ---- lib/ask/survey_results.ex | 56 ++++++------------- .../controllers/respondent_controller.ex | 6 +- 3 files changed, 18 insertions(+), 56 deletions(-) diff --git a/lib/ask/questionnaire.ex b/lib/ask/questionnaire.ex index e921d1890..1ffb1f1e4 100644 --- a/lib/ask/questionnaire.ex +++ b/lib/ask/questionnaire.ex @@ -146,8 +146,6 @@ defmodule Ask.Questionnaire do nil end - def sanitize_variable_name(s), do: s |> String.trim() |> String.replace(" ", "_") - def all_steps(%Questionnaire{steps: steps, quota_completed_steps: nil}) do get_steps(steps) end @@ -156,16 +154,6 @@ defmodule Ask.Questionnaire do get_steps(steps) ++ quota_completed_steps end - def all_questionnaires_fields(questionnaires, sanitize) do - fields = - questionnaires - |> Enum.flat_map(&Questionnaire.variables/1) - |> Enum.uniq() - |> Enum.reject(fn s -> String.length(s) == 0 end) - - if sanitize, do: sanitize_fields(fields), else: fields - end - defp get_steps(steps) do result = steps diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index 8ce9d5157..b438e8929 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -16,6 +16,7 @@ defmodule Ask.SurveyResults do Repo, Respondent, RespondentDispositionHistory, + RespondentsFilter, Stats, Survey, SurveyLogEntry @@ -196,7 +197,7 @@ defmodule Ask.SurveyResults do tz_offset = Survey.timezone_offset(survey) questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires - all_fields = Questionnaire.all_questionnaires_fields(questionnaires, true) + all_fields = all_questionnaires_fields(questionnaires, true) has_comparisons = length(survey.comparisons) > 0 respondents = survey_respondents_where(survey, filter) @@ -302,7 +303,7 @@ defmodule Ask.SurveyResults do response = responses |> Enum.filter(fn response -> - response.field_name |> Questionnaire.sanitize_variable_name() == field_name + response.field_name |> sanitize_variable_name() == field_name end) case response do @@ -451,29 +452,6 @@ defmodule Ask.SurveyResults do Ask.TimeUtil.format(dt, tz_offset_in_seconds, tz_offset) end - defp experiment_name(quiz, mode) do - "#{questionnaire_name(quiz)} - #{mode_label(mode)}" - end - - defp mode_label(mode) do - case mode do - ["sms"] -> "SMS" - ["sms", "ivr"] -> "SMS with phone call fallback" - ["sms", "mobileweb"] -> "SMS with Mobile Web fallback" - ["ivr"] -> "Phone call" - ["ivr", "sms"] -> "Phone call with SMS fallback" - ["ivr", "mobileweb"] -> "Phone call with Mobile Web fallback" - ["mobileweb"] -> "Mobile Web" - ["mobileweb", "sms"] -> "Mobile Web with SMS fallback" - ["mobileweb", "ivr"] -> "Mobile Web with phone call fallback" - _ -> "Unknown mode" - end - end - - defp questionnaire_name(quiz) do - quiz.name || "Untitled questionnaire" - end - defp survey_respondent_questionnaires(survey) do from(q in Questionnaire, where: @@ -496,20 +474,10 @@ defmodule Ask.SurveyResults do defp respondent_stat(respondent, key), do: apply(Stats, key, [respondent.stats]) - # FIXME: duplicated from respondent_controller - # defp all_questionnaires_fields(questionnaires, sanitize) do - # fields = - # questionnaires - # |> Enum.flat_map(&Questionnaire.variables/1) - # |> Enum.uniq() - # |> Enum.reject(fn s -> String.length(s) == 0 end) - - # if sanitize, do: sanitize_fields(fields), else: fields - # end - - # FIXME: duplicated from respondent_controller - defp sanitize_fields(fields), - do: Enum.map(fields, fn field -> Questionnaire.sanitize_variable_name(field) end) + def sanitize_variable_name(variable), do: variable |> String.trim() |> String.replace(" ", "_") + + defp sanitize_variable_names(fields), + do: Enum.map(fields, &sanitize_variable_name/1) defp experiment_name(quiz, mode) do "#{questionnaire_name(quiz)} - #{mode_label(mode)}" @@ -610,4 +578,14 @@ defmodule Ask.SurveyResults do respondents end end + + def all_questionnaires_fields(questionnaires, sanitize \\ false) do + fields = + questionnaires + |> Enum.flat_map(&Questionnaire.variables/1) + |> Enum.uniq() + |> Enum.reject(fn s -> String.length(s) == 0 end) + + if sanitize, do: sanitize_variable_names(fields), else: fields + end end diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 217b24057..2a38f1551 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -6,7 +6,6 @@ defmodule AskWeb.RespondentController do ActivityLog, CompletedRespondents, Logger, - Questionnaire, Respondent, Survey, RespondentsFilter @@ -88,7 +87,7 @@ defmodule AskWeb.RespondentController do defp index_fields_for_render("response" = field_type, questionnaires) do order_alphabetically = &(String.downcase(&1) < String.downcase(&2)) - Questionnaire.all_questionnaires_fields(questionnaires) + SurveyResults.all_questionnaires_fields(questionnaires) |> Enum.sort(&order_alphabetically.(&1, &2)) |> map_fields_with_type(field_type) end @@ -748,9 +747,6 @@ defmodule AskWeb.RespondentController do conn |> send_resp(200, "OK") end - defp sanitize_fields(fields), - do: Enum.map(fields, fn field -> Questionnaire.sanitize_variable_name(field) end) - defp add_params_to_filter(filter, params) do filter = if params["disposition"], From 20171149f3dab9aeb47057f671a0b4c4040a4f05 Mon Sep 17 00:00:00 2001 From: Ana Perez Ghiglia Date: Mon, 5 Aug 2024 15:20:28 -0300 Subject: [PATCH 13/46] remove unused import --- lib/ask/survey.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ask/survey.ex b/lib/ask/survey.ex index 0d0e15546..bdf2101bf 100644 --- a/lib/ask/survey.ex +++ b/lib/ask/survey.ex @@ -18,7 +18,6 @@ defmodule Ask.Survey do FloipEndpoint, Folder, RespondentStats, - RespondentsFilter, ConfigHelper, SystemTime, PanelSurvey From 12643658d4535e6c76fae1f12b099ff2cd49da57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 13 Aug 2024 16:53:49 -0300 Subject: [PATCH 14/46] Scaffold Survey Results tests See #2350 --- lib/ask/survey_results.ex | 12 +++++++++--- test/ask/survey_results_test.exs | 12 ++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 test/ask/survey_results_test.exs diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index b438e8929..bf55acc4a 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -359,10 +359,14 @@ defmodule Ask.SurveyResults do defp do_generate_file(file_type, _, _), do: Logger.warn("No function for generating #{file_type} files") - defp write_to_file(file_type, survey, rows) do + def file_path(survey, file_type) do filename = csv_filename(survey, file_prefix(file_type)) + "#{@target_dir}/#{filename}" + end + + defp write_to_file(file_type, survey, rows) do File.mkdir_p!(@target_dir) - file = File.open!("#{@target_dir}/#{filename}", [:write, :utf8]) + file = File.open!(file_path(survey, file_type), [:write, :utf8]) initial_datetime = Timex.now() rows @@ -382,8 +386,10 @@ defmodule Ask.SurveyResults do defp file_prefix(:respondent_result), do: "respondents" defp file_prefix(_), do: "" - defp should_generate_file(:interactions, survey) do + # FIXME: we probably don't need to check if we should generate the file + defp should_generate_file(:xxxx_interactions, survey) do # TODO: when do we want to skip the re-generation of the file? + File.mkdir_p!(@target_dir) # ensure the directory exists existing_files = File.ls!(@target_dir) exists_file = diff --git a/test/ask/survey_results_test.exs b/test/ask/survey_results_test.exs new file mode 100644 index 000000000..e0f8a6f3f --- /dev/null +++ b/test/ask/survey_results_test.exs @@ -0,0 +1,12 @@ +defmodule Ask.SurveyResultsTest do + use Ask.DataCase + + alias Ask.SurveyResults + + test "generates empty interactions file" do + survey = insert(:survey) + assert {:noreply, _, _} = SurveyResults.handle_cast({:interactions, survey.id, nil}, nil) + path = SurveyResults.file_path(survey, :interactions) + assert "ID,Respondent ID,Mode,Channel,Disposition,Action Type,Action Data,Timestamp\r\n" == File.read!(path) + end +end From 346e8661c5b97d87311e3d4e1769fdfe3ab0fcab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 14 Aug 2024 11:21:26 -0300 Subject: [PATCH 15/46] More tests --- test/ask/survey_results_test.exs | 131 +++++++++++++++++- .../respondent_controller_test.exs | 77 ++-------- 2 files changed, 141 insertions(+), 67 deletions(-) diff --git a/test/ask/survey_results_test.exs b/test/ask/survey_results_test.exs index e0f8a6f3f..cff17bdfb 100644 --- a/test/ask/survey_results_test.exs +++ b/test/ask/survey_results_test.exs @@ -1,7 +1,21 @@ defmodule Ask.SurveyResultsTest do use Ask.DataCase - alias Ask.SurveyResults + alias Ask.{ + SurveyLogEntry, + SurveyResults, + } + + defp cast!(str) do + case DateTime.from_iso8601(str) do + {:ok, datetime, _offset} -> datetime + {:error, x} -> {:error, x} + end + end + + defp completed_schedule() do + Ask.Schedule.always() + end test "generates empty interactions file" do survey = insert(:survey) @@ -9,4 +23,119 @@ defmodule Ask.SurveyResultsTest do path = SurveyResults.file_path(survey, :interactions) assert "ID,Respondent ID,Mode,Channel,Disposition,Action Type,Action Data,Timestamp\r\n" == File.read!(path) end + + test "generates interactions with data" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() + ) + + channel_1 = insert(:channel, name: "test_channel_ivr", type: "ivr") + group_1 = insert(:respondent_group, survey: survey) + insert(:respondent_group_channel, respondent_group: group_1, channel: channel_1, mode: "ivr") + + channel_2 = insert(:channel, name: "test_channel_sms", type: "sms") + group_2 = insert(:respondent_group, survey: survey) + insert(:respondent_group_channel, respondent_group: group_2, channel: channel_2, mode: "sms") + + channel_3 = insert(:channel, name: "test_channel_mobile_web", type: "mobileweb") + group_3 = insert(:respondent_group, survey: survey) + insert(:respondent_group_channel, respondent_group: group_3, channel: channel_3, mode: "mobileweb") + + respondent_1 = insert(:respondent, survey: survey, hashed_number: "1234", respondent_group: group_1) + respondent_2 = insert(:respondent, survey: survey, hashed_number: "5678", respondent_group: group_2) + respondent_3 = insert(:respondent, survey: survey, hashed_number: "8901", respondent_group: group_3) + + for _ <- 1..200 do + insert(:survey_log_entry, + survey: survey, + mode: "ivr", + respondent: respondent_1, + respondent_hashed_number: "1234", + channel: nil, + disposition: "partial", + action_type: "contact", + action_data: "explanation", + timestamp: cast!("2000-01-01T02:03:04Z") + ) + + insert(:survey_log_entry, + survey: survey, + mode: "sms", + respondent: respondent_2, + respondent_hashed_number: "5678", + channel: channel_2, + disposition: "completed", + action_type: "prompt", + action_data: "explanation", + timestamp: cast!("2000-01-01T01:02:03Z") + ) + + insert(:survey_log_entry, + survey: survey, + mode: "mobileweb", + respondent: respondent_3, + respondent_hashed_number: "8901", + channel: channel_3, + disposition: "partial", + action_type: "contact", + action_data: "explanation", + timestamp: cast!("2000-01-01T03:04:05Z") + ) + end + + assert {:noreply, _, _} = SurveyResults.handle_cast({:interactions, survey.id, nil}, nil) + + respondent_1_interactions_ids = + Repo.all( + from entry in SurveyLogEntry, + where: entry.respondent_id == ^respondent_1.id, + order_by: entry.id, + select: entry.id + ) + + respondent_2_interactions_ids = + Repo.all( + from entry in SurveyLogEntry, + where: entry.respondent_id == ^respondent_2.id, + order_by: entry.id, + select: entry.id + ) + respondent_3_interactions_ids = + Repo.all( + from entry in SurveyLogEntry, + where: entry.respondent_id == ^respondent_3.id, + order_by: entry.id, + select: entry.id + ) + + expected_list = + List.flatten([ + "ID,Respondent ID,Mode,Channel,Disposition,Action Type,Action Data,Timestamp", + for i <- 0..199 do + interaction_id = respondent_1_interactions_ids |> Enum.at(i) + "#{interaction_id},1234,IVR,,Partial,Contact attempt,explanation,2000-01-01 02:03:04 UTC" + end, + for i <- 0..199 do + interaction_id_sms = respondent_2_interactions_ids |> Enum.at(i) + "#{interaction_id_sms},5678,SMS,test_channel_sms,Completed,Prompt,explanation,2000-01-01 01:02:03 UTC" + end, + for i <- 0..199 do + interaction_id_web = respondent_3_interactions_ids |> Enum.at(i) + "#{interaction_id_web},8901,Mobile Web,test_channel_mobile_web,Partial,Contact attempt,explanation,2000-01-01 03:04:05 UTC" + end + ]) + + path = SurveyResults.file_path(survey, :interactions) + lines = File.read!(path) |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) + assert length(lines) == length(expected_list) + assert lines == expected_list + end end diff --git a/test/ask_web/controllers/respondent_controller_test.exs b/test/ask_web/controllers/respondent_controller_test.exs index c07b639f5..abdccb825 100644 --- a/test/ask_web/controllers/respondent_controller_test.exs +++ b/test/ask_web/controllers/respondent_controller_test.exs @@ -3834,7 +3834,10 @@ defmodule AskWeb.RespondentControllerTest do ] end + @tag :skip test "download interactions", %{conn: conn, user: user} do + # FIXME: we probably don't need this much data - just setup a survey, create a link + # and check it returns a CSV with the header project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project) @@ -3847,28 +3850,17 @@ defmodule AskWeb.RespondentControllerTest do schedule: completed_schedule() ) - channel_1 = insert(:channel, name: "test_channel_ivr", type: "ivr") - group_1 = insert(:respondent_group, survey: survey) - insert(:respondent_group_channel, respondent_group: group_1, channel: channel_1, mode: "ivr") + channel = insert(:channel, name: "test_channel_ivr", type: "ivr") + group = insert(:respondent_group, survey: survey) + insert(:respondent_group_channel, respondent_group: group, channel: channel, mode: "ivr") - channel_2 = insert(:channel, name: "test_channel_sms", type: "sms") - group_2 = insert(:respondent_group, survey: survey) - insert(:respondent_group_channel, respondent_group: group_2, channel: channel_2, mode: "sms") - - channel_3 = insert(:channel, name: "test_channel_mobile_web", type: "mobileweb") - group_3 = insert(:respondent_group, survey: survey) - insert(:respondent_group_channel, respondent_group: group_3, channel: channel_3, mode: "mobileweb") - - - respondent_1 = insert(:respondent, survey: survey, hashed_number: "1234", respondent_group: group_1) - respondent_2 = insert(:respondent, survey: survey, hashed_number: "5678", respondent_group: group_2) - respondent_3 = insert(:respondent, survey: survey, hashed_number: "8901", respondent_group: group_3) + respondent = insert(:respondent, survey: survey, hashed_number: "1234", respondent_group: group) for _ <- 1..200 do insert(:survey_log_entry, survey: survey, mode: "ivr", - respondent: respondent_1, + respondent: respondent, respondent_hashed_number: "1234", channel: nil, disposition: "partial", @@ -3876,30 +3868,6 @@ defmodule AskWeb.RespondentControllerTest do action_data: "explanation", timestamp: cast!("2000-01-01T02:03:04Z") ) - - insert(:survey_log_entry, - survey: survey, - mode: "sms", - respondent: respondent_2, - respondent_hashed_number: "5678", - channel: channel_2, - disposition: "completed", - action_type: "prompt", - action_data: "explanation", - timestamp: cast!("2000-01-01T01:02:03Z") - ) - - insert(:survey_log_entry, - survey: survey, - mode: "mobileweb", - respondent: respondent_3, - respondent_hashed_number: "8901", - channel: channel_3, - disposition: "partial", - action_type: "contact", - action_data: "explanation", - timestamp: cast!("2000-01-01T03:04:05Z") - ) end conn = @@ -3916,25 +3884,10 @@ defmodule AskWeb.RespondentControllerTest do csv = response(conn, 200) - respondent_1_interactions_ids = + respondent_interactions_ids = Repo.all( from entry in SurveyLogEntry, - where: entry.respondent_id == ^respondent_1.id, - order_by: entry.id, - select: entry.id - ) - - respondent_2_interactions_ids = - Repo.all( - from entry in SurveyLogEntry, - where: entry.respondent_id == ^respondent_2.id, - order_by: entry.id, - select: entry.id - ) - respondent_3_interactions_ids = - Repo.all( - from entry in SurveyLogEntry, - where: entry.respondent_id == ^respondent_3.id, + where: entry.respondent_id == ^respondent.id, order_by: entry.id, select: entry.id ) @@ -3943,17 +3896,9 @@ defmodule AskWeb.RespondentControllerTest do List.flatten([ "ID,Respondent ID,Mode,Channel,Disposition,Action Type,Action Data,Timestamp", for i <- 0..199 do - interaction_id = respondent_1_interactions_ids |> Enum.at(i) + interaction_id = respondent_interactions_ids |> Enum.at(i) "#{interaction_id},1234,IVR,,Partial,Contact attempt,explanation,2000-01-01 02:03:04 UTC" end, - for i <- 0..199 do - interaction_id_sms = respondent_2_interactions_ids |> Enum.at(i) - "#{interaction_id_sms},5678,SMS,test_channel_sms,Completed,Prompt,explanation,2000-01-01 01:02:03 UTC" - end, - for i <- 0..199 do - interaction_id_web = respondent_3_interactions_ids |> Enum.at(i) - "#{interaction_id_web},8901,Mobile Web,test_channel_mobile_web,Partial,Contact attempt,explanation,2000-01-01 03:04:05 UTC" - end ]) lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) From 2244b16c9db5a971813d04e8d7df525939f38cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 14 Aug 2024 20:49:58 -0300 Subject: [PATCH 16/46] Keep moving tests See #2350 --- test/ask/survey_results_test.exs | 135 ++++++++++++++++++ .../respondent_controller_test.exs | 30 ++-- 2 files changed, 144 insertions(+), 21 deletions(-) diff --git a/test/ask/survey_results_test.exs b/test/ask/survey_results_test.exs index cff17bdfb..1d4f6b755 100644 --- a/test/ask/survey_results_test.exs +++ b/test/ask/survey_results_test.exs @@ -1,7 +1,10 @@ defmodule Ask.SurveyResultsTest do use Ask.DataCase + use Ask.DummySteps alias Ask.{ + RespondentsFilter, + Stats, SurveyLogEntry, SurveyResults, } @@ -138,4 +141,136 @@ defmodule Ask.SurveyResultsTest do assert length(lines) == length(expected_list) assert lines == expected_list end + + test "generates results csv" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule(), + mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1, + stats: %Stats{ + total_received_sms: 4, + total_sent_sms: 3, + total_call_time_seconds: 12, + call_durations: %{"call-3" => 45}, + attempts: %{sms: 1, mobileweb: 2, ivr: 3}, + pending_call: false + } + ) + + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") + insert(:response, respondent: respondent_1, field_name: "Perfect Number", value: "100") + group_2 = insert(:respondent_group) + + respondent_2 = + insert(:respondent, + survey: survey, + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"], + respondent_group: group_2, + stats: %Stats{total_sent_sms: 1}, + user_stopped: true + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, line3, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" + + [ + line_2_hashed_number, + line_2_disp, + _, + line_2_modes, + line_2_user_stopped, + line_2_total_sent_sms, + line_2_total_received_sms, + line_2_sms_attempts, + line_2_total_call_time, + line_2_ivr_attempts, + line_2_mobileweb_attempts, + line_2_section_order, + line_2_respondent_group, + line_2_smoke, + line_2_exercises, + line_2_perfect_number, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_modes == "SMS, Phone call" + assert line_2_respondent_group == group_1.name + assert line_2_smoke == "Yes" + assert line_2_exercises == "No" + assert line_2_disp == "Partial" + assert line_2_total_sent_sms == "3" + assert line_2_total_received_sms == "4" + assert line_2_total_call_time == "0m 57s" + assert line_2_perfect_number == "100" + assert line_2_section_order == "" + assert line_2_sms_attempts == "1" + assert line_2_mobileweb_attempts == "2" + assert line_2_ivr_attempts == "3" + assert line_2_user_stopped == "false" + + [ + line_3_hashed_number, + line_3_disp, + _, + line_3_modes, + line_3_user_stopped, + line_3_total_sent_sms, + line_3_total_received_sms, + line_3_sms_attempts, + line_3_total_call_time, + line_3_ivr_attempts, + line_3_mobileweb_attempts, + line_3_section_order, + line_3_respondent_group, + line_3_smoke, + line_3_exercises, + _, + _ + ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_3_hashed_number == respondent_2.hashed_number + assert line_3_modes == "Mobile Web" + assert line_3_respondent_group == group_2.name + assert line_3_smoke == "No" + assert line_3_exercises == "" + assert line_3_disp == "Registered" + assert line_3_total_sent_sms == "1" + assert line_3_total_received_sms == "0" + assert line_3_total_call_time == "0m 0s" + assert line_3_section_order == "" + assert line_3_sms_attempts == "0" + assert line_3_mobileweb_attempts == "0" + assert line_3_ivr_attempts == "0" + assert line_3_user_stopped == "true" + end end diff --git a/test/ask_web/controllers/respondent_controller_test.exs b/test/ask_web/controllers/respondent_controller_test.exs index abdccb825..86ca3a128 100644 --- a/test/ask_web/controllers/respondent_controller_test.exs +++ b/test/ask_web/controllers/respondent_controller_test.exs @@ -2215,6 +2215,7 @@ defmodule AskWeb.RespondentControllerTest do describe "download" do setup :user + @tag :skip test "download results csv", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) @@ -2229,15 +2230,15 @@ defmodule AskWeb.RespondentControllerTest do mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] ) - group_1 = insert(:respondent_group) + group = insert(:respondent_group) - respondent_1 = + respondent = insert(:respondent, survey: survey, hashed_number: "1asd12451eds", disposition: "partial", effective_modes: ["sms", "ivr"], - respondent_group: group_1, + respondent_group: group, stats: %Stats{ total_received_sms: 4, total_sent_sms: 3, @@ -2248,22 +2249,9 @@ defmodule AskWeb.RespondentControllerTest do } ) - insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") - insert(:response, respondent: respondent_1, field_name: "Perfect Number", value: "100") - group_2 = insert(:respondent_group) - - respondent_2 = - insert(:respondent, - survey: survey, - hashed_number: "34y5345tjyet", - effective_modes: ["mobileweb"], - respondent_group: group_2, - stats: %Stats{total_sent_sms: 1}, - user_stopped: true - ) - - insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + insert(:response, respondent: respondent, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent, field_name: "Exercises", value: "No") + insert(:response, respondent: respondent, field_name: "Perfect Number", value: "100") conn = get( @@ -2301,9 +2289,9 @@ defmodule AskWeb.RespondentControllerTest do _ ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_hashed_number == respondent.hashed_number assert line_2_modes == "SMS, Phone call" - assert line_2_respondent_group == group_1.name + assert line_2_respondent_group == group.name assert line_2_smoke == "Yes" assert line_2_exercises == "No" assert line_2_disp == "Partial" From 2cc060524c8bd818c18c87ade6dfd80b0f0ca7a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 15 Aug 2024 19:37:17 -0300 Subject: [PATCH 17/46] Make test suite "pass" by skipping broken tests We should un-skip them by the end of the PR. See #2350 --- lib/ask_web/router.ex | 8 +- test/ask/survey_results_test.exs | 974 +++++++++++++++ .../respondent_controller_test.exs | 1082 ++--------------- .../survey_link_controller_test.exs | 9 + 4 files changed, 1114 insertions(+), 959 deletions(-) diff --git a/lib/ask_web/router.ex b/lib/ask_web/router.ex index f2f115746..5080cda73 100644 --- a/lib/ask_web/router.ex +++ b/lib/ask_web/router.ex @@ -194,14 +194,14 @@ defmodule AskWeb.Router do get "/results", RespondentController, :results, as: :get_respondents_results - post "/results", RespondentController, :trigger_results, as: :respondents_results + post "/results", RespondentController, :generate_results, as: :respondents_results - post "/disposition_history", RespondentController, :disposition_history, + post "/disposition_history", RespondentController, :generate_disposition_history, as: :respondents_disposition_history - post "/incentives", RespondentController, :incentives, as: :respondents_incentives + post "/incentives", RespondentController, :generate_incentives, as: :respondents_incentives - post "/interactions", RespondentController, :interactions, + post "/interactions", RespondentController, :generate_interactions, as: :respondents_interactions end end diff --git a/test/ask/survey_results_test.exs b/test/ask/survey_results_test.exs index 1d4f6b755..b5963ff90 100644 --- a/test/ask/survey_results_test.exs +++ b/test/ask/survey_results_test.exs @@ -273,4 +273,978 @@ defmodule Ask.SurveyResultsTest do assert line_3_ivr_attempts == "0" assert line_3_user_stopped == "true" end + + test "download results csv with non-started last call" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule(), + mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] + ) + + group = insert(:respondent_group) + + respondent = + insert(:respondent, + survey: survey, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group, + stats: %Stats{ + total_received_sms: 4, + total_sent_sms: 3, + total_call_time_seconds: 12, + call_durations: %{"call-3" => 45}, + attempts: %{sms: 1, mobileweb: 2, ivr: 3}, + pending_call: true + } + ) + + insert(:response, respondent: respondent, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent, field_name: "Exercises", value: "No") + insert(:response, respondent: respondent, field_name: "Perfect Number", value: "100") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" + + line_2_ivr_attempts = + [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd |> Enum.at(9) + + assert line_2_ivr_attempts == "2" + end + + test "download results csv with sections" do + project = insert(:project) + + questionnaire = + insert(:questionnaire, name: "test", project: project, steps: @three_sections) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule(), + mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + questionnaire: questionnaire, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1, + section_order: [0, 1, 2], + stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} + ) + + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Refresh", value: "No") + insert(:response, respondent: respondent_1, field_name: "Perfect_Number", value: "4") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") + group_2 = insert(:respondent_group) + + respondent_2 = + insert(:respondent, + survey: survey, + questionnaire: questionnaire, + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"], + respondent_group: group_2, + section_order: [2, 1, 0], + stats: %Stats{total_sent_sms: 1} + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, line3, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Refresh,Probability,Last,Perfect_Number,Question" + + [ + line_2_hashed_number, + line_2_disp, + _, + line_2_modes, + _, + line_2_total_sent_sms, + line_2_total_received_sms, + _, + line_2_total_call_time, + _, + _, + line_2_section_order, + line_2_respondent_group, + line_2_smoke, + line_2_exercises, + line_2_refresh, + _, + _, + line_2_perfect_number, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_modes == "SMS, Phone call" + assert line_2_respondent_group == group_1.name + assert line_2_smoke == "Yes" + assert line_2_exercises == "No" + assert line_2_perfect_number == "4" + assert line_2_refresh == "No" + assert line_2_disp == "Partial" + assert line_2_total_sent_sms == "3" + assert line_2_total_received_sms == "4" + assert line_2_total_call_time == "12m 0s" + assert line_2_section_order == "First section, Second section, Third section" + + [ + line_3_hashed_number, + line_3_disp, + _, + line_3_modes, + _, + line_3_total_sent_sms, + line_3_total_received_sms, + _, + line_3_total_call_time, + _, + _, + line_3_section_order, + line_3_respondent_group, + line_3_smoke, + line_3_exercises, + line_3_refresh, + _, + _, + _, + _ + ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_3_hashed_number == respondent_2.hashed_number + assert line_3_modes == "Mobile Web" + assert line_3_respondent_group == group_2.name + assert line_3_smoke == "No" + assert line_3_exercises == "" + assert line_3_refresh == "" + assert line_3_disp == "Registered" + assert line_3_total_sent_sms == "1" + assert line_3_total_received_sms == "0" + assert line_3_total_call_time == "0m 0s" + assert line_3_section_order == "Third section, Second section, First section" + end + + test "download results csv with untitled sections" do + project = insert(:project) + + questionnaire = + insert(:questionnaire, name: "test", project: project, steps: @three_sections_untitled) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule(), + mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + questionnaire: questionnaire, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1, + section_order: [0, 1, 2], + stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} + ) + + group_2 = insert(:respondent_group) + + respondent_2 = + insert(:respondent, + survey: survey, + questionnaire: questionnaire, + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"], + respondent_group: group_2, + section_order: [2, 1, 0], + stats: %Stats{total_sent_sms: 1} + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, line3, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Refresh,Probability,Last,Perfect_Number,Question" + + [ + line_2_hashed_number, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + line_2_section_order, + _, + _, + _, + _, + _, + _, + _, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_section_order == "Untitled 1, Second section, Untitled 3" + + [ + line_3_hashed_number, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + line_3_section_order, + _, + _, + _, + _, + _, + _, + _, + _ + ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_3_hashed_number == respondent_2.hashed_number + assert line_3_section_order == "Untitled 3, Second section, Untitled 1" + end + + test "download results csv with filter by disposition" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1 + ) + + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") + + respondent_2 = + insert(:respondent, + survey: survey, + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"] + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{disposition: :registered}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" + + [ + line_2_hashed_number, + line_2_disp, + _, + line_2_modes, + _, + _, + _, + _, + _, + line_2_respondent_group, + line_2_smoke, + line_2_exercises, + _, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_2.hashed_number + assert line_2_modes == "Mobile Web" + assert line_2_respondent_group == group_1.name + assert line_2_smoke == "No" + assert line_2_exercises == "" + assert line_2_disp == "Registered" + end + + test "download results csv with filter by update timestamp" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1 + ) + + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") + group_2 = insert(:respondent_group) + + respondent_2 = + insert(:respondent, + survey: survey, + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"], + respondent_group: group_2, + updated_at: Timex.shift(Timex.now(), hours: 2, minutes: 3) + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{since: Timex.shift(Timex.now(), hours: 2)}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" + + [ + line_2_hashed_number, + line_2_disp, + _, + line_2_modes, + _, + _, + _, + _, + _, + line_2_respondent_group, + line_2_smoke, + line_2_exercises, + _, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_2.hashed_number + assert line_2_modes == "Mobile Web" + assert line_2_respondent_group == group_1.name + assert line_2_smoke == "No" + assert line_2_exercises == "" + assert line_2_disp == "Registered" + end + + test "download results csv with filter by final state" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1, + state: "completed" + ) + + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") + + respondent_2 = + insert(:respondent, + survey: survey, + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"] + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{state: :completed}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" + + [ + line_2_hashed_number, + line_2_disp, + _, + line_2_modes, + _, + _, + _, + _, + _, + line_2_respondent_group, + line_2_smoke, + line_2_exercises, + _, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_modes == "SMS, Phone call" + assert line_2_respondent_group == group_1.name + assert line_2_smoke == "Yes" + assert line_2_exercises == "No" + assert line_2_disp == "Partial" + end + + test "download results csv with sample file column and two different respondent groups" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule(), + mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] + ) + + group_1 = insert(:respondent_group, name: "respondent_group_1_example.csv") + + respondent_1 = + insert(:respondent, + survey: survey, + hashed_number: "1asd12451eds", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1, + stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} + ) + + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") + group_2 = insert(:respondent_group, name: "respondent_group_2_example.csv") + + respondent_2 = + insert(:respondent, + survey: survey, + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"], + respondent_group: group_2, + stats: %Stats{total_sent_sms: 1} + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + respondent_3 = + insert(:respondent, + survey: survey, + hashed_number: "1hsd13451ftj", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_1, + stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} + ) + + insert(:response, respondent: respondent_3, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_3, field_name: "Exercises", value: "No") + + respondent_4 = + insert(:respondent, + survey: survey, + hashed_number: "67y5634tjsdfg", + disposition: "partial", + effective_modes: ["sms", "ivr"], + respondent_group: group_2, + stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} + ) + + insert(:response, respondent: respondent_4, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_4, field_name: "Exercises", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + assert !String.contains?(group_1.name, [" ", ",", "*", ":", "?", "\\", "|", "/", "<", ">"]) + assert !String.contains?(group_2.name, [" ", ",", "*", ":", "?", "\\", "|", "/", "<", ">"]) + + [line1, line2, line3, line4, line5, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" + + [ + line_2_hashed_number, + _, + _, + line_2_modes, + _, + _, + _, + _, + _, + _, + _, + _, + line_2_respondent_group, + _, + _, + _, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_modes == "SMS, Phone call" + assert line_2_respondent_group == group_1.name + + [ + line_3_hashed_number, + _, + _, + line_3_modes, + _, + _, + _, + _, + _, + _, + _, + _, + line_3_respondent_group, + _, + _, + _, + _ + ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_3_hashed_number == respondent_2.hashed_number + assert line_3_modes == "Mobile Web" + assert line_3_respondent_group == group_2.name + + [ + line_4_hashed_number, + _, + _, + line_4_modes, + _, + _, + _, + _, + _, + _, + _, + _, + line_4_respondent_group, + _, + _, + _, + _ + ] = [line4] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_4_hashed_number == respondent_3.hashed_number + assert line_4_modes == "SMS, Phone call" + assert line_4_respondent_group == group_1.name + + [ + line_5_hashed_number, + _, + _, + line_5_modes, + _, + _, + _, + _, + _, + _, + _, + _, + line_5_respondent_group, + _, + _, + _, + _ + ] = [line5] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_5_hashed_number == respondent_4.hashed_number + assert line_5_modes == "SMS, Phone call" + assert line_5_respondent_group == group_2.name + end + + test "download results csv with comparisons" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) + + questionnaire2 = + insert(:questionnaire, name: "test 2", project: project, steps: @dummy_steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire, questionnaire2], + state: :ready, + schedule: completed_schedule(), + comparisons: [ + %{"mode" => ["sms"], "questionnaire_id" => questionnaire.id, "ratio" => 50}, + %{"mode" => ["sms"], "questionnaire_id" => questionnaire2.id, "ratio" => 50} + ] + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + questionnaire_id: questionnaire.id, + mode: ["sms"], + respondent_group: group_1, + disposition: "partial", + stats: %Stats{attempts: %{sms: 2}} + ) + + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Perfect_Number", value: "No") + + respondent_2 = + insert(:respondent, + survey: survey, + questionnaire_id: questionnaire2.id, + mode: ["sms", "ivr"], + respondent_group: group_1, + disposition: "completed", + stats: %Stats{attempts: %{sms: 5}} + ) + + insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, line3, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,variant,Smokes,Exercises,Perfect_Number,Question" + + [ + line_2_hashed_number, + line_2_disp, + _, + _, + _, + _, + _, + line_2_sms_attempts, + _, + _, + line_2_variant, + line_2_smoke, + _, + line_2_number, + _ + ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_1.hashed_number |> to_string + assert line_2_smoke == "Yes" + assert line_2_number == "No" + assert line_2_variant == "test - SMS" + assert line_2_disp == "Partial" + assert line_2_sms_attempts == "2" + + [ + line_3_hashed_number, + line_3_disp, + _, + _, + _, + _, + _, + line_3_sms_attempts, + _, + _, + line_3_variant, + line_3_smoke, + _, + line_3_number, + _ + ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_3_hashed_number == respondent_2.hashed_number |> to_string + assert line_3_smoke == "No" + assert line_3_number == "" + assert line_3_variant == "test 2 - SMS with phone call fallback" + assert line_3_disp == "Completed" + assert line_3_sms_attempts == "5" + end + + test "download csv with language" do + languageStep = %{ + "id" => "1234-5678", + "type" => "language-selection", + "title" => "Language selection", + "store" => "language", + "prompt" => %{ + "sms" => "1 for English, 2 for Spanish", + "ivr" => %{ + "text" => "1 para ingles, 2 para español", + "audioSource" => "tts" + } + }, + "language_choices" => ["en", "es"] + } + + steps = [languageStep] + + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: steps) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() + ) + + group_1 = insert(:respondent_group) + + respondent_1 = + insert(:respondent, + survey: survey, + hashed_number: "1asd12451eds", + disposition: "partial", + respondent_group: group_1 + ) + + insert(:response, respondent: respondent_1, field_name: "language", value: "es") + + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + path = SurveyResults.file_path(survey, :respondent_result) + csv = File.read!(path) + + [line1, line2, _] = csv |> String.split("\r\n") + + assert line1 == + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,language" + + [line_2_hashed_number, _, _, _, _, _, _, _, _, _, line_2_language] = + [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd + + assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_language == "español" + end + + test "download disposition history csv" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() + ) + + respondent_1 = + insert(:respondent, survey: survey, hashed_number: "1asd12451eds", disposition: "partial") + + respondent_2 = insert(:respondent, survey: survey, hashed_number: "34y5345tjyet") + + insert(:respondent_disposition_history, + survey: survey, + respondent: respondent_1, + respondent_hashed_number: respondent_1.hashed_number, + disposition: "partial", + mode: "sms", + inserted_at: cast!("2000-01-01T01:02:03Z") + ) + + insert(:respondent_disposition_history, + survey: survey, + respondent: respondent_1, + respondent_hashed_number: respondent_1.hashed_number, + disposition: "completed", + mode: "sms", + inserted_at: cast!("2000-01-01T02:03:04Z") + ) + + insert(:respondent_disposition_history, + survey: survey, + respondent: respondent_2, + respondent_hashed_number: respondent_2.hashed_number, + disposition: "partial", + mode: "ivr", + inserted_at: cast!("2000-01-01 03:04:05Z") + ) + + insert(:respondent_disposition_history, + survey: survey, + respondent: respondent_2, + respondent_hashed_number: respondent_2.hashed_number, + disposition: "completed", + mode: "ivr", + inserted_at: cast!("2000-01-01 04:05:06Z") + ) + + assert {:noreply, _, _} = SurveyResults.handle_cast({:disposition_history, survey.id, %RespondentsFilter{}}, nil) + path = SurveyResults.file_path(survey, :disposition_history) + csv = File.read!(path) + + lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) + + assert lines == [ + "Respondent ID,Disposition,Mode,Timestamp", + "1asd12451eds,partial,SMS,2000-01-01 01:02:03 UTC", + "1asd12451eds,completed,SMS,2000-01-01 02:03:04 UTC", + "34y5345tjyet,partial,Phone call,2000-01-01 03:04:05 UTC", + "34y5345tjyet,completed,Phone call,2000-01-01 04:05:06 UTC" + ] + end + + test "download incentives" do + project = insert(:project) + questionnaire = insert(:questionnaire, name: "test", project: project) + + survey = + insert(:survey, + project: project, + cutoff: 4, + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() + ) + + completed_at = cast!("2019-11-10T09:00:00Z") + + insert(:respondent, + survey: survey, + phone_number: "1234", + disposition: "partial", + questionnaire_id: questionnaire.id, + mode: ["sms"] + ) + + insert(:respondent, + survey: survey, + phone_number: "5678", + disposition: "completed", + questionnaire_id: questionnaire.id, + mode: ["sms", "ivr"], + completed_at: completed_at + ) + + insert(:respondent, + survey: survey, + phone_number: "9012", + disposition: "completed", + mode: ["sms", "ivr"] + ) + + insert(:respondent, + survey: survey, + phone_number: "4321", + disposition: "completed", + questionnaire_id: questionnaire.id, + mode: ["ivr"] + ) + + assert {:noreply, _, _} = SurveyResults.handle_cast({:incentives, survey.id, %RespondentsFilter{}}, nil) + path = SurveyResults.file_path(survey, :incentives) + csv = File.read!(path) + + lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) + + assert lines == [ + "Telephone number,Questionnaire-Mode,Completion date", + "5678,test - SMS with phone call fallback,2019-11-10 09:00:00 UTC", + "4321,test - Phone call," + ] + end end diff --git a/test/ask_web/controllers/respondent_controller_test.exs b/test/ask_web/controllers/respondent_controller_test.exs index 86ca3a128..97408c250 100644 --- a/test/ask_web/controllers/respondent_controller_test.exs +++ b/test/ask_web/controllers/respondent_controller_test.exs @@ -1809,6 +1809,7 @@ defmodule AskWeb.RespondentControllerTest do assert_partial_relevant_index_respondent(respondents, 0, 2) end + @tag :skip test "CSV", %{ conn: conn, survey: survey, @@ -1996,6 +1997,7 @@ defmodule AskWeb.RespondentControllerTest do assert_partial_relevant_index_respondent(respondents, 1, 0) end + @tag :skip test "CSV", %{ conn: conn, survey: survey, @@ -2135,6 +2137,7 @@ defmodule AskWeb.RespondentControllerTest do refute_partial_relevant_index_field(fields, expected_field_index_on_index) end + @tag :skip test "CSV", %{ conn: conn, survey: survey, @@ -2183,6 +2186,7 @@ defmodule AskWeb.RespondentControllerTest do assert_partial_relevant_index_respondent(respondents, 1, 0) end + @tag :skip test "CSV", %{ conn: conn, survey: survey, @@ -2264,7 +2268,7 @@ defmodule AskWeb.RespondentControllerTest do csv = response(conn, 200) - [line1, line2, line3, _] = csv |> String.split("\r\n") + [line1, line2, _] = csv |> String.split("\r\n") assert line1 == "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" @@ -2304,44 +2308,11 @@ defmodule AskWeb.RespondentControllerTest do assert line_2_mobileweb_attempts == "2" assert line_2_ivr_attempts == "3" assert line_2_user_stopped == "false" - - [ - line_3_hashed_number, - line_3_disp, - _, - line_3_modes, - line_3_user_stopped, - line_3_total_sent_sms, - line_3_total_received_sms, - line_3_sms_attempts, - line_3_total_call_time, - line_3_ivr_attempts, - line_3_mobileweb_attempts, - line_3_section_order, - line_3_respondent_group, - line_3_smoke, - line_3_exercises, - _, - _ - ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_3_hashed_number == respondent_2.hashed_number - assert line_3_modes == "Mobile Web" - assert line_3_respondent_group == group_2.name - assert line_3_smoke == "No" - assert line_3_exercises == "" - assert line_3_disp == "Registered" - assert line_3_total_sent_sms == "1" - assert line_3_total_received_sms == "0" - assert line_3_total_call_time == "0m 0s" - assert line_3_section_order == "" - assert line_3_sms_attempts == "0" - assert line_3_mobileweb_attempts == "0" - assert line_3_ivr_attempts == "0" - assert line_3_user_stopped == "true" end - test "download results csv with non-started last call", %{conn: conn, user: user} do + @tag :skip + test "download results csv with filter by disposition", %{conn: conn, user: user} do + # FIXME: should only check that the file is correctly generated with the filter project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) @@ -2351,69 +2322,7 @@ defmodule AskWeb.RespondentControllerTest do cutoff: 4, questionnaires: [questionnaire], state: :ready, - schedule: completed_schedule(), - mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] - ) - - group = insert(:respondent_group) - - respondent = - insert(:respondent, - survey: survey, - hashed_number: "1asd12451eds", - disposition: "partial", - effective_modes: ["sms", "ivr"], - respondent_group: group, - stats: %Stats{ - total_received_sms: 4, - total_sent_sms: 3, - total_call_time_seconds: 12, - call_durations: %{"call-3" => 45}, - attempts: %{sms: 1, mobileweb: 2, ivr: 3}, - pending_call: true - } - ) - - insert(:response, respondent: respondent, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent, field_name: "Exercises", value: "No") - insert(:response, respondent: respondent, field_name: "Perfect Number", value: "100") - - conn = - get( - conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ - "offset" => "0", - "_format" => "csv" - }) - ) - - csv = response(conn, 200) - - [line1, line2, _] = csv |> String.split("\r\n") - - assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" - - line_2_ivr_attempts = - [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd |> Enum.at(9) - - assert line_2_ivr_attempts == "2" - end - - test "download results csv with sections", %{conn: conn, user: user} do - project = create_project_for_user(user) - - questionnaire = - insert(:questionnaire, name: "test", project: project, steps: @three_sections) - - survey = - insert(:survey, - project: project, - cutoff: 4, - questionnaires: [questionnaire], - state: :ready, - schedule: completed_schedule(), - mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] + schedule: completed_schedule() ) group_1 = insert(:respondent_group) @@ -2421,30 +2330,20 @@ defmodule AskWeb.RespondentControllerTest do respondent_1 = insert(:respondent, survey: survey, - questionnaire: questionnaire, hashed_number: "1asd12451eds", disposition: "partial", effective_modes: ["sms", "ivr"], - respondent_group: group_1, - section_order: [0, 1, 2], - stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} + respondent_group: group_1 ) insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_1, field_name: "Refresh", value: "No") - insert(:response, respondent: respondent_1, field_name: "Perfect_Number", value: "4") insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") - group_2 = insert(:respondent_group) respondent_2 = insert(:respondent, survey: survey, - questionnaire: questionnaire, hashed_number: "34y5345tjyet", - effective_modes: ["mobileweb"], - respondent_group: group_2, - section_order: [2, 1, 0], - stats: %Stats{total_sent_sms: 1} + effective_modes: ["mobileweb"] ) insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") @@ -2454,16 +2353,17 @@ defmodule AskWeb.RespondentControllerTest do conn, project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ "offset" => "0", - "_format" => "csv" + "_format" => "csv", + "disposition" => "registered" }) ) csv = response(conn, 200) - [line1, line2, line3, _] = csv |> String.split("\r\n") + [line1, line2, _] = csv |> String.split("\r\n") assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Refresh,Probability,Last,Perfect_Number,Question" + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" [ line_2_hashed_number, @@ -2471,77 +2371,29 @@ defmodule AskWeb.RespondentControllerTest do _, line_2_modes, _, - line_2_total_sent_sms, - line_2_total_received_sms, _, - line_2_total_call_time, _, _, - line_2_section_order, + _, line_2_respondent_group, line_2_smoke, line_2_exercises, - line_2_refresh, - _, _, - line_2_perfect_number, _ ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - assert line_2_hashed_number == respondent_1.hashed_number - assert line_2_modes == "SMS, Phone call" + assert line_2_hashed_number == respondent_2.hashed_number + assert line_2_modes == "Mobile Web" assert line_2_respondent_group == group_1.name - assert line_2_smoke == "Yes" - assert line_2_exercises == "No" - assert line_2_perfect_number == "4" - assert line_2_refresh == "No" - assert line_2_disp == "Partial" - assert line_2_total_sent_sms == "3" - assert line_2_total_received_sms == "4" - assert line_2_total_call_time == "12m 0s" - assert line_2_section_order == "First section, Second section, Third section" - - [ - line_3_hashed_number, - line_3_disp, - _, - line_3_modes, - _, - line_3_total_sent_sms, - line_3_total_received_sms, - _, - line_3_total_call_time, - _, - _, - line_3_section_order, - line_3_respondent_group, - line_3_smoke, - line_3_exercises, - line_3_refresh, - _, - _, - _, - _ - ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_3_hashed_number == respondent_2.hashed_number - assert line_3_modes == "Mobile Web" - assert line_3_respondent_group == group_2.name - assert line_3_smoke == "No" - assert line_3_exercises == "" - assert line_3_refresh == "" - assert line_3_disp == "Registered" - assert line_3_total_sent_sms == "1" - assert line_3_total_received_sms == "0" - assert line_3_total_call_time == "0m 0s" - assert line_3_section_order == "Third section, Second section, First section" + assert line_2_smoke == "No" + assert line_2_exercises == "" + assert line_2_disp == "Registered" end - test "download results csv with untitled sections", %{conn: conn, user: user} do + @tag :skip + test "download results csv with filter by update timestamp", %{conn: conn, user: user} do project = create_project_for_user(user) - - questionnaire = - insert(:questionnaire, name: "test", project: project, steps: @three_sections_untitled) + questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) survey = insert(:survey, @@ -2549,8 +2401,7 @@ defmodule AskWeb.RespondentControllerTest do cutoff: 4, questionnaires: [questionnaire], state: :ready, - schedule: completed_schedule(), - mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] + schedule: completed_schedule() ) group_1 = insert(:respondent_group) @@ -2558,26 +2409,23 @@ defmodule AskWeb.RespondentControllerTest do respondent_1 = insert(:respondent, survey: survey, - questionnaire: questionnaire, hashed_number: "1asd12451eds", disposition: "partial", effective_modes: ["sms", "ivr"], - respondent_group: group_1, - section_order: [0, 1, 2], - stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} + respondent_group: group_1 ) + insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") group_2 = insert(:respondent_group) respondent_2 = insert(:respondent, survey: survey, - questionnaire: questionnaire, hashed_number: "34y5345tjyet", effective_modes: ["mobileweb"], respondent_group: group_2, - section_order: [2, 1, 0], - stats: %Stats{total_sent_sms: 1} + updated_at: Timex.shift(Timex.now(), hours: 2, minutes: 3) ) insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") @@ -2587,71 +2435,46 @@ defmodule AskWeb.RespondentControllerTest do conn, project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ "offset" => "0", - "_format" => "csv" + "_format" => "csv", + "since" => Timex.format!(Timex.shift(Timex.now(), hours: 2), "%FT%T%:z", :strftime) }) ) csv = response(conn, 200) - [line1, line2, line3, _] = csv |> String.split("\r\n") + [line1, line2, _] = csv |> String.split("\r\n") assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Refresh,Probability,Last,Perfect_Number,Question" + "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" [ line_2_hashed_number, + line_2_disp, _, + line_2_modes, _, _, _, _, _, - _, - _, - _, - _, - line_2_section_order, - _, - _, - _, - _, - _, - _, + line_2_respondent_group, + line_2_smoke, + line_2_exercises, _, _ ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - assert line_2_hashed_number == respondent_1.hashed_number - assert line_2_section_order == "Untitled 1, Second section, Untitled 3" - - [ - line_3_hashed_number, - _, - _, - _, - _, - _, - _, - _, - _, - _, - _, - line_3_section_order, - _, - _, - _, - _, - _, - _, - _, - _ - ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_3_hashed_number == respondent_2.hashed_number - assert line_3_section_order == "Untitled 3, Second section, Untitled 1" + assert line_2_hashed_number == respondent_2.hashed_number + assert line_2_modes == "Mobile Web" + assert line_2_respondent_group == group_1.name + assert line_2_smoke == "No" + assert line_2_exercises == "" + assert line_2_disp == "Registered" end - test "download results csv with filter by disposition", %{conn: conn, user: user} do + @tag :skip + test "download results csv with filter by final state", %{conn: conn, user: user} do + # FIXME: check that triggering the generation of this file sends the correct filter project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) @@ -2672,7 +2495,8 @@ defmodule AskWeb.RespondentControllerTest do hashed_number: "1asd12451eds", disposition: "partial", effective_modes: ["sms", "ivr"], - respondent_group: group_1 + respondent_group: group_1, + state: "completed" ) insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") @@ -2693,7 +2517,7 @@ defmodule AskWeb.RespondentControllerTest do project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ "offset" => "0", "_format" => "csv", - "disposition" => "registered" + "final" => true }) ) @@ -2721,15 +2545,15 @@ defmodule AskWeb.RespondentControllerTest do _ ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - assert line_2_hashed_number == respondent_2.hashed_number - assert line_2_modes == "Mobile Web" + assert line_2_hashed_number == respondent_1.hashed_number + assert line_2_modes == "SMS, Phone call" assert line_2_respondent_group == group_1.name - assert line_2_smoke == "No" - assert line_2_exercises == "" - assert line_2_disp == "Registered" + assert line_2_smoke == "Yes" + assert line_2_exercises == "No" + assert line_2_disp == "Partial" end - test "download results csv with filter by update timestamp", %{conn: conn, user: user} do + test "download results json", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) @@ -2742,387 +2566,41 @@ defmodule AskWeb.RespondentControllerTest do schedule: completed_schedule() ) - group_1 = insert(:respondent_group) - respondent_1 = insert(:respondent, survey: survey, hashed_number: "1asd12451eds", disposition: "partial", effective_modes: ["sms", "ivr"], - respondent_group: group_1 + questionnaire_id: questionnaire.id ) + respondent_1 = Repo.get(Respondent, respondent_1.id) insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") - group_2 = insert(:respondent_group) + + response_1 = + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") + + response_1 = Repo.get(Response, response_1.id) respondent_2 = insert(:respondent, survey: survey, hashed_number: "34y5345tjyet", effective_modes: ["mobileweb"], - respondent_group: group_2, - updated_at: Timex.shift(Timex.now(), hours: 2, minutes: 3) + questionnaire_id: questionnaire.id ) - insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + respondent_2 = Repo.get(Respondent, respondent_2.id) + response_2 = insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + response_2 = Repo.get(Response, response_2.id) conn = get( conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ + project_survey_get_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ "offset" => "0", - "_format" => "csv", - "since" => Timex.format!(Timex.shift(Timex.now(), hours: 2), "%FT%T%:z", :strftime) - }) - ) - - csv = response(conn, 200) - - [line1, line2, _] = csv |> String.split("\r\n") - - assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" - - [ - line_2_hashed_number, - line_2_disp, - _, - line_2_modes, - _, - _, - _, - _, - _, - line_2_respondent_group, - line_2_smoke, - line_2_exercises, - _, - _ - ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_2_hashed_number == respondent_2.hashed_number - assert line_2_modes == "Mobile Web" - assert line_2_respondent_group == group_1.name - assert line_2_smoke == "No" - assert line_2_exercises == "" - assert line_2_disp == "Registered" - end - - test "download results csv with filter by final state", %{conn: conn, user: user} do - project = create_project_for_user(user) - questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) - - survey = - insert(:survey, - project: project, - cutoff: 4, - questionnaires: [questionnaire], - state: :ready, - schedule: completed_schedule() - ) - - group_1 = insert(:respondent_group) - - respondent_1 = - insert(:respondent, - survey: survey, - hashed_number: "1asd12451eds", - disposition: "partial", - effective_modes: ["sms", "ivr"], - respondent_group: group_1, - state: "completed" - ) - - insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") - - respondent_2 = - insert(:respondent, - survey: survey, - hashed_number: "34y5345tjyet", - effective_modes: ["mobileweb"] - ) - - insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - - conn = - get( - conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ - "offset" => "0", - "_format" => "csv", - "final" => true - }) - ) - - csv = response(conn, 200) - - [line1, line2, _] = csv |> String.split("\r\n") - - assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" - - [ - line_2_hashed_number, - line_2_disp, - _, - line_2_modes, - _, - _, - _, - _, - _, - line_2_respondent_group, - line_2_smoke, - line_2_exercises, - _, - _ - ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_2_hashed_number == respondent_1.hashed_number - assert line_2_modes == "SMS, Phone call" - assert line_2_respondent_group == group_1.name - assert line_2_smoke == "Yes" - assert line_2_exercises == "No" - assert line_2_disp == "Partial" - end - - test "download results csv with sample file column and two different respondent groups", %{ - conn: conn, - user: user - } do - project = create_project_for_user(user) - questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) - - survey = - insert(:survey, - project: project, - cutoff: 4, - questionnaires: [questionnaire], - state: :ready, - schedule: completed_schedule(), - mode: [["sms", "ivr"], ["mobileweb"], ["sms", "mobileweb"]] - ) - - group_1 = insert(:respondent_group, name: "respondent_group_1_example.csv") - - respondent_1 = - insert(:respondent, - survey: survey, - hashed_number: "1asd12451eds", - disposition: "partial", - effective_modes: ["sms", "ivr"], - respondent_group: group_1, - stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} - ) - - insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") - group_2 = insert(:respondent_group, name: "respondent_group_2_example.csv") - - respondent_2 = - insert(:respondent, - survey: survey, - hashed_number: "34y5345tjyet", - effective_modes: ["mobileweb"], - respondent_group: group_2, - stats: %Stats{total_sent_sms: 1} - ) - - insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - - respondent_3 = - insert(:respondent, - survey: survey, - hashed_number: "1hsd13451ftj", - disposition: "partial", - effective_modes: ["sms", "ivr"], - respondent_group: group_1, - stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} - ) - - insert(:response, respondent: respondent_3, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_3, field_name: "Exercises", value: "No") - - respondent_4 = - insert(:respondent, - survey: survey, - hashed_number: "67y5634tjsdfg", - disposition: "partial", - effective_modes: ["sms", "ivr"], - respondent_group: group_2, - stats: %Stats{total_received_sms: 4, total_sent_sms: 3, total_call_time: 12} - ) - - insert(:response, respondent: respondent_4, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_4, field_name: "Exercises", value: "No") - - conn = - get( - conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ - "offset" => "0", - "_format" => "csv" - }) - ) - - csv = response(conn, 200) - - assert !String.contains?(group_1.name, [" ", ",", "*", ":", "?", "\\", "|", "/", "<", ">"]) - assert !String.contains?(group_2.name, [" ", ",", "*", ":", "?", "\\", "|", "/", "<", ">"]) - - [line1, line2, line3, line4, line5, _] = csv |> String.split("\r\n") - - assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,total_call_time,ivr_attempts,mobileweb_attempts,section_order,sample_file,Smokes,Exercises,Perfect_Number,Question" - - [ - line_2_hashed_number, - _, - _, - line_2_modes, - _, - _, - _, - _, - _, - _, - _, - _, - line_2_respondent_group, - _, - _, - _, - _ - ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_2_hashed_number == respondent_1.hashed_number - assert line_2_modes == "SMS, Phone call" - assert line_2_respondent_group == group_1.name - - [ - line_3_hashed_number, - _, - _, - line_3_modes, - _, - _, - _, - _, - _, - _, - _, - _, - line_3_respondent_group, - _, - _, - _, - _ - ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_3_hashed_number == respondent_2.hashed_number - assert line_3_modes == "Mobile Web" - assert line_3_respondent_group == group_2.name - - [ - line_4_hashed_number, - _, - _, - line_4_modes, - _, - _, - _, - _, - _, - _, - _, - _, - line_4_respondent_group, - _, - _, - _, - _ - ] = [line4] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_4_hashed_number == respondent_3.hashed_number - assert line_4_modes == "SMS, Phone call" - assert line_4_respondent_group == group_1.name - - [ - line_5_hashed_number, - _, - _, - line_5_modes, - _, - _, - _, - _, - _, - _, - _, - _, - line_5_respondent_group, - _, - _, - _, - _ - ] = [line5] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_5_hashed_number == respondent_4.hashed_number - assert line_5_modes == "SMS, Phone call" - assert line_5_respondent_group == group_2.name - end - - test "download results json", %{conn: conn, user: user} do - project = create_project_for_user(user) - questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) - - survey = - insert(:survey, - project: project, - cutoff: 4, - questionnaires: [questionnaire], - state: :ready, - schedule: completed_schedule() - ) - - respondent_1 = - insert(:respondent, - survey: survey, - hashed_number: "1asd12451eds", - disposition: "partial", - effective_modes: ["sms", "ivr"], - questionnaire_id: questionnaire.id - ) - - respondent_1 = Repo.get(Respondent, respondent_1.id) - insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - - response_1 = - insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") - - response_1 = Repo.get(Response, response_1.id) - - respondent_2 = - insert(:respondent, - survey: survey, - hashed_number: "34y5345tjyet", - effective_modes: ["mobileweb"], - questionnaire_id: questionnaire.id - ) - - respondent_2 = Repo.get(Respondent, respondent_2.id) - response_2 = insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - response_2 = Repo.get(Response, response_2.id) - - conn = - get( - conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ - "offset" => "0", - "_format" => "json" + "_format" => "json" }) ) @@ -3214,7 +2692,7 @@ defmodule AskWeb.RespondentControllerTest do conn = get( conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ + project_survey_get_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ "offset" => "0", "_format" => "json", "disposition" => "partial" @@ -3288,7 +2766,7 @@ defmodule AskWeb.RespondentControllerTest do conn = get( conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ + project_survey_get_respondents_results_path(conn, :results, survey.project.id, survey.id , %{ "offset" => "0", "_format" => "json", "since" => Timex.format!(Timex.shift(Timex.now(), hours: 2), "%FT%T%:z", :strftime) @@ -3325,180 +2803,66 @@ defmodule AskWeb.RespondentControllerTest do insert(:survey, project: project, cutoff: 4, - questionnaires: [questionnaire], - state: :ready, - schedule: completed_schedule() - ) - - respondent_1 = - insert(:respondent, - survey: survey, - hashed_number: "1asd12451eds", - disposition: "partial", - effective_modes: ["sms", "ivr"], - questionnaire_id: questionnaire.id - ) - - insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") - - respondent_2 = - insert(:respondent, - survey: survey, - hashed_number: "34y5345tjyet", - effective_modes: ["mobileweb"], - questionnaire_id: questionnaire.id, - state: "completed" - ) - - respondent_2 = Repo.get(Respondent, respondent_2.id) - response_2 = insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - response_2 = Repo.get(Response, response_2.id) - - conn = - get( - conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ - "offset" => "0", - "_format" => "json", - "final" => true - }) - ) - - assert json_response(conn, 200)["data"]["respondents"] == [ - %{ - "id" => respondent_2.id, - "phone_number" => respondent_2.hashed_number, - "survey_id" => survey.id, - "mode" => nil, - "effective_modes" => ["mobileweb"], - "questionnaire_id" => questionnaire.id, - "disposition" => "registered", - "date" => DateTime.to_iso8601(response_2.updated_at), - "updated_at" => DateTime.to_iso8601(respondent_2.updated_at), - "responses" => [ - %{ - "value" => "No", - "name" => "Smokes" - } - ], - "stats" => @empty_stats - } - ] - end - - test "download results csv with comparisons", %{conn: conn, user: user} do - project = create_project_for_user(user) - questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) - - questionnaire2 = - insert(:questionnaire, name: "test 2", project: project, steps: @dummy_steps) - - survey = - insert(:survey, - project: project, - cutoff: 4, - questionnaires: [questionnaire, questionnaire2], - state: :ready, - schedule: completed_schedule(), - comparisons: [ - %{"mode" => ["sms"], "questionnaire_id" => questionnaire.id, "ratio" => 50}, - %{"mode" => ["sms"], "questionnaire_id" => questionnaire2.id, "ratio" => 50} - ] + questionnaires: [questionnaire], + state: :ready, + schedule: completed_schedule() ) - group_1 = insert(:respondent_group) - respondent_1 = insert(:respondent, survey: survey, - questionnaire_id: questionnaire.id, - mode: ["sms"], - respondent_group: group_1, + hashed_number: "1asd12451eds", disposition: "partial", - stats: %Stats{attempts: %{sms: 2}} + effective_modes: ["sms", "ivr"], + questionnaire_id: questionnaire.id ) insert(:response, respondent: respondent_1, field_name: "Smokes", value: "Yes") - insert(:response, respondent: respondent_1, field_name: "Perfect_Number", value: "No") + insert(:response, respondent: respondent_1, field_name: "Exercises", value: "No") respondent_2 = insert(:respondent, survey: survey, - questionnaire_id: questionnaire2.id, - mode: ["sms", "ivr"], - respondent_group: group_1, - disposition: "completed", - stats: %Stats{attempts: %{sms: 5}} + hashed_number: "34y5345tjyet", + effective_modes: ["mobileweb"], + questionnaire_id: questionnaire.id, + state: "completed" ) - insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + respondent_2 = Repo.get(Respondent, respondent_2.id) + response_2 = insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") + response_2 = Repo.get(Response, response_2.id) conn = get( conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ + project_survey_get_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ "offset" => "0", - "_format" => "csv" + "_format" => "json", + "final" => true }) ) - csv = response(conn, 200) - - [line1, line2, line3, _] = csv |> String.split("\r\n") - - assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,variant,Smokes,Exercises,Perfect_Number,Question" - - [ - line_2_hashed_number, - line_2_disp, - _, - _, - _, - _, - _, - line_2_sms_attempts, - _, - _, - line_2_variant, - line_2_smoke, - _, - line_2_number, - _ - ] = [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_2_hashed_number == respondent_1.hashed_number |> to_string - assert line_2_smoke == "Yes" - assert line_2_number == "No" - assert line_2_variant == "test - SMS" - assert line_2_disp == "Partial" - assert line_2_sms_attempts == "2" - - [ - line_3_hashed_number, - line_3_disp, - _, - _, - _, - _, - _, - line_3_sms_attempts, - _, - _, - line_3_variant, - line_3_smoke, - _, - line_3_number, - _ - ] = [line3] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_3_hashed_number == respondent_2.hashed_number |> to_string - assert line_3_smoke == "No" - assert line_3_number == "" - assert line_3_variant == "test 2 - SMS with phone call fallback" - assert line_3_disp == "Completed" - assert line_3_sms_attempts == "5" + assert json_response(conn, 200)["data"]["respondents"] == [ + %{ + "id" => respondent_2.id, + "phone_number" => respondent_2.hashed_number, + "survey_id" => survey.id, + "mode" => nil, + "effective_modes" => ["mobileweb"], + "questionnaire_id" => questionnaire.id, + "disposition" => "registered", + "date" => DateTime.to_iso8601(response_2.updated_at), + "updated_at" => DateTime.to_iso8601(respondent_2.updated_at), + "responses" => [ + %{ + "value" => "No", + "name" => "Smokes" + } + ], + "stats" => @empty_stats + } + ] end test "download results json with comparisons", %{conn: conn, user: user} do @@ -3556,7 +2920,7 @@ defmodule AskWeb.RespondentControllerTest do conn = get( conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ + project_survey_get_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ "offset" => "0", "_format" => "json" }) @@ -3608,151 +2972,9 @@ defmodule AskWeb.RespondentControllerTest do ] end - test "download csv with language", %{conn: conn, user: user} do - languageStep = %{ - "id" => "1234-5678", - "type" => "language-selection", - "title" => "Language selection", - "store" => "language", - "prompt" => %{ - "sms" => "1 for English, 2 for Spanish", - "ivr" => %{ - "text" => "1 para ingles, 2 para español", - "audioSource" => "tts" - } - }, - "language_choices" => ["en", "es"] - } - - steps = [languageStep] - - project = create_project_for_user(user) - questionnaire = insert(:questionnaire, name: "test", project: project, steps: steps) - - survey = - insert(:survey, - project: project, - cutoff: 4, - questionnaires: [questionnaire], - state: :ready, - schedule: completed_schedule() - ) - - group_1 = insert(:respondent_group) - - respondent_1 = - insert(:respondent, - survey: survey, - hashed_number: "1asd12451eds", - disposition: "partial", - respondent_group: group_1 - ) - - insert(:response, respondent: respondent_1, field_name: "language", value: "es") - - conn = - get( - conn, - project_survey_respondents_results_path(conn, :results, survey.project.id, survey.id, %{ - "offset" => "0", - "_format" => "csv" - }) - ) - - csv = response(conn, 200) - - [line1, line2, _] = csv |> String.split("\r\n") - - assert line1 == - "respondent_id,disposition,date,modes,user_stopped,total_sent_sms,total_received_sms,sms_attempts,section_order,sample_file,language" - - [line_2_hashed_number, _, _, _, _, _, _, _, _, _, line_2_language] = - [line2] |> Stream.map(& &1) |> CSV.decode() |> Enum.to_list() |> hd - - assert line_2_hashed_number == respondent_1.hashed_number - assert line_2_language == "español" - end - - test "download disposition history csv", %{conn: conn, user: user} do - project = create_project_for_user(user) - questionnaire = insert(:questionnaire, name: "test", project: project) - - survey = - insert(:survey, - project: project, - cutoff: 4, - questionnaires: [questionnaire], - state: :ready, - schedule: completed_schedule() - ) - - respondent_1 = - insert(:respondent, survey: survey, hashed_number: "1asd12451eds", disposition: "partial") - - respondent_2 = insert(:respondent, survey: survey, hashed_number: "34y5345tjyet") - - insert(:respondent_disposition_history, - survey: survey, - respondent: respondent_1, - respondent_hashed_number: respondent_1.hashed_number, - disposition: "partial", - mode: "sms", - inserted_at: cast!("2000-01-01T01:02:03Z") - ) - - insert(:respondent_disposition_history, - survey: survey, - respondent: respondent_1, - respondent_hashed_number: respondent_1.hashed_number, - disposition: "completed", - mode: "sms", - inserted_at: cast!("2000-01-01T02:03:04Z") - ) - - insert(:respondent_disposition_history, - survey: survey, - respondent: respondent_2, - respondent_hashed_number: respondent_2.hashed_number, - disposition: "partial", - mode: "ivr", - inserted_at: cast!("2000-01-01 03:04:05Z") - ) - - insert(:respondent_disposition_history, - survey: survey, - respondent: respondent_2, - respondent_hashed_number: respondent_2.hashed_number, - disposition: "completed", - mode: "ivr", - inserted_at: cast!("2000-01-01 04:05:06Z") - ) - - conn = - get( - conn, - project_survey_respondents_disposition_history_path( - conn, - :disposition_history, - survey.project.id, - survey.id, - %{"_format" => "csv"} - ) - ) - - csv = response(conn, 200) - - lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) - - assert lines == [ - "Respondent ID,Disposition,Mode,Timestamp", - "1asd12451eds,partial,SMS,2000-01-01 01:02:03 UTC", - "1asd12451eds,completed,SMS,2000-01-01 02:03:04 UTC", - "34y5345tjyet,partial,Phone call,2000-01-01 03:04:05 UTC", - "34y5345tjyet,completed,Phone call,2000-01-01 04:05:06 UTC" - ] - end - + @tag :skip test "download incentives", %{conn: conn, user: user} do + # FIXME: just check we can download the file project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project) @@ -3898,6 +3120,7 @@ defmodule AskWeb.RespondentControllerTest do describe "links" do setup :user + @tag :skip test "download results csv using a download link", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) @@ -4002,6 +3225,7 @@ defmodule AskWeb.RespondentControllerTest do assert line_3_disp == "Registered" end + @tag :skip test "generates log when downloading results csv", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) @@ -4040,6 +3264,7 @@ defmodule AskWeb.RespondentControllerTest do }) end + @tag :skip test "download disposition history using download link", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project) @@ -4122,6 +3347,7 @@ defmodule AskWeb.RespondentControllerTest do ] end + @tag :skip test "generates log when downloading disposition_history csv", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) @@ -4164,6 +3390,7 @@ defmodule AskWeb.RespondentControllerTest do }) end + @tag :skip test "download incentives using download link", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project) @@ -4224,6 +3451,7 @@ defmodule AskWeb.RespondentControllerTest do ] end + @tag :skip test "generates log when downloading incentives csv", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project) @@ -4261,6 +3489,7 @@ defmodule AskWeb.RespondentControllerTest do }) end + @tag :skip test "download interactions using download link", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project) @@ -4274,28 +3503,17 @@ defmodule AskWeb.RespondentControllerTest do schedule: completed_schedule() ) - channel_1 = insert(:channel, name: "test_channel_ivr", type: "ivr") - group_1 = insert(:respondent_group, survey: survey) - insert(:respondent_group_channel, respondent_group: group_1, channel: channel_1, mode: "ivr") - - channel_2 = insert(:channel, name: "test_channel_sms", type: "sms") - group_2 = insert(:respondent_group, survey: survey) - insert(:respondent_group_channel, respondent_group: group_2, channel: channel_2, mode: "sms") - - channel_3 = insert(:channel, name: "test_channel_mobile_web", type: "mobileweb") - group_3 = insert(:respondent_group, survey: survey) - insert(:respondent_group_channel, respondent_group: group_3, channel: channel_3, mode: "mobileweb") - + channel = insert(:channel, name: "test_channel_ivr", type: "ivr") + group = insert(:respondent_group, survey: survey) + insert(:respondent_group_channel, respondent_group: group, channel: channel, mode: "ivr") - respondent_1 = insert(:respondent, survey: survey, hashed_number: "1234", respondent_group: group_1) - respondent_2 = insert(:respondent, survey: survey, hashed_number: "5678", respondent_group: group_2) - respondent_3 = insert(:respondent, survey: survey, hashed_number: "8901", respondent_group: group_3) + respondent = insert(:respondent, survey: survey, hashed_number: "1234", respondent_group: group) for _ <- 1..200 do insert(:survey_log_entry, survey: survey, mode: "ivr", - respondent: respondent_1, + respondent: respondent, respondent_hashed_number: "1234", channel: nil, disposition: "partial", @@ -4303,30 +3521,6 @@ defmodule AskWeb.RespondentControllerTest do action_data: "explanation", timestamp: cast!("2000-01-01T02:03:04Z") ) - - insert(:survey_log_entry, - survey: survey, - mode: "sms", - respondent: respondent_2, - respondent_hashed_number: "5678", - channel: channel_2, - disposition: "completed", - action_type: "prompt", - action_data: "explanation", - timestamp: cast!("2000-01-01T01:02:03Z") - ) - - insert(:survey_log_entry, - survey: survey, - mode: "mobileweb", - respondent: respondent_3, - respondent_hashed_number: "8901", - channel: channel_3, - disposition: "partial", - action_type: "contact", - action_data: "explanation", - timestamp: cast!("2000-01-01T03:04:05Z") - ) end {:ok, link} = @@ -4337,25 +3531,10 @@ defmodule AskWeb.RespondentControllerTest do }) ) - respondent_1_interactions_ids = - Repo.all( - from entry in SurveyLogEntry, - where: entry.respondent_id == ^respondent_1.id, - order_by: entry.id, - select: entry.id - ) - - respondent_2_interactions_ids = - Repo.all( - from entry in SurveyLogEntry, - where: entry.respondent_id == ^respondent_2.id, - order_by: entry.id, - select: entry.id - ) - respondent_3_interactions_ids = + respondent_interactions_ids = Repo.all( from entry in SurveyLogEntry, - where: entry.respondent_id == ^respondent_3.id, + where: entry.respondent_id == ^respondent.id, order_by: entry.id, select: entry.id ) @@ -4367,17 +3546,9 @@ defmodule AskWeb.RespondentControllerTest do List.flatten([ "ID,Respondent ID,Mode,Channel,Disposition,Action Type,Action Data,Timestamp", for i <- 0..199 do - interaction_id = respondent_1_interactions_ids |> Enum.at(i) + interaction_id = respondent_interactions_ids |> Enum.at(i) "#{interaction_id},1234,IVR,,Partial,Contact attempt,explanation,2000-01-01 02:03:04 UTC" end, - for i <- 0..199 do - interaction_id_sms = respondent_2_interactions_ids |> Enum.at(i) - "#{interaction_id_sms},5678,SMS,test_channel_sms,Completed,Prompt,explanation,2000-01-01 01:02:03 UTC" - end, - for i <- 0..199 do - interaction_id_web = respondent_3_interactions_ids |> Enum.at(i) - "#{interaction_id_web},8901,Mobile Web,test_channel_mobile_web,Partial,Contact attempt,explanation,2000-01-01 03:04:05 UTC" - end ]) lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) @@ -4385,6 +3556,7 @@ defmodule AskWeb.RespondentControllerTest do assert lines == expected_list end + @tag :skip test "generates log when downloading interactions csv", %{conn: conn, user: user} do project = create_project_for_user(user) questionnaire = insert(:questionnaire, name: "test", project: project) diff --git a/test/ask_web/controllers/survey_link_controller_test.exs b/test/ask_web/controllers/survey_link_controller_test.exs index 9178f7f5c..d9699923b 100644 --- a/test/ask_web/controllers/survey_link_controller_test.exs +++ b/test/ask_web/controllers/survey_link_controller_test.exs @@ -62,6 +62,7 @@ defmodule AskWeb.SurveyLinkControllerTest do assert [] == ShortLink |> Repo.all() end + @tag :skip test "incentives link generation", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -109,6 +110,7 @@ defmodule AskWeb.SurveyLinkControllerTest do assert [] == ShortLink |> Repo.all() end + @tag :skip test "interactions link generation", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -156,6 +158,7 @@ defmodule AskWeb.SurveyLinkControllerTest do assert [] == ShortLink |> Repo.all() end + @tag :skip test "disposition_history link generation", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -359,6 +362,7 @@ defmodule AskWeb.SurveyLinkControllerTest do end end + @tag :skip test "allows editors to create some links", %{conn: conn, user: user} do project = create_project_for_user(user, level: "editor") survey = insert(:survey, project: project) @@ -383,6 +387,7 @@ defmodule AskWeb.SurveyLinkControllerTest do end end + @tag :skip test "allows editors to refresh some links", %{conn: conn, user: user} do project = create_project_for_user(user, level: "editor") survey = insert(:survey, project: project) @@ -409,6 +414,7 @@ defmodule AskWeb.SurveyLinkControllerTest do end end + @tag :skip test "forbids editor to delete some links", %{conn: conn, user: user} do project = create_project_for_user(user, level: "editor") survey = insert(:survey, project: project) @@ -495,6 +501,7 @@ defmodule AskWeb.SurveyLinkControllerTest do }) end + @tag :skip test "generates logs for incentives link", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -545,6 +552,7 @@ defmodule AskWeb.SurveyLinkControllerTest do }) end + @tag :skip test "generates logs for interactions link", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -595,6 +603,7 @@ defmodule AskWeb.SurveyLinkControllerTest do }) end + @tag :skip test "generates logs for disposition_history link", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) From 744e17a003737c5f38930c0c3069d2cdb0e2bc6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 11 Sep 2024 13:25:25 -0300 Subject: [PATCH 18/46] File downloads UI - first take The info about the generated files is still pending. See #2350 --- assets/css/_buttons.scss | 2 +- assets/css/_card-modal.scss | 81 +++++------ .../respondents/RespondentIndex.jsx | 127 +++++++++--------- locales/template/translation.json | 7 +- 4 files changed, 105 insertions(+), 112 deletions(-) diff --git a/assets/css/_buttons.scss b/assets/css/_buttons.scss index 60dd7661c..f5c4ec48d 100644 --- a/assets/css/_buttons.scss +++ b/assets/css/_buttons.scss @@ -50,7 +50,7 @@ i { @extend .grey-text; font-size: 17px; display: inline-block; - margin-left: 10px; + margin-left: 10px; // TODO: the spacing should always be placed in the container, not in the content cursor: pointer; span { vertical-align: middle; diff --git a/assets/css/_card-modal.scss b/assets/css/_card-modal.scss index 9c546d5d1..8bbf58b20 100644 --- a/assets/css/_card-modal.scss +++ b/assets/css/_card-modal.scss @@ -74,49 +74,11 @@ li.collection-item { border: none; } - li.collection-item.download { - min-height: 6rem; - } - a.download { - @extend .grey-text; - position: relative; - > div { - padding-left: 70px; - max-width: 85%; - &:first-child { - background-color: #999999; - text-align: center; - border-radius: 50%; - height: 45px; - width: 45px; - padding-left: 0; - max-width: initial; - position: absolute; - top: 4px; - left: 5px; - i.material-icons { - @extend .white-text; - font-size: 1.5rem; - line-height: 45px; - } - } - p { - font-size: 1.1rem; - margin: 0.2rem 0; - } - } - &:after { - content: ""; - display: table; - clear: both; - zoom: 1; - } - .button { - background-color: color("grey", "darken-1"); - } - .button[disabled] { - border: 1px rgba(0, 0, 0, 0.1) solid; - background-color: transparent; + li.collection-item .file-section { + padding-left: 70px; + p { + font-size: 1.1rem; + margin: 0.2rem 0; } .title { @extend .black-text; @@ -132,9 +94,31 @@ width: fit-content; } } + a.download { + display: block; + text-align: center; + height: 45px; + width: 45px; + padding-left: 0; + max-width: initial; + i.material-icons { + @extend .black-text; + font-size: 1.5rem; + } + position: absolute; + top: 4px; + left: 5px; + &:after { + content: ""; + display: table; + clear: both; + zoom: 1; + } + } .access-link { padding: 0.5rem 0; - max-width: 85%; + display: flex; + align-items: center; .switch { display: inline-block; vertical-align: middle; @@ -179,6 +163,15 @@ } } } + .file-download { + display: flex; + align-items: center; + gap: 0.5rem; + + .btn-icon-grey { + margin-left: initial; + } + } div.link { @extend .grey-text; > div { diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index ebc16cb5d..6411d04df 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -58,7 +58,7 @@ type Props = { } type State = { - csvType: string, + shownFile: string, } class RespondentIndex extends Component { @@ -74,7 +74,7 @@ class RespondentIndex extends Component { constructor(props) { super(props) - this.state = { csvType: "" } + this.state = { shownFile: null } this.toggleResultsLink = this.toggleResultsLink.bind(this) this.toggleIncentivesLink = this.toggleIncentivesLink.bind(this) this.toggleInteractionsLink = this.toggleInteractionsLink.bind(this) @@ -249,7 +249,7 @@ class RespondentIndex extends Component { ) : ( @@ -260,7 +260,7 @@ class RespondentIndex extends Component { {link.url}
{!project.readOnly ? ( - + refresh @@ -317,8 +317,15 @@ class RespondentIndex extends Component { return numericFields.some((field) => field == filterField) } - downloadItem(id, itemType) { + toggleFile(fileId, event) { + event.preventDefault() + this.setState({ shownFile: this.state.shownFile == fileId ? null : fileId }) + } + + downloadItem(id) { const { t, totalCount, filter } = this.props + const { shownFile } = this.state + const currentFile = shownFile == id let item: ?{ title: String, description: String, @@ -407,41 +414,52 @@ class RespondentIndex extends Component { return null } else { const disabled = item.disabled - const titleDescription = ( -
-

- {item.title} -

-

{item.description}

- {disabled ?

{item.disabledText}

: null} + + const downloadButton = ( +
+ + + get_app + + + {t("Download last generated file")} + + { // (15 min ago + // FIXME: this should be calculated + } + + + refresh + + + { // ) + } +
) return (
  • - {itemType == "file" ? ( - { - if (disabled) return - e.preventDefault() - item && item.onDownload() - }} - > -
    - {disabled ? ( -
    - ) : ( - get_app - )} +
    + this.toggleFile(id, e)}> +
    + {currentFile ? "expand_more" : "chevron_right"} +
    +     +
    +

    + {item.title} +

    +

    {item.description}

    + {disabled ?

    {item.disabledText}

    : null} + { currentFile ? +
    + { item.downloadLink } + { downloadButton }
    - {titleDescription} - - ) : ( -
    {titleDescription}
    - )} - {itemType == "link" ? item.downloadLink : null} + : "" + } +
    • ) } @@ -458,29 +476,22 @@ class RespondentIndex extends Component { return this.onFilterChange(value)} /> } - downloadModal({ itemType }) { + downloadModal() { const { userLevel, t, filter } = this.props const ownerOrAdmin = userLevel == "owner" || userLevel == "admin" - const [title, description] = - itemType == "file" - ? [t("Download CSV"), t("Choose the data you want to download")] - : [ - t("Public links"), - t("Choose the data you want to be able to access through a public link"), - ] return ( - +
    -
    {title}
    -

    {description}

    +
    {t("Download CSV")}
    +

    {t("Choose the data you want to download")}

      - {itemType == "file" && filter ? this.downloadItem("filtered-results", itemType) : null} - {this.downloadItem("results", itemType)} - {this.downloadItem("disposition-history", itemType)} - {ownerOrAdmin ? this.downloadItem("incentives", itemType) : null} - {ownerOrAdmin ? this.downloadItem("interactions", itemType) : null} + {filter ? this.downloadItem("filtered-results") : null} + {this.downloadItem("results")} + {this.downloadItem("disposition-history")} + {ownerOrAdmin ? this.downloadItem("incentives") : null} + {ownerOrAdmin ? this.downloadItem("interactions") : null}
    ) @@ -676,20 +687,8 @@ class RespondentIndex extends Component { __html: "", }} /> - - $(`#downloadCSV-${fileId}`).modal("open")} - /> - $(`#downloadCSV-${linkId}`).modal("open")} - /> - - {this.downloadModal({ itemType: fileId })} - {this.downloadModal({ itemType: linkId })} + $('#downloadCSV').modal("open")}/> + {this.downloadModal()} {this.renderColumnPickerModal()} {this.respondentsFilter()} <0>{{ names }} <3>{{text}}<4>{\" \"} {timestamp && <6>}": "Channels <1><0>{{ names }} <3>{{text}}<4>{\" \"} {timestamp && <6>}", "Checking this box will make the survey accept written numbers as valid numeric responses, like \"one\" or \"fifty five\". Written numbers are supported up to one hundred (100).": "", "Choose a key for each language": "", - "Choose the data you want to be able to access through a public link": "", "Choose the data you want to download": "", "Choose the disposition you want to set at this point of the questionnaire.": "", "Choose the questionnaire answers you want to use to define quotas": "", @@ -144,6 +143,8 @@ "Done": "", "Download CSV": "", "Download contents as CSV": "", + "Download file": "", + "Download last generated file": "", "Downloaded {{surveyName}} {{reportType}}": "", "Drop your CSV file here, or click to browse": "", "Drop your MP3, WAV, M4A, ACC or MP4 file here, or click to browse": "", @@ -345,8 +346,7 @@ "Project successfully unarchived": "", "Projects": "", "Provider": "", - "Public link:": "", - "Public links": "", + "Public link": "", "QUEUE SIZE": "", "Quantity": "", "Question Prompt": "", @@ -373,6 +373,7 @@ "Ready to launch_survey": "", "Record": "", "Refused": "", + "Regenerate file": "", "Registered": "", "Rejected": "", "Remove collaborator": "", From 1f50e33d69d222535510f2c4761524ccd7bc5ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 24 Sep 2024 16:37:12 -0300 Subject: [PATCH 19/46] Stub downloading previously generated result files See #2350 --- assets/js/api.js | 19 +++++- .../respondents/RespondentIndex.jsx | 18 ++--- .../controllers/respondent_controller.ex | 65 ++++++++++++++++++- lib/ask_web/router.ex | 11 ++-- lib/ask_web/views/respondent_view.ex | 6 ++ 5 files changed, 103 insertions(+), 16 deletions(-) diff --git a/assets/js/api.js b/assets/js/api.js index 195886d1d..929829208 100644 --- a/assets/js/api.js +++ b/assets/js/api.js @@ -693,8 +693,25 @@ export const refreshDispositionHistoryLink = (projectId, surveyId) => { export const triggerRespondentsResultFile = (projectId, surveyId, q) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/results?${ +const downloadGeneratedFile = (sourceUrl) => + apiFetch(sourceUrl, null, null).then((response) => { + response.json().then((data) => { + window.open(data.url) + }) + } + ) + +export const downloadRespondentsResultsFile = (projectId, surveyId, q) => + downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/results_csv?${ (q && `&q=${encodeURIComponent(q)}`) || "" - }`, null, null) + }`) +export const downloadRespondentsDispositionHistoryFile = (projectId, surveyId) => + downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/disposition_history`) +export const downloadRespondentsIncentivesFile = (projectId, surveyId) => + downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/incentives`) +export const downloadRespondentsInteractionsFile = (projectId, surveyId) => + downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/interactions`) + export const triggerRespondentsDispositionHistoryCSV = (projectId, surveyId) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/disposition_history`, null, null) export const triggerRespondentsIncentivesCSV = (projectId, surveyId) => diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 6411d04df..c689e1bfb 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -121,25 +121,25 @@ class RespondentIndex extends Component { this.fetchRespondents(pageNumber - 1) } - downloadCSV(applyUserFilter = false) { + downloadResultsCSV(applyUserFilter = false) { const { projectId, surveyId, filter } = this.props const q = (applyUserFilter && filter) || null - api.triggerRespondentsResultFile(projectId, surveyId, q) + api.downloadRespondentsResultsFile(projectId, surveyId, q) } downloadDispositionHistoryCSV() { const { projectId, surveyId } = this.props - api.triggerRespondentsDispositionHistoryCSV(projectId, surveyId) + api.downloadRespondentsDispositionHistoryFile(projectId, surveyId) } downloadIncentivesCSV() { const { projectId, surveyId } = this.props - api.triggerRespondentsIncentivesCSV(projectId, surveyId) + api.downloadRespondentsIncentivesFile(projectId, surveyId) } downloadInteractionsCSV() { const { projectId, surveyId } = this.props - api.triggerRespondentsInteractionsCSV(projectId, surveyId) + api.downloadRespondentsInteractionsFile(projectId, surveyId) } sortBy(name) { @@ -343,7 +343,7 @@ class RespondentIndex extends Component { { totalCount, filter } ), downloadLink: null, - onDownload: () => this.downloadCSV(true), + onDownload: () => this.downloadResultsCSV(true), } break case "results": @@ -358,7 +358,7 @@ class RespondentIndex extends Component { this.refreshResultsLink, "resultsLink" ), - onDownload: () => this.downloadCSV(), + onDownload: () => this.downloadResultsCSV(), } break case "disposition-history": @@ -418,7 +418,7 @@ class RespondentIndex extends Component { const downloadButton = (
    - + get_app @@ -481,7 +481,7 @@ class RespondentIndex extends Component { const ownerOrAdmin = userLevel == "owner" || userLevel == "admin" return ( - +
    {t("Download CSV")}

    {t("Choose the data you want to download")}

    diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 2a38f1551..7926cc177 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -729,6 +729,30 @@ defmodule AskWeb.RespondentController do conn end + # FIXME: this function should return the proper file URL/path + defp generated_file_url(%{id: survey_id}, {:results, _filter}), do: "/?some-thingy" + defp generated_file_url(%{id: survey_id}, :disposition_history), do: "#dispositon_history" + defp generated_file_url(%{id: survey_id}, :incentives), do: "#incentives-#{survey_id}" + defp generated_file_url(%{id: survey_id}, :interactions), do: "#interactions-#{survey_id}" + + defp file_redirection(conn, survey, file_type) do + file_url = generated_file_url(survey, file_type) + + render(conn, "file-redirect.json", + file_url: file_url + ) + end + + def results_csv(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do + project = load_project(conn, project_id) + survey = load_survey(project, survey_id) + + filter = RespondentsFilter.parse(Map.get(params, "q", "")) + filter = add_params_to_filter(filter, params) + + file_redirection(conn, survey, {:results, filter}) + end + def trigger_results(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do project = load_project(conn, project_id) survey = load_survey(project, survey_id) @@ -766,6 +790,15 @@ defmodule AskWeb.RespondentController do filter end + def generate_disposition_history(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do + project = load_project(conn, project_id) + survey = load_survey(project, survey_id) + + SurveyResults.generate_disposition_history_file(survey.id) + + conn |> render("ok.json") + end + def disposition_history(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do project = load_project(conn, project_id) survey = load_survey(project, survey_id) @@ -774,8 +807,7 @@ defmodule AskWeb.RespondentController do # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "disposition_history") |> Repo.insert() - SurveyResults.generate_disposition_history_file(survey_id) - conn |> send_resp(200, "OK") + file_redirection(conn, survey, :disposition_history) end def incentives(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do @@ -793,6 +825,24 @@ defmodule AskWeb.RespondentController do # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() + file_redirection(conn, survey, :incentives) + end + + def generate_incentives(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do + project = + conn + |> load_project_for_owner(project_id) + + survey = + project + |> assoc(:surveys) + |> where([s], s.incentives_enabled) + |> Repo.get!(survey_id) + + # TODO: We just change this for "trigger generation" + # and add another log when actually downloading? + ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() + SurveyResults.generate_incentives_file(survey_id) conn |> send_resp(200, "OK") end @@ -805,6 +855,17 @@ defmodule AskWeb.RespondentController do # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "interactions") |> Repo.insert() + file_redirection(conn, survey, :interactions) + end + + def generate_interactions(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do + project = load_project_for_owner(conn, project_id) + survey = load_survey(project, survey_id) + + # TODO: We just change this for "trigger generation" + # and add another log when actually downloading? + ActivityLog.download(project, conn, survey, "interactions") |> Repo.insert() + SurveyResults.generate_interactions_file(survey_id) conn |> send_resp(200, "OK") end diff --git a/lib/ask_web/router.ex b/lib/ask_web/router.ex index 5080cda73..2adc6b400 100644 --- a/lib/ask_web/router.ex +++ b/lib/ask_web/router.ex @@ -193,16 +193,19 @@ defmodule AskWeb.Router do pipe_through :api get "/results", RespondentController, :results, as: :get_respondents_results - + get "/results_csv", RespondentController, :results_csv post "/results", RespondentController, :generate_results, as: :respondents_results + get "/disposition_history", RespondentController, :disposition_history, as: :respondents_disposition_history post "/disposition_history", RespondentController, :generate_disposition_history, - as: :respondents_disposition_history + as: :generate_disposition_history - post "/incentives", RespondentController, :generate_incentives, as: :respondents_incentives + get "/incentives", RespondentController, :incentives, as: :respondents_incentives + post "/incentives", RespondentController, :generate_incentives, as: :generate_respondents_incentives + get "/interactions", RespondentController, :interactions, as: :respondents_interactions post "/interactions", RespondentController, :generate_interactions, - as: :respondents_interactions + as: :generate_respondents_interactions end end end diff --git a/lib/ask_web/views/respondent_view.ex b/lib/ask_web/views/respondent_view.ex index 8bae12d80..1b1985e1f 100644 --- a/lib/ask_web/views/respondent_view.ex +++ b/lib/ask_web/views/respondent_view.ex @@ -119,6 +119,10 @@ defmodule AskWeb.RespondentView do %{data: %{}} end + def render("file-redirect.json", %{file_url: url}) do + %{url: url} + end + def render("respondent.json", %{ respondent: respondent, partial_relevant_enabled: partial_relevant_enabled @@ -269,6 +273,8 @@ defmodule AskWeb.RespondentView do percent: percentage } end + + def render("ok.json", _), do: %{status: :ok} defp render_index_meta(%{respondents_count: respondents_count, index_fields: index_fields}), do: %{ From af1a87e247e2307a03ace3d3f73f3b14b3a82ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 24 Sep 2024 17:19:55 -0300 Subject: [PATCH 20/46] Typo See #2350 --- lib/ask/survey_results.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index bf55acc4a..fca3e5ac3 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -525,7 +525,7 @@ defmodule Ask.SurveyResults do end def generate_respondent_result_file(survey_id, filters) do - Logger.info("Enqueueing generation of survey (id: #{survey_id}) disposition_history file") + Logger.info("Enqueueing generation of survey (id: #{survey_id}) respondent_result file") GenServer.cast(server_ref(), {:respondent_result, survey_id, filters}) end From 8d0f518711ab4a0d0c44c1f1c35edc2446a7acc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 25 Sep 2024 14:24:58 -0300 Subject: [PATCH 21/46] Generate async files in a stable path This will allow us to check if there already is a file generated or not. Also, we move the decision of whether to regenerate a file or not to the user (instead of checking if we should generate the file again or not). See #2350 --- lib/ask/survey_results.ex | 69 ++++++++++++++------------------------- 1 file changed, 24 insertions(+), 45 deletions(-) diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index fca3e5ac3..afc0eea2d 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -44,11 +44,7 @@ defmodule Ask.SurveyResults do def handle_cast({file_type, survey_id, args}, state) do survey = Repo.get!(Survey, survey_id) - if should_generate_file(file_type, survey) do - do_generate_file(file_type, survey, args) - else - Logger.info("Ignoring generation of #{file_type} file (survey_id: #{survey_id})") - end + generate_file(file_type, survey, args) {:noreply, state, :hibernate} end @@ -59,7 +55,7 @@ defmodule Ask.SurveyResults do {:noreply, state, :hibernate} end - defp do_generate_file(:interactions, survey, _) do + defp generate_file(:interactions, survey, _) do channels = survey_log_entry_channel_names(survey) Logger.info("Starting to build interaction file (survey_id: #{survey.id})") @@ -121,7 +117,7 @@ defmodule Ask.SurveyResults do write_to_file(:interactions, survey, rows) end - defp do_generate_file(:incentives, survey, _) do + defp generate_file(:incentives, survey, _) do questionnaires = survey_respondent_questionnaires(survey) tz_offset_in_seconds = Survey.timezone_offset_in_seconds(survey) @@ -153,7 +149,7 @@ defmodule Ask.SurveyResults do end) end - defp do_generate_file(:disposition_history, survey, _) do + defp generate_file(:disposition_history, survey, _) do history = Stream.resource( fn -> 0 end, @@ -193,7 +189,7 @@ defmodule Ask.SurveyResults do write_to_file(:disposition_history, survey, rows) end - defp do_generate_file(:respondent_result, survey, filter) do + defp generate_file(:respondent_result, survey, filter) do tz_offset = Survey.timezone_offset(survey) questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires @@ -353,26 +349,38 @@ defmodule Ask.SurveyResults do rows = Stream.concat([[header], csv_rows]) + # FIXME: we should somehow include the filter here write_to_file(:respondent_result, survey, rows) end - defp do_generate_file(file_type, _, _), - do: Logger.warn("No function for generating #{file_type} files") - - def file_path(survey, file_type) do - filename = csv_filename(survey, file_prefix(file_type)) - "#{@target_dir}/#{filename}" + def file_path(survey, file_type, target_dir \\ @target_dir) do + # FIXME: as a first iteration, generate a stable name and have a single file per type + # but we should probably include the date and respondent results filter in the name + prefix = file_prefix(file_type) + # name = survey.name || "survey_id_#{survey.id}" + # name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") + # current_time = Timex.format!(DateTime.utc_now(), "%Y-%m-%d-%H-%M-%S", :strftime) + # "#{target_dir}/#{name}_#{survey.state}-#{prefix}_#{current_time}.csv" + "#{target_dir}/survey_#{survey.id}-#{survey.state}-#{prefix}.csv" end defp write_to_file(file_type, survey, rows) do File.mkdir_p!(@target_dir) - file = File.open!(file_path(survey, file_type), [:write, :utf8]) + target_file = file_path(survey, file_type, @target_dir) + if File.exists?(target_file), do: File.rm!(target_file) + + # Poor man's mktemp. We only want to avoid having the file living at the stable + # path while it's still being written to avoid partial downloads + temporal_file = for _ <- 1..10, into: "#{target_file}.tmp.", do: <> + file = File.open!(temporal_file, [:write, :utf8]) initial_datetime = Timex.now() rows |> CSV.encode() |> Enum.each(&IO.write(file, &1)) + File.rename!(temporal_file, target_file) + seconds_to_process_file = Timex.diff(Timex.now(), initial_datetime, :seconds) Logger.info( @@ -384,24 +392,6 @@ defmodule Ask.SurveyResults do defp file_prefix(:incentives), do: "respondents_incentives" defp file_prefix(:disposition_history), do: "disposition_history" defp file_prefix(:respondent_result), do: "respondents" - defp file_prefix(_), do: "" - - # FIXME: we probably don't need to check if we should generate the file - defp should_generate_file(:xxxx_interactions, survey) do - # TODO: when do we want to skip the re-generation of the file? - File.mkdir_p!(@target_dir) # ensure the directory exists - existing_files = File.ls!(@target_dir) - - exists_file = - existing_files - |> Enum.any?(fn file -> - file |> String.starts_with?(survey_filename_prefix(survey, file_prefix(:interactions))) - end) - - !exists_file - end - - defp should_generate_file(_type, _survey), do: true defp survey_log_entry_channel_names(survey) do respondent_groups = Repo.preload(survey, respondent_groups: [:channels]).respondent_groups @@ -436,17 +426,6 @@ defmodule Ask.SurveyResults do end end - defp csv_filename(survey, prefix) do - prefix = survey_filename_prefix(survey, prefix) - Timex.format!(DateTime.utc_now(), "#{prefix}_%Y-%m-%d-%H-%M-%S.csv", :strftime) - end - - defp survey_filename_prefix(survey, prefix) do - name = survey.name || "survey_id_#{survey.id}" - name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") - "#{name}_#{survey.state}-#{prefix}" - end - defp csv_datetime(nil, _, _), do: "" defp csv_datetime(dt, tz_offset_in_seconds, tz_offset) when is_binary(dt) do From 17496f6a2d1c6a4bb4fd2156eae0c5349693f9cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 25 Sep 2024 20:11:19 -0300 Subject: [PATCH 22/46] Trigger files generation From the UI, request that the CSV files are generated by the backend. We still miss checking if the files exist or are currently being generated. See #2350 --- assets/js/actions/survey.js | 36 ++++++++++++++++ assets/js/api.js | 42 ++++++++++++++++++- .../respondents/RespondentIndex.jsx | 28 ++++++++++++- lib/ask/survey_results.ex | 3 +- .../controllers/respondent_controller.ex | 4 +- 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/assets/js/actions/survey.js b/assets/js/actions/survey.js index 8f058ebad..087aba8c9 100644 --- a/assets/js/actions/survey.js +++ b/assets/js/actions/survey.js @@ -44,6 +44,7 @@ export const REFRESH_LINK = "SURVEY_REFRESH_LINK" export const DELETE_LINK = "SURVEY_DELETE_LINK" export const RECEIVE_SURVEY_STATS = "RECEIVE_SURVEY_STATS" export const RECEIVE_SURVEY_RETRIES_HISTOGRAMS = "RECEIVE_SURVEY_RETRIES_HISTOGRAMS" +export const GENERATING_FILE = "GENERATING_FILE" export const createSurvey = (projectId: number, folderId?: number) => (dispatch: Function, getState: () => Store) => @@ -366,6 +367,11 @@ export const deleteLink = (link: Link) => ({ link, }) +export const generatingFile = (file: String) => ({ + type: GENERATING_FILE, + file +}) + export const createResultsLink = (projectId: number, surveyId: number) => (dispatch: Function) => { api.createResultsLink(projectId, surveyId).then((response) => { return dispatch(receiveLink(response)) @@ -448,3 +454,33 @@ export const deleteDispositionHistoryLink = return dispatch(deleteLink(link)) }) } + +export const generateResultsFile = + (projectId: number, surveyId: number, filter?: string) => (dispatch: Function) => { + api.generateResults(projectId, surveyId, filter).then((response) => { + return dispatch(generatingFile("respondent-results")) // FIXME: what should we dispatch? + }) + } + +export const generateIncentivesFile = + (projectId: number, surveyId: number) => (dispatch: Function) => { + // TODO: better handle when incentives download are disabled (due to sample with IDs) + api.generateIncentives(projectId, surveyId).then((response) => { + return dispatch(generatingFile("incentives")) // FIXME: what should we dispatch? + }) + } + +export const generateInteractionsFile = + (projectId: number, surveyId: number) => (dispatch: Function) => { + api.generateInteractions(projectId, surveyId).then((response) => { + return dispatch(generatingFile("interactions")) // FIXME: what should we dispatch? + }) + } + +export const generateDispositionHistoryFile = + (projectId: number, surveyId: number) => (dispatch: Function) => { + api.generateDispositionHistory(projectId, surveyId).then((response) => { + return dispatch(generatingFile("disposition-history")) // FIXME: what should we dispatch? + }) + } + diff --git a/assets/js/api.js b/assets/js/api.js index 929829208..7b5426a4c 100644 --- a/assets/js/api.js +++ b/assets/js/api.js @@ -691,8 +691,46 @@ export const refreshDispositionHistoryLink = (projectId, surveyId) => { ) } -export const triggerRespondentsResultFile = (projectId, surveyId, q) => - apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/results?${ +export const generateResults = (projectId, surveyId, filter) => { + return apiPutOrPostJSONWithCallback( + `projects/${projectId}/surveys/${surveyId}/respondents/results`, + arrayOf(referenceSchema), + "POST", + { q: filter }, + passthroughCallback + ) +} + +export const generateIncentives = (projectId, surveyId) => { + return apiPutOrPostJSONWithCallback( + `projects/${projectId}/surveys/${surveyId}/respondents/incentives`, + arrayOf(referenceSchema), + "POST", + {}, + passthroughCallback + ) +} + +export const generateInteractions = (projectId, surveyId) => { + return apiPutOrPostJSONWithCallback( + `projects/${projectId}/surveys/${surveyId}/respondents/interactions`, + arrayOf(referenceSchema), + "POST", + {}, + passthroughCallback + ) +} + +export const generateDispositionHistory = (projectId, surveyId) => { + return apiPutOrPostJSONWithCallback( + `projects/${projectId}/surveys/${surveyId}/respondents/disposition_history`, + arrayOf(referenceSchema), + "POST", + {}, + passthroughCallback + ) +} + const downloadGeneratedFile = (sourceUrl) => apiFetch(sourceUrl, null, null).then((response) => { response.json().then((data) => { diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index c689e1bfb..fff846b1b 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -227,6 +227,26 @@ class RespondentIndex extends Component { surveyActions.refreshDispositionHistoryLink(projectId, surveyId, link) } + generateResults(filter) { + const { projectId, surveyId, surveyActions } = this.props + surveyActions.generateResultsFile(projectId, surveyId, filter) + } + + generateIncentives() { + const { projectId, surveyId, surveyActions } = this.props + surveyActions.generateIncentivesFile(projectId, surveyId) + } + + generateInteractions() { + const { projectId, surveyId, surveyActions } = this.props + surveyActions.generateInteractionsFile(projectId, surveyId) + } + + generateDispositionHistory() { + const { projectId, surveyId, surveyActions } = this.props + surveyActions.generateDispositionHistoryFile(projectId, surveyId) + } + copyLink(link) { try { window.getSelection().selectAllChildren(link) @@ -333,6 +353,7 @@ class RespondentIndex extends Component { disabled?: boolean, downloadLink: any, onDownload: Function, + onGenerate: Function, } = null switch (id) { case "filtered-results": @@ -344,6 +365,7 @@ class RespondentIndex extends Component { ), downloadLink: null, onDownload: () => this.downloadResultsCSV(true), + onGenerate: () => this.generateResults(filter), } break case "results": @@ -359,6 +381,7 @@ class RespondentIndex extends Component { "resultsLink" ), onDownload: () => this.downloadResultsCSV(), + onGenerate: () => this.generateResults(), } break case "disposition-history": @@ -374,6 +397,7 @@ class RespondentIndex extends Component { "dispositionHistoryLink" ), onDownload: () => this.downloadDispositionHistoryCSV(), + onGenerate: () => this.generateDispositionHistory(), } break case "incentives": @@ -392,6 +416,7 @@ class RespondentIndex extends Component { "incentivesLink" ), onDownload: () => this.downloadIncentivesCSV(), + onGenerate: () => this.generateIncentives(), } break case "interactions": @@ -407,6 +432,7 @@ class RespondentIndex extends Component { "interactionsLink" ), onDownload: () => this.downloadInteractionsCSV(), + onGenerate: () => this.generateInteractions(), } } @@ -428,7 +454,7 @@ class RespondentIndex extends Component { // FIXME: this should be calculated } - + refresh diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index afc0eea2d..75ce3477f 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -350,7 +350,7 @@ defmodule Ask.SurveyResults do rows = Stream.concat([[header], csv_rows]) # FIXME: we should somehow include the filter here - write_to_file(:respondent_result, survey, rows) + write_to_file(:filtered_respondent_result, survey, rows) end def file_path(survey, file_type, target_dir \\ @target_dir) do @@ -392,6 +392,7 @@ defmodule Ask.SurveyResults do defp file_prefix(:incentives), do: "respondents_incentives" defp file_prefix(:disposition_history), do: "disposition_history" defp file_prefix(:respondent_result), do: "respondents" + defp file_prefix(:filtered_respondent_result), do: "respondents_filtered" defp survey_log_entry_channel_names(survey) do respondent_groups = Repo.preload(survey, respondent_groups: [:channels]).respondent_groups diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 7926cc177..a4e6f6f56 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -753,7 +753,7 @@ defmodule AskWeb.RespondentController do file_redirection(conn, survey, {:results, filter}) end - def trigger_results(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do + def generate_results(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do project = load_project(conn, project_id) survey = load_survey(project, survey_id) @@ -768,7 +768,7 @@ defmodule AskWeb.RespondentController do ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() - conn |> send_resp(200, "OK") + conn |> render("ok.json") end defp add_params_to_filter(filter, params) do From 014cb13935439a19f7e82ee9e6736464cd586a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 26 Sep 2024 18:15:00 -0300 Subject: [PATCH 23/46] Statically serve the generated files See #2350 --- lib/ask/survey_results.ex | 4 ++-- lib/ask_web/controllers/respondent_controller.ex | 10 ++-------- lib/ask_web/router.ex | 4 ++++ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index 75ce3477f..0f3818b5c 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -365,8 +365,8 @@ defmodule Ask.SurveyResults do end defp write_to_file(file_type, survey, rows) do - File.mkdir_p!(@target_dir) - target_file = file_path(survey, file_type, @target_dir) + File.mkdir_p!("./priv/static/" <> @target_dir) + target_file = "./priv/static/" <> file_path(survey, file_type, @target_dir) if File.exists?(target_file), do: File.rm!(target_file) # Poor man's mktemp. We only want to avoid having the file living at the stable diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index a4e6f6f56..838b359ef 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -729,17 +729,11 @@ defmodule AskWeb.RespondentController do conn end - # FIXME: this function should return the proper file URL/path - defp generated_file_url(%{id: survey_id}, {:results, _filter}), do: "/?some-thingy" - defp generated_file_url(%{id: survey_id}, :disposition_history), do: "#dispositon_history" - defp generated_file_url(%{id: survey_id}, :incentives), do: "#incentives-#{survey_id}" - defp generated_file_url(%{id: survey_id}, :interactions), do: "#interactions-#{survey_id}" - defp file_redirection(conn, survey, file_type) do - file_url = generated_file_url(survey, file_type) + file_url = SurveyResults.file_path(survey, file_type) render(conn, "file-redirect.json", - file_url: file_url + file_url: "/#{file_url}" # TODO: there may be better ways of avoiding relative URLs ) end diff --git a/lib/ask_web/router.ex b/lib/ask_web/router.ex index 2adc6b400..ae70a82ff 100644 --- a/lib/ask_web/router.ex +++ b/lib/ask_web/router.ex @@ -11,6 +11,10 @@ defmodule AskWeb.Router do plug :protect_from_forgery plug :put_secure_browser_headers plug Plug.Static, at: "files/", from: "assets/static/files/" + + # FIXME: this is temporary. We have to apply access control here: https://elixirforum.com/t/how-can-you-hide-certain-static-assets-behind-a-require-authenticated-user-path/53106/6 + plug Plug.Static, at: "generated_files/", from: "priv/static/generated_files/" + plug Coherence.Authentication.Session, db_model: Ask.User end From c76fada2cd132165c7e256577c57b8571b48a107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 1 Oct 2024 06:02:54 -0300 Subject: [PATCH 24/46] Add file status endpoint See #2350 --- assets/js/api.js | 48 ++------------- lib/ask/respondents_filter.ex | 3 + lib/ask/survey_results.ex | 60 ++++++++++++++++--- .../controllers/respondent_controller.ex | 23 ++++++- lib/ask_web/router.ex | 2 + lib/ask_web/views/respondent_view.ex | 4 ++ 6 files changed, 87 insertions(+), 53 deletions(-) diff --git a/assets/js/api.js b/assets/js/api.js index 7b5426a4c..d1198c94a 100644 --- a/assets/js/api.js +++ b/assets/js/api.js @@ -691,46 +691,6 @@ export const refreshDispositionHistoryLink = (projectId, surveyId) => { ) } -export const generateResults = (projectId, surveyId, filter) => { - return apiPutOrPostJSONWithCallback( - `projects/${projectId}/surveys/${surveyId}/respondents/results`, - arrayOf(referenceSchema), - "POST", - { q: filter }, - passthroughCallback - ) -} - -export const generateIncentives = (projectId, surveyId) => { - return apiPutOrPostJSONWithCallback( - `projects/${projectId}/surveys/${surveyId}/respondents/incentives`, - arrayOf(referenceSchema), - "POST", - {}, - passthroughCallback - ) -} - -export const generateInteractions = (projectId, surveyId) => { - return apiPutOrPostJSONWithCallback( - `projects/${projectId}/surveys/${surveyId}/respondents/interactions`, - arrayOf(referenceSchema), - "POST", - {}, - passthroughCallback - ) -} - -export const generateDispositionHistory = (projectId, surveyId) => { - return apiPutOrPostJSONWithCallback( - `projects/${projectId}/surveys/${surveyId}/respondents/disposition_history`, - arrayOf(referenceSchema), - "POST", - {}, - passthroughCallback - ) -} - const downloadGeneratedFile = (sourceUrl) => apiFetch(sourceUrl, null, null).then((response) => { response.json().then((data) => { @@ -750,11 +710,13 @@ export const downloadRespondentsIncentivesFile = (projectId, surveyId) => export const downloadRespondentsInteractionsFile = (projectId, surveyId) => downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/interactions`) -export const triggerRespondentsDispositionHistoryCSV = (projectId, surveyId) => +export const generateResults = (projectId, surveyId, filter) => + apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/results`, null, { q: filter }) +export const generateDispositionHistory = (projectId, surveyId) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/disposition_history`, null, null) -export const triggerRespondentsIncentivesCSV = (projectId, surveyId) => +export const generateIncentives = (projectId, surveyId) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/incentives`, null, null) -export const triggerRespondentsInteractionsCSV = (projectId, surveyId) => +export const generateInteractions = (projectId, surveyId) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/interactions`, null, null) export const startSimulation = (projectId, questionnaireId, mode) => { diff --git a/lib/ask/respondents_filter.ex b/lib/ask/respondents_filter.ex index c54174494..09f780b65 100644 --- a/lib/ask/respondents_filter.ex +++ b/lib/ask/respondents_filter.ex @@ -29,6 +29,9 @@ defmodule Ask.RespondentsFilter do def date_format_string(), do: @date_format_string + def empty?(%__MODULE__{disposition: nil, since: nil, state: nil, mode: nil}), do: true + def empty?(%__MODULE__{}), do: false + @doc """ By putting since directly (without parsing it) we're trying to cover the case where Surveda is being used by external services like SurvedaOnaConnector diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index 0f3818b5c..2916ee0ca 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -189,7 +189,7 @@ defmodule Ask.SurveyResults do write_to_file(:disposition_history, survey, rows) end - defp generate_file(:respondent_result, survey, filter) do + defp generate_file(:respondents_results, survey, filter) do tz_offset = Survey.timezone_offset(survey) questionnaires = (survey |> Repo.preload(:questionnaires)).questionnaires @@ -349,8 +349,7 @@ defmodule Ask.SurveyResults do rows = Stream.concat([[header], csv_rows]) - # FIXME: we should somehow include the filter here - write_to_file(:filtered_respondent_result, survey, rows) + write_to_file({:respondents_results, filter}, survey, rows) end def file_path(survey, file_type, target_dir \\ @target_dir) do @@ -388,11 +387,56 @@ defmodule Ask.SurveyResults do ) end + def files_status(survey, file_types) do + %{ + survey_id: survey.id, + survey_state: survey.state, + files: + file_types + |> Enum.uniq + |> Enum.map(fn file_type -> file_status(survey, file_type) end) + } + end + + defp file_creating?(target_path), do: Path.wildcard(target_path <> ".tmp.*") |> Enum.any? + + defp created_at(_path, false), do: nil + defp created_at(path, true) do + case File.stat(path, [time: :posix]) do + {:ok, %{ mtime: last_mod }} -> last_mod + _ -> nil + end + end + + defp file_type_symbol({:respondents_results, filter}) do + if RespondentsFilter.empty?(filter) do + :respondents_results + else + :respondents_filtered + end + end + defp file_type_symbol(type), do: type + + defp file_status(survey, file_type) do + path = "./priv/static/" <> file_path(survey, file_type) + exists = File.exists?(path) + %{ + file_type: file_type_symbol(file_type), + creating: file_creating?(path), + created_at: created_at(path, exists) + } + end + defp file_prefix(:interactions), do: "respondents_interactions" defp file_prefix(:incentives), do: "respondents_incentives" defp file_prefix(:disposition_history), do: "disposition_history" - defp file_prefix(:respondent_result), do: "respondents" - defp file_prefix(:filtered_respondent_result), do: "respondents_filtered" + defp file_prefix({:respondents_results, filter}) do + if RespondentsFilter.empty?(filter) do + "respondents" + else + "respondents_filtered" + end + end defp survey_log_entry_channel_names(survey) do respondent_groups = Repo.preload(survey, respondent_groups: [:channels]).respondent_groups @@ -504,9 +548,9 @@ defmodule Ask.SurveyResults do GenServer.cast(server_ref(), {:disposition_history, survey_id, nil}) end - def generate_respondent_result_file(survey_id, filters) do - Logger.info("Enqueueing generation of survey (id: #{survey_id}) respondent_result file") - GenServer.cast(server_ref(), {:respondent_result, survey_id, filters}) + def generate_respondents_results_file(survey_id, filters) do + Logger.info("Enqueueing generation of survey (id: #{survey_id}) respondents_results file") + GenServer.cast(server_ref(), {:respondents_results, survey_id, filters}) end ## Public Module diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 838b359ef..7a3634f65 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -729,6 +729,25 @@ defmodule AskWeb.RespondentController do conn end + def files_status(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do + project = load_project(conn, project_id) + survey = load_survey(project, survey_id) + + filter = RespondentsFilter.parse(Map.get(params, "q", "")) + filter = add_params_to_filter(filter, params) + + # FIXME: filter according to permissions + status = SurveyResults.files_status(survey, [ + {:respondents_results, %RespondentsFilter{}}, + {:respondents_results, filter}, + :interactions, + :incentives, + :disposition_history + ]) + + render(conn, "status.json", status: status) + end + defp file_redirection(conn, survey, file_type) do file_url = SurveyResults.file_path(survey, file_type) @@ -744,7 +763,7 @@ defmodule AskWeb.RespondentController do filter = RespondentsFilter.parse(Map.get(params, "q", "")) filter = add_params_to_filter(filter, params) - file_redirection(conn, survey, {:results, filter}) + file_redirection(conn, survey, {:respondents_results, filter}) end def generate_results(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do @@ -758,7 +777,7 @@ defmodule AskWeb.RespondentController do # ?param1=value is more specific than ?q=param1:value filter = add_params_to_filter(filter, params) - SurveyResults.generate_respondent_result_file(survey_id, filter) + SurveyResults.generate_respondents_results_file(survey_id, filter) ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() diff --git a/lib/ask_web/router.ex b/lib/ask_web/router.ex index ae70a82ff..498a325b6 100644 --- a/lib/ask_web/router.ex +++ b/lib/ask_web/router.ex @@ -196,6 +196,8 @@ defmodule AskWeb.Router do scope "/respondents" do pipe_through :api + get "/files", RespondentController, :files_status, as: :files + get "/results", RespondentController, :results, as: :get_respondents_results get "/results_csv", RespondentController, :results_csv post "/results", RespondentController, :generate_results, as: :respondents_results diff --git a/lib/ask_web/views/respondent_view.ex b/lib/ask_web/views/respondent_view.ex index 1b1985e1f..313a26d03 100644 --- a/lib/ask_web/views/respondent_view.ex +++ b/lib/ask_web/views/respondent_view.ex @@ -123,6 +123,10 @@ defmodule AskWeb.RespondentView do %{url: url} end + def render("status.json", %{status: status}) do + status + end + def render("respondent.json", %{ respondent: respondent, partial_relevant_enabled: partial_relevant_enabled From 0d545fdd1a3dfefda7e53d0ab6d7a6602cc9be08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 2 Oct 2024 18:09:08 -0300 Subject: [PATCH 25/46] Make API return a map of files instead of an array See #2350 --- lib/ask/survey_results.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index 2916ee0ca..e835de770 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -394,7 +394,7 @@ defmodule Ask.SurveyResults do files: file_types |> Enum.uniq - |> Enum.map(fn file_type -> file_status(survey, file_type) end) + |> Map.new(fn file_type -> {file_type_symbol(file_type), file_status(survey, file_type)} end) } end From 8ef0ffd404d71e2d2f05ee06c74ab812516da4d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 2 Oct 2024 18:08:07 -0300 Subject: [PATCH 26/46] Fetch files status upon Downloads dialog load See #2350 --- assets/js/actions/survey.js | 22 +++++++++++++++++++ assets/js/api.js | 3 +++ .../respondents/RespondentIndex.jsx | 10 ++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/assets/js/actions/survey.js b/assets/js/actions/survey.js index 087aba8c9..5a1f683d5 100644 --- a/assets/js/actions/survey.js +++ b/assets/js/actions/survey.js @@ -45,6 +45,8 @@ export const DELETE_LINK = "SURVEY_DELETE_LINK" export const RECEIVE_SURVEY_STATS = "RECEIVE_SURVEY_STATS" export const RECEIVE_SURVEY_RETRIES_HISTOGRAMS = "RECEIVE_SURVEY_RETRIES_HISTOGRAMS" export const GENERATING_FILE = "GENERATING_FILE" +export const FETCHING_FILES_STATUS = "FETCHING_FILES_STATUS" +export const RECEIVE_FILES_STATUS = "RECEIVE_FILES_STATUS" export const createSurvey = (projectId: number, folderId?: number) => (dispatch: Function, getState: () => Store) => @@ -455,6 +457,26 @@ export const deleteDispositionHistoryLink = }) } +export const fetchingRespondentFilesStatus = (projectId: number, surveyId: number) => ({ + type: FETCHING_FILES_STATUS, + projectId, + surveyId, +}) + +export const receiveRespondentsFilesStatus = (projectId: number, surveyId: number, files) => ({ + type: RECEIVE_FILES_STATUS, + projectId, + surveyId, + files, +}) + +export const fetchRespondentsFilesStatus = (projectId: number, surveyId: number, filter?: string) => (dispatch: Function) => { + dispatch(fetchingRespondentFilesStatus(projectId, surveyId)) + api.fetchRespondentsFilesStatus(projectId, surveyId, filter).then((response) => { + dispatch(receiveRespondentsFilesStatus(projectId, surveyId, response.files)) + }) +} + export const generateResultsFile = (projectId: number, surveyId: number, filter?: string) => (dispatch: Function) => { api.generateResults(projectId, surveyId, filter).then((response) => { diff --git a/assets/js/api.js b/assets/js/api.js index d1198c94a..b840541f2 100644 --- a/assets/js/api.js +++ b/assets/js/api.js @@ -719,6 +719,9 @@ export const generateIncentives = (projectId, surveyId) => export const generateInteractions = (projectId, surveyId) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/interactions`, null, null) +export const fetchRespondentsFilesStatus = (projectId, surveyId, filter) => + apiFetch(`projects/${projectId}/surveys/${surveyId}/respondents/files?q=${filter}`).then((response) => response.json()) + export const startSimulation = (projectId, questionnaireId, mode) => { return apiPutOrPostJSONWithCallback( `projects/${projectId}/questionnaires/${questionnaireId}/simulation`, diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index fff846b1b..a49e26efc 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -111,6 +111,14 @@ class RespondentIndex extends Component { ) } + showDownloadsModal() { + const { projectId, surveyId, filter } = this.props + + // FIXME: don't fetch if we're already fetching + this.props.surveyActions.fetchRespondentsFilesStatus(projectId, surveyId, filter) + $('#downloadCSV').modal("open") + } + nextPage() { const { pageNumber } = this.props this.fetchRespondents(pageNumber + 1) @@ -713,7 +721,7 @@ class RespondentIndex extends Component { __html: "", }} /> - $('#downloadCSV').modal("open")}/> + this.showDownloadsModal()} /> {this.downloadModal()} {this.renderColumnPickerModal()} {this.respondentsFilter()} From 1b1718135291263467b942e036e1442a9c202aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Mon, 14 Oct 2024 17:51:19 -0300 Subject: [PATCH 27/46] Fetch files status upon download dialog See #2350 --- assets/js/actions/survey.js | 11 +++++------ assets/js/decls/survey.js | 14 ++++++++++++++ assets/js/reducers/index.js | 2 ++ assets/js/reducers/respondentsFiles.js | 26 ++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 assets/js/reducers/respondentsFiles.js diff --git a/assets/js/actions/survey.js b/assets/js/actions/survey.js index 5a1f683d5..4166c4af2 100644 --- a/assets/js/actions/survey.js +++ b/assets/js/actions/survey.js @@ -457,23 +457,22 @@ export const deleteDispositionHistoryLink = }) } -export const fetchingRespondentFilesStatus = (projectId: number, surveyId: number) => ({ +export const fetchingRespondentFilesStatus = (surveyId: number) => ({ type: FETCHING_FILES_STATUS, - projectId, surveyId, }) -export const receiveRespondentsFilesStatus = (projectId: number, surveyId: number, files) => ({ +export const receiveRespondentsFilesStatus = (surveyId: number, surveyState: string, files: SurveyFiles) => ({ type: RECEIVE_FILES_STATUS, - projectId, surveyId, + surveyState, files, }) export const fetchRespondentsFilesStatus = (projectId: number, surveyId: number, filter?: string) => (dispatch: Function) => { - dispatch(fetchingRespondentFilesStatus(projectId, surveyId)) + dispatch(fetchingRespondentFilesStatus(surveyId)) api.fetchRespondentsFilesStatus(projectId, surveyId, filter).then((response) => { - dispatch(receiveRespondentsFilesStatus(projectId, surveyId, response.files)) + dispatch(receiveRespondentsFilesStatus(surveyId, response.survey_state, response.files)) }) } diff --git a/assets/js/decls/survey.js b/assets/js/decls/survey.js index 79fb7cbd2..00f9a1ace 100644 --- a/assets/js/decls/survey.js +++ b/assets/js/decls/survey.js @@ -146,3 +146,17 @@ export type Response = { } export type Disposition = null | "completed" | "partial" | "ineligible" + +export type FileStatus = { + created_at: Date?, + creating: boolean, + file_type: string, +} + +export type SurveyFiles = { + disposition_history: ?FileStatus, + incentives: ?FileStatus, + interactions: ?FileStatus, + respondents_filtered: ?FileStatus, + respondents_results: ?FileStatus, +} diff --git a/assets/js/reducers/index.js b/assets/js/reducers/index.js index ae68072d8..e213d2be8 100644 --- a/assets/js/reducers/index.js +++ b/assets/js/reducers/index.js @@ -16,6 +16,7 @@ import respondentGroups from "./respondentGroups" import respondents from "./respondents" import respondentsCount from "./respondentsCount" import respondentsStats from "./respondentsStats" +import respondentsFiles from "./respondentsFiles" import survey from "./survey" import surveys from "./surveys" import timezones from "./timezones" @@ -47,6 +48,7 @@ export default combineReducers({ respondentGroups, respondents, respondentsCount, + respondentsFiles, respondentsStats, routing, survey, diff --git a/assets/js/reducers/respondentsFiles.js b/assets/js/reducers/respondentsFiles.js new file mode 100644 index 000000000..96b444020 --- /dev/null +++ b/assets/js/reducers/respondentsFiles.js @@ -0,0 +1,26 @@ +import * as actions from "../actions/survey" + +export default (state = {}, action) => { + switch (action.type) { + case actions.FETCHING_FILES_STATUS: + if (action.surveyId != state.surveyId) { + return { + ...state, + surveyId: null, + surveyState: null, + files: null, + } + } else { + return state + } + case actions.RECEIVE_FILES_STATUS: + return { + ...state, + surveyId: action.surveyId, + surveyState: action.surveyState, + files: action.files, + } + default: + return state + } +} From b14a27a5c6d25e39c4d672481d710559a9112cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 29 Oct 2024 18:18:42 -0300 Subject: [PATCH 28/46] Enable/disable file buttons depending on state See #2350 --- assets/js/actions/survey.js | 1 + .../respondents/RespondentIndex.jsx | 75 ++++++++++++++----- locales/template/translation.json | 14 ++++ 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/assets/js/actions/survey.js b/assets/js/actions/survey.js index 4166c4af2..02b18f2cb 100644 --- a/assets/js/actions/survey.js +++ b/assets/js/actions/survey.js @@ -479,6 +479,7 @@ export const fetchRespondentsFilesStatus = (projectId: number, surveyId: number, export const generateResultsFile = (projectId: number, surveyId: number, filter?: string) => (dispatch: Function) => { api.generateResults(projectId, surveyId, filter).then((response) => { + // FIXME: periodically check for file generation to have finished return dispatch(generatingFile("respondent-results")) // FIXME: what should we dispatch? }) } diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index a49e26efc..47971426c 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -2,6 +2,7 @@ import React, { Component } from "react" import { bindActionCreators } from "redux" import { connect } from "react-redux" +import TimeAgo from "react-timeago" import * as api from "../../api" import * as actions from "../../actions/respondents" import { fieldUniqueKey, isFieldSelected } from "../../reducers/respondents" @@ -71,6 +72,7 @@ class RespondentIndex extends Component { refreshInteractionsLink: Function refreshDispositionHistoryLink: Function columnPickerModalId: string + timeFormatter: Function constructor(props) { super(props) @@ -83,6 +85,7 @@ class RespondentIndex extends Component { this.refreshIncentivesLink = this.refreshIncentivesLink.bind(this) this.refreshInteractionsLink = this.refreshInteractionsLink.bind(this) this.refreshDispositionHistoryLink = this.refreshDispositionHistoryLink.bind(this) + this.timeFormatter = this.timeFormatter.bind(this) this.columnPickerModalId = uniqueId("column-picker-modal-id_") } @@ -111,6 +114,27 @@ class RespondentIndex extends Component { ) } + timeFormatter(number, unit, suffix, date, defaultFormatter) { + const { t } = this.props + + switch (unit) { + case "second": + return t("{{count}} seconds ago", { count: number }) + case "minute": + return t("{{count}} minutes ago", { count: number }) + case "hour": + return t("{{count}} hours ago", { count: number }) + case "day": + return t("{{count}} days ago", { count: number }) + case "week": + return t("{{count}} weeks ago", { count: number }) + case "month": + return t("{{count}} months ago", { count: number }) + case "year": + return t("{{count}} years ago", { count: number }) + } + } + showDownloadsModal() { const { projectId, surveyId, filter } = this.props @@ -351,7 +375,7 @@ class RespondentIndex extends Component { } downloadItem(id) { - const { t, totalCount, filter } = this.props + const { t, totalCount, filter, respondentsFiles } = this.props const { shownFile } = this.state const currentFile = shownFile == id let item: ?{ @@ -363,8 +387,9 @@ class RespondentIndex extends Component { onDownload: Function, onGenerate: Function, } = null + const fileStatus = currentFile ? respondentsFiles.files?.[id] : null switch (id) { - case "filtered-results": + case "respondents_filtered": item = { title: t("Filtered survey results"), description: t( @@ -376,7 +401,7 @@ class RespondentIndex extends Component { onGenerate: () => this.generateResults(filter), } break - case "results": + case "respondents_results": item = { title: t("Survey results"), description: t( @@ -392,7 +417,7 @@ class RespondentIndex extends Component { onGenerate: () => this.generateResults(), } break - case "disposition-history": + case "disposition_history": item = { title: t("Disposition History"), description: t( @@ -449,25 +474,34 @@ class RespondentIndex extends Component { } else { const disabled = item.disabled - const downloadButton = ( + const fileExists = !!fileStatus?.created_at + + const downloadButtonClass = fileExists ? "black-text" : "grey-text" + const downloadButtonOnClick = fileExists ? item.onDownload : null + const createdAtLabel = fileExists ? : null + + const fileCreating = !!fileStatus?.creating + const generateButtonClass = fileCreating ? "btn-icon-grey" : "black-text" + const generateButtonOnClick = fileCreating ? null : item.onGenerate + const generatingFileLabel = fileCreating ? "Generating..." : "" + + // TODO: we could avoid generating the whole section for files that are not the current one + const downloadButton = !currentFile ? null : (
    - + get_app {t("Download last generated file")} - { // (15 min ago - // FIXME: this should be calculated - } - - - refresh - - - { // ) - } + { createdAtLabel } + { generatingFileLabel } + + + refresh + +
    ) @@ -521,9 +555,9 @@ class RespondentIndex extends Component {

    {t("Choose the data you want to download")}

      - {filter ? this.downloadItem("filtered-results") : null} - {this.downloadItem("results")} - {this.downloadItem("disposition-history")} + {filter ? this.downloadItem("respondents_filtered") : null} + {this.downloadItem("respondents_results")} + {this.downloadItem("disposition_history")} {ownerOrAdmin ? this.downloadItem("incentives") : null} {ownerOrAdmin ? this.downloadItem("interactions") : null}
    @@ -799,7 +833,7 @@ class RespondentIndex extends Component { } const mapStateToProps = (state, ownProps) => { - const { project, survey, questionnaires, respondents } = state + const { project, survey, questionnaires, respondents, respondentsFiles } = state const { page, sortBy, sortAsc, order, filter, items, fields, selectedFields } = respondents const { number: pageNumber, size: pageSize, totalCount } = page const { projectId, surveyId } = ownProps.params @@ -814,6 +848,7 @@ const mapStateToProps = (state, ownProps) => { project: project.data, questionnaires: questionnaires.items, respondents: items, + respondentsFiles, order, userLevel: project.data ? project.data.level : "", pageNumber, diff --git a/locales/template/translation.json b/locales/template/translation.json index 16be0a353..7ca49c888 100644 --- a/locales/template/translation.json +++ b/locales/template/translation.json @@ -597,14 +597,28 @@ "{{count}} collaborator_plural": "", "{{count}} day": "", "{{count}} day_plural": "", + "{{count}} days ago": "", + "{{count}} days ago_plural": "", "{{count}} hour": "", "{{count}} hour_plural": "", + "{{count}} hours ago": "", + "{{count}} hours ago_plural": "", "{{count}} minute": "", "{{count}} minute_plural": "", + "{{count}} minutes ago": "", + "{{count}} minutes ago_plural": "", + "{{count}} months ago": "", + "{{count}} months ago_plural": "", "{{count}} respondent": "", "{{count}} respondent has been added": "", "{{count}} respondent has been added_plural": "", "{{count}} respondent_plural": "", + "{{count}} seconds ago": "", + "{{count}} seconds ago_plural": "", + "{{count}} weeks ago": "", + "{{count}} weeks ago_plural": "", + "{{count}} years ago": "", + "{{count}} years ago_plural": "", "{{exhausted}} exhausted": "", "{{groupName}} ({{count}} contact)": "", "{{groupName}} ({{count}} contact)_plural": "", From d3e1923ee94369bc6f671b304ae69dd9786befec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 31 Oct 2024 19:01:30 -0300 Subject: [PATCH 29/46] Regularly fetch respondent files status See #2350 --- assets/js/components/respondents/RespondentIndex.jsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 47971426c..6f9d15da9 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -60,6 +60,7 @@ type Props = { type State = { shownFile: string, + filesFetchTimer: ?IntervalID, } class RespondentIndex extends Component { @@ -76,7 +77,7 @@ class RespondentIndex extends Component { constructor(props) { super(props) - this.state = { shownFile: null } + this.state = { shownFile: null , filesFetchTimer: null } this.toggleResultsLink = this.toggleResultsLink.bind(this) this.toggleIncentivesLink = this.toggleIncentivesLink.bind(this) this.toggleInteractionsLink = this.toggleInteractionsLink.bind(this) @@ -140,6 +141,12 @@ class RespondentIndex extends Component { // FIXME: don't fetch if we're already fetching this.props.surveyActions.fetchRespondentsFilesStatus(projectId, surveyId, filter) + if (this.state.filesFetchTimer == null) { + const filesFetchTimer = setInterval(() => { + this.props.surveyActions.fetchRespondentsFilesStatus(projectId, surveyId, filter) + }, 20_000); + this.setState({ filesFetchTimer }) + } $('#downloadCSV').modal("open") } From 1b7d32937d917541a908f9b1c66d6a8a0757a03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 5 Nov 2024 10:57:24 -0300 Subject: [PATCH 30/46] Fix SurveyResults tests Small changes, nothing too relevant. See #2350 --- lib/ask/survey_results.ex | 3 +- test/ask/survey_results_test.exs | 51 +++++++++++++++++--------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index e835de770..1cdb4b538 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -383,7 +383,7 @@ defmodule Ask.SurveyResults do seconds_to_process_file = Timex.diff(Timex.now(), initial_datetime, :seconds) Logger.info( - "Generation of #{file_type} file (survey_id: #{survey.id}) took #{seconds_to_process_file} seconds" + "Generation of #{file_prefix(file_type)} file (survey_id: #{survey.id}) took #{seconds_to_process_file} seconds" ) end @@ -430,6 +430,7 @@ defmodule Ask.SurveyResults do defp file_prefix(:interactions), do: "respondents_interactions" defp file_prefix(:incentives), do: "respondents_incentives" defp file_prefix(:disposition_history), do: "disposition_history" + defp file_prefix(:respondents_results), do: "respondents" defp file_prefix({:respondents_results, filter}) do if RespondentsFilter.empty?(filter) do "respondents" diff --git a/test/ask/survey_results_test.exs b/test/ask/survey_results_test.exs index b5963ff90..a51aca33d 100644 --- a/test/ask/survey_results_test.exs +++ b/test/ask/survey_results_test.exs @@ -23,7 +23,7 @@ defmodule Ask.SurveyResultsTest do test "generates empty interactions file" do survey = insert(:survey) assert {:noreply, _, _} = SurveyResults.handle_cast({:interactions, survey.id, nil}, nil) - path = SurveyResults.file_path(survey, :interactions) + path = "./priv/static/" <> SurveyResults.file_path(survey, :interactions) assert "ID,Respondent ID,Mode,Channel,Disposition,Action Type,Action Data,Timestamp\r\n" == File.read!(path) end @@ -136,7 +136,7 @@ defmodule Ask.SurveyResultsTest do end ]) - path = SurveyResults.file_path(survey, :interactions) + path = "./priv/static/" <> SurveyResults.file_path(survey, :interactions) lines = File.read!(path) |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) assert length(lines) == length(expected_list) assert lines == expected_list @@ -192,9 +192,9 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -311,9 +311,9 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent, field_name: "Exercises", value: "No") insert(:response, respondent: respondent, field_name: "Perfect Number", value: "100") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -376,8 +376,8 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -502,8 +502,8 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -600,8 +600,9 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{disposition: :registered}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + filter = %RespondentsFilter{disposition: :registered} + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, {:respondents_results, filter}) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -673,8 +674,9 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{since: Timex.shift(Timex.now(), hours: 2)}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + filter = %RespondentsFilter{since: Timex.shift(Timex.now(), hours: 2)} + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, {:respondents_results, filter}) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -744,8 +746,9 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{state: :completed}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + filter = %RespondentsFilter{state: :completed} + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, {:respondents_results, filter}) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -845,8 +848,8 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_4, field_name: "Smokes", value: "Yes") insert(:response, respondent: respondent_4, field_name: "Exercises", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) assert !String.contains?(group_1.name, [" ", ",", "*", ":", "?", "\\", "|", "/", "<", ">"]) @@ -1001,8 +1004,8 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -1103,8 +1106,8 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_1, field_name: "language", value: "es") - assert {:noreply, _, _} = SurveyResults.handle_cast({:respondent_result, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :respondent_result) + assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) + path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -1174,7 +1177,7 @@ defmodule Ask.SurveyResultsTest do ) assert {:noreply, _, _} = SurveyResults.handle_cast({:disposition_history, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :disposition_history) + path = "./priv/static/" <> SurveyResults.file_path(survey, :disposition_history) csv = File.read!(path) lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) @@ -1236,7 +1239,7 @@ defmodule Ask.SurveyResultsTest do ) assert {:noreply, _, _} = SurveyResults.handle_cast({:incentives, survey.id, %RespondentsFilter{}}, nil) - path = SurveyResults.file_path(survey, :incentives) + path = "./priv/static/" <> SurveyResults.file_path(survey, :incentives) csv = File.read!(path) lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) From 2dd4b5863363945320bfe9e3b93e7b8cc08a3686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 5 Nov 2024 13:12:47 -0300 Subject: [PATCH 31/46] Fixing FlowJS typing There's probably still an issue with react-timeago not being properly ignored yet. See #2350 --- assets/js/actions/survey.js | 2 +- assets/js/components/respondents/RespondentIndex.jsx | 5 +++-- assets/js/decls/survey.js | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/assets/js/actions/survey.js b/assets/js/actions/survey.js index 02b18f2cb..4c508dd56 100644 --- a/assets/js/actions/survey.js +++ b/assets/js/actions/survey.js @@ -369,7 +369,7 @@ export const deleteLink = (link: Link) => ({ link, }) -export const generatingFile = (file: String) => ({ +export const generatingFile = (file: string) => ({ type: GENERATING_FILE, file }) diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 6f9d15da9..5a1951b90 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -56,10 +56,11 @@ type Props = { q: string, fields: Array, selectedFields: Array, + respondentsFiles: { files: SurveyFiles }, } type State = { - shownFile: string, + shownFile: ?string, filesFetchTimer: ?IntervalID, } @@ -485,7 +486,7 @@ class RespondentIndex extends Component { const downloadButtonClass = fileExists ? "black-text" : "grey-text" const downloadButtonOnClick = fileExists ? item.onDownload : null - const createdAtLabel = fileExists ? : null + const createdAtLabel = fileExists ? : null const fileCreating = !!fileStatus?.creating const generateButtonClass = fileCreating ? "btn-icon-grey" : "black-text" diff --git a/assets/js/decls/survey.js b/assets/js/decls/survey.js index 00f9a1ace..a343f64b8 100644 --- a/assets/js/decls/survey.js +++ b/assets/js/decls/survey.js @@ -148,7 +148,7 @@ export type Response = { export type Disposition = null | "completed" | "partial" | "ineligible" export type FileStatus = { - created_at: Date?, + created_at: ?Date, creating: boolean, file_type: string, } From bacac9c3de75b43912ee09e7d3c8149f9db64031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 6 Nov 2024 16:32:58 -0300 Subject: [PATCH 32/46] Ignore Flow-typing react-timeago There are no definitions available. See #2350 --- assets/js/components/respondents/RespondentIndex.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 5a1951b90..717480173 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -2,6 +2,7 @@ import React, { Component } from "react" import { bindActionCreators } from "redux" import { connect } from "react-redux" +// $FlowFixMe - there's no react-timeago definitions in flow-typed import TimeAgo from "react-timeago" import * as api from "../../api" import * as actions from "../../actions/respondents" From 2526d5eac30538525fcf12286f4a5fc91ecc8f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 6 Nov 2024 16:45:24 -0300 Subject: [PATCH 33/46] Remove unused variables Thanks, eslint! See #2350 --- assets/js/components/respondents/RespondentIndex.jsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 717480173..253af800f 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -19,7 +19,6 @@ import { Tooltip, PagingFooter, MainAction, - Action, } from "../ui" import RespondentRow from "./RespondentRow" import * as routes from "../../routes" @@ -755,7 +754,6 @@ class RespondentIndex extends Component { const fixedFieldsCount = this.props.fields.filter((field) => field.type == "fixed").length let colspan = responseKeys.length + fixedFieldsCount - const [fileId, linkId] = ["file", "link"] return (
    From b8fd21cf451d20019f74894654fd1a0c91d6b868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 7 Nov 2024 20:15:00 -0300 Subject: [PATCH 34/46] Drop old FIXMEs They've already been solved or there's nothing to do about them. --- assets/js/actions/survey.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/assets/js/actions/survey.js b/assets/js/actions/survey.js index 4c508dd56..5e142ead9 100644 --- a/assets/js/actions/survey.js +++ b/assets/js/actions/survey.js @@ -479,8 +479,7 @@ export const fetchRespondentsFilesStatus = (projectId: number, surveyId: number, export const generateResultsFile = (projectId: number, surveyId: number, filter?: string) => (dispatch: Function) => { api.generateResults(projectId, surveyId, filter).then((response) => { - // FIXME: periodically check for file generation to have finished - return dispatch(generatingFile("respondent-results")) // FIXME: what should we dispatch? + return dispatch(generatingFile("respondent-results")) }) } @@ -488,21 +487,21 @@ export const generateIncentivesFile = (projectId: number, surveyId: number) => (dispatch: Function) => { // TODO: better handle when incentives download are disabled (due to sample with IDs) api.generateIncentives(projectId, surveyId).then((response) => { - return dispatch(generatingFile("incentives")) // FIXME: what should we dispatch? + return dispatch(generatingFile("incentives")) }) } export const generateInteractionsFile = (projectId: number, surveyId: number) => (dispatch: Function) => { api.generateInteractions(projectId, surveyId).then((response) => { - return dispatch(generatingFile("interactions")) // FIXME: what should we dispatch? + return dispatch(generatingFile("interactions")) }) } export const generateDispositionHistoryFile = (projectId: number, surveyId: number) => (dispatch: Function) => { api.generateDispositionHistory(projectId, surveyId).then((response) => { - return dispatch(generatingFile("disposition-history")) // FIXME: what should we dispatch? + return dispatch(generatingFile("disposition-history")) }) } From e02a40b4d9e21d06c8fa49cfe084aa12f12c6894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 7 Nov 2024 20:52:18 -0300 Subject: [PATCH 35/46] Log file download vs generation in ActivityLog See #2350 --- .../activity/ActivityDescription.jsx | 5 +++++ lib/ask/activity_log.ex | 8 ++++++++ .../controllers/respondent_controller.ex | 20 +++++++------------ locales/template/translation.json | 1 + 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/assets/js/components/activity/ActivityDescription.jsx b/assets/js/components/activity/ActivityDescription.jsx index 9d3c3a2f1..85e198282 100644 --- a/assets/js/components/activity/ActivityDescription.jsx +++ b/assets/js/components/activity/ActivityDescription.jsx @@ -115,6 +115,11 @@ class ActivityDescription extends Component { surveyName: surveyName, reportType: reportType, }) + case "generate_file": + return t("Generated {{surveyName}} {{reportType}} file", { + surveyName: surveyName, + reportType: reportType, + }) case "enable_public_link": return t("Enabled {{surveyName}} {{reportType}} link", { surveyName: surveyName, diff --git a/lib/ask/activity_log.ex b/lib/ask/activity_log.ex index da32cae59..1f13cf437 100644 --- a/lib/ask/activity_log.ex +++ b/lib/ask/activity_log.ex @@ -46,6 +46,7 @@ defmodule Ask.ActivityLog do "request_cancel", "completed_cancel", "download", + "generate_file", "enable_public_link", "regenerate_public_link", "disable_public_link", @@ -194,6 +195,13 @@ defmodule Ask.ActivityLog do }) end + def generate_file(project, conn, survey, report_type) do + create("generate_file", project, conn, survey, %{ + survey_name: survey.name, + report_type: report_type + }) + end + def enable_public_link(project, conn, survey, target_name) do create("enable_public_link", project, conn, survey, %{ survey_name: survey.name, diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 7a3634f65..1a4d62c32 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -763,6 +763,8 @@ defmodule AskWeb.RespondentController do filter = RespondentsFilter.parse(Map.get(params, "q", "")) filter = add_params_to_filter(filter, params) + ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() + file_redirection(conn, survey, {:respondents_results, filter}) end @@ -779,7 +781,7 @@ defmodule AskWeb.RespondentController do SurveyResults.generate_respondents_results_file(survey_id, filter) - ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() + ActivityLog.generate_file(project, conn, survey, "survey_results") |> Repo.insert() conn |> render("ok.json") end @@ -809,6 +811,8 @@ defmodule AskWeb.RespondentController do SurveyResults.generate_disposition_history_file(survey.id) + ActivityLog.generate_file(project, conn, survey, "disposition_history") |> Repo.insert() + conn |> render("ok.json") end @@ -816,8 +820,6 @@ defmodule AskWeb.RespondentController do project = load_project(conn, project_id) survey = load_survey(project, survey_id) - # TODO: We just change this for "trigger generation" - # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "disposition_history") |> Repo.insert() file_redirection(conn, survey, :disposition_history) @@ -834,8 +836,6 @@ defmodule AskWeb.RespondentController do |> where([s], s.incentives_enabled) |> Repo.get!(survey_id) - # TODO: We just change this for "trigger generation" - # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() file_redirection(conn, survey, :incentives) @@ -852,9 +852,7 @@ defmodule AskWeb.RespondentController do |> where([s], s.incentives_enabled) |> Repo.get!(survey_id) - # TODO: We just change this for "trigger generation" - # and add another log when actually downloading? - ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() + ActivityLog.generate_file(project, conn, survey, "incentives") |> Repo.insert() SurveyResults.generate_incentives_file(survey_id) conn |> send_resp(200, "OK") @@ -864,8 +862,6 @@ defmodule AskWeb.RespondentController do project = load_project_for_owner(conn, project_id) survey = load_survey(project, survey_id) - # TODO: We just change this for "trigger generation" - # and add another log when actually downloading? ActivityLog.download(project, conn, survey, "interactions") |> Repo.insert() file_redirection(conn, survey, :interactions) @@ -875,9 +871,7 @@ defmodule AskWeb.RespondentController do project = load_project_for_owner(conn, project_id) survey = load_survey(project, survey_id) - # TODO: We just change this for "trigger generation" - # and add another log when actually downloading? - ActivityLog.download(project, conn, survey, "interactions") |> Repo.insert() + ActivityLog.generate_file(project, conn, survey, "interactions") |> Repo.insert() SurveyResults.generate_interactions_file(survey_id) conn |> send_resp(200, "OK") diff --git a/locales/template/translation.json b/locales/template/translation.json index 7ca49c888..514af57b6 100644 --- a/locales/template/translation.json +++ b/locales/template/translation.json @@ -190,6 +190,7 @@ "Folder": "", "Fri": "", "From": "", + "Generated {{surveyName}} {{reportType}} file": "", "Ignored Values": "", "Import questionnaire": "", "Incentive download was disabled because respondent ids were uploaded": "", From d95502b01eb2ad98ba04749b65446a1ac651b737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Tue, 12 Nov 2024 16:36:51 -0300 Subject: [PATCH 36/46] Style Download CSV files modal See #2350 --- assets/css/_card-modal.scss | 26 +++++++++++++++++-- .../respondents/RespondentIndex.jsx | 7 ++--- .../controllers/respondent_controller.ex | 15 +++++------ locales/template/translation.json | 1 + 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/assets/css/_card-modal.scss b/assets/css/_card-modal.scss index 8bbf58b20..f74da9e9b 100644 --- a/assets/css/_card-modal.scss +++ b/assets/css/_card-modal.scss @@ -120,8 +120,13 @@ display: flex; align-items: center; .switch { - display: inline-block; - vertical-align: middle; + display: flex; + align-items: center; + gap: 0.5rem; + label { + display: flex; + align-items: center; + } .lever { margin-left: 0; } @@ -129,6 +134,8 @@ @extend .grey-text; font-size: 1rem; margin-right: 0.5rem; + display: flex; + white-space: nowrap; } } .link { @@ -144,6 +151,7 @@ vertical-align: middle; width: 98%; overflow: hidden; + text-overflow: ellipsis; display: inline-block; } .buttons { @@ -171,6 +179,20 @@ .btn-icon-grey { margin-left: initial; } + + .file-generation { + display: flex; + align-items: center; + gap: 0.5rem; + } + + a { + display: inline-flex; + } + + .material-icons { + display: inline-flex; + } } div.link { @extend .grey-text; diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 253af800f..b15dc6fa5 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -484,25 +484,26 @@ class RespondentIndex extends Component { const fileExists = !!fileStatus?.created_at + const downloadButtonTooltip = fileExists ? t("Download file") : t("File not yet generated") const downloadButtonClass = fileExists ? "black-text" : "grey-text" const downloadButtonOnClick = fileExists ? item.onDownload : null const createdAtLabel = fileExists ? : null const fileCreating = !!fileStatus?.creating - const generateButtonClass = fileCreating ? "btn-icon-grey" : "black-text" + const generateButtonClass = fileCreating ? "grey-text" : "black-text" const generateButtonOnClick = fileCreating ? null : item.onGenerate const generatingFileLabel = fileCreating ? "Generating..." : "" // TODO: we could avoid generating the whole section for files that are not the current one const downloadButton = !currentFile ? null : (
    - + get_app {t("Download last generated file")} - + { createdAtLabel } { generatingFileLabel } diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 1a4d62c32..48663938b 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -748,12 +748,11 @@ defmodule AskWeb.RespondentController do render(conn, "status.json", status: status) end - defp file_redirection(conn, survey, file_type) do + defp serve_file(conn, survey, file_type) do file_url = SurveyResults.file_path(survey, file_type) - render(conn, "file-redirect.json", - file_url: "/#{file_url}" # TODO: there may be better ways of avoiding relative URLs - ) + # FIME: /priv/static is probably temporary + send_file(conn, 200, "./priv/static/" <> file_url) end def results_csv(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do @@ -765,7 +764,7 @@ defmodule AskWeb.RespondentController do ActivityLog.download(project, conn, survey, "survey_results") |> Repo.insert() - file_redirection(conn, survey, {:respondents_results, filter}) + serve_file(conn, survey, {:respondents_results, filter}) end def generate_results(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do @@ -822,7 +821,7 @@ defmodule AskWeb.RespondentController do ActivityLog.download(project, conn, survey, "disposition_history") |> Repo.insert() - file_redirection(conn, survey, :disposition_history) + serve_file(conn, survey, :disposition_history) end def incentives(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do @@ -838,7 +837,7 @@ defmodule AskWeb.RespondentController do ActivityLog.download(project, conn, survey, "incentives") |> Repo.insert() - file_redirection(conn, survey, :incentives) + serve_file(conn, survey, :incentives) end def generate_incentives(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do @@ -864,7 +863,7 @@ defmodule AskWeb.RespondentController do ActivityLog.download(project, conn, survey, "interactions") |> Repo.insert() - file_redirection(conn, survey, :interactions) + serve_file(conn, survey, :interactions) end def generate_interactions(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do diff --git a/locales/template/translation.json b/locales/template/translation.json index 514af57b6..8ecc86513 100644 --- a/locales/template/translation.json +++ b/locales/template/translation.json @@ -184,6 +184,7 @@ "Fallback delay": "", "Fallback delay is invalid": "", "Fallback mode": "", + "File not yet generated": "", "Filter using the following format": "", "Filtered survey results": "", "Flag": "", From d3e967d2c37e72f19ac4b3b8aa0e48b3c0bf0dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 13 Nov 2024 17:53:36 -0300 Subject: [PATCH 37/46] Access control generated CSV files They were previously placed in a public directory, statically served. We now apply the usual access control we have for generation (and had for streaming). See #2350 --- assets/js/api.js | 19 ----------- .../respondents/RespondentIndex.jsx | 34 ++++++++++--------- lib/ask/survey_results.ex | 6 ++-- .../controllers/respondent_controller.ex | 4 +-- lib/ask_web/router.ex | 4 --- test/ask/survey_results_test.exs | 28 +++++++-------- 6 files changed, 37 insertions(+), 58 deletions(-) diff --git a/assets/js/api.js b/assets/js/api.js index b840541f2..886b2b113 100644 --- a/assets/js/api.js +++ b/assets/js/api.js @@ -691,25 +691,6 @@ export const refreshDispositionHistoryLink = (projectId, surveyId) => { ) } -const downloadGeneratedFile = (sourceUrl) => - apiFetch(sourceUrl, null, null).then((response) => { - response.json().then((data) => { - window.open(data.url) - }) - } - ) - -export const downloadRespondentsResultsFile = (projectId, surveyId, q) => - downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/results_csv?${ - (q && `&q=${encodeURIComponent(q)}`) || "" - }`) -export const downloadRespondentsDispositionHistoryFile = (projectId, surveyId) => - downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/disposition_history`) -export const downloadRespondentsIncentivesFile = (projectId, surveyId) => - downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/incentives`) -export const downloadRespondentsInteractionsFile = (projectId, surveyId) => - downloadGeneratedFile(`projects/${projectId}/surveys/${surveyId}/respondents/interactions`) - export const generateResults = (projectId, surveyId, filter) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/results`, null, { q: filter }) export const generateDispositionHistory = (projectId, surveyId) => diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index b15dc6fa5..039f15273 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -161,25 +161,27 @@ class RespondentIndex extends Component { this.fetchRespondents(pageNumber - 1) } - downloadResultsCSV(applyUserFilter = false) { + resultsFileUrl(applyUserFilter = false) { const { projectId, surveyId, filter } = this.props const q = (applyUserFilter && filter) || null - api.downloadRespondentsResultsFile(projectId, surveyId, q) + return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/results_csv${ + (q && `?q=${encodeURIComponent(q)}`) || "" + }` } - downloadDispositionHistoryCSV() { + dispositionHistoryFileUrl() { const { projectId, surveyId } = this.props - api.downloadRespondentsDispositionHistoryFile(projectId, surveyId) + return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/disposition_history` } - downloadIncentivesCSV() { + incentivesFileUrl() { const { projectId, surveyId } = this.props - api.downloadRespondentsIncentivesFile(projectId, surveyId) + return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/incentives` } - downloadInteractionsCSV() { + interactionsFileUrl() { const { projectId, surveyId } = this.props - api.downloadRespondentsInteractionsFile(projectId, surveyId) + return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/interactions` } sortBy(name) { @@ -392,7 +394,7 @@ class RespondentIndex extends Component { disabledText?: String, disabled?: boolean, downloadLink: any, - onDownload: Function, + fileUrl: string, onGenerate: Function, } = null const fileStatus = currentFile ? respondentsFiles.files?.[id] : null @@ -405,7 +407,7 @@ class RespondentIndex extends Component { { totalCount, filter } ), downloadLink: null, - onDownload: () => this.downloadResultsCSV(true), + fileUrl: this.resultsFileUrl(true), onGenerate: () => this.generateResults(filter), } break @@ -421,7 +423,7 @@ class RespondentIndex extends Component { this.refreshResultsLink, "resultsLink" ), - onDownload: () => this.downloadResultsCSV(), + fileUrl: this.resultsFileUrl(), onGenerate: () => this.generateResults(), } break @@ -437,7 +439,7 @@ class RespondentIndex extends Component { this.refreshDispositionHistoryLink, "dispositionHistoryLink" ), - onDownload: () => this.downloadDispositionHistoryCSV(), + fileUrl: this.dispositionHistoryFileUrl(), onGenerate: () => this.generateDispositionHistory(), } break @@ -456,7 +458,7 @@ class RespondentIndex extends Component { this.refreshIncentivesLink, "incentivesLink" ), - onDownload: () => this.downloadIncentivesCSV(), + fileUrl: this.incentivesFileUrl(), onGenerate: () => this.generateIncentives(), } break @@ -472,7 +474,7 @@ class RespondentIndex extends Component { this.refreshInteractionsLink, "interactionsLink" ), - onDownload: () => this.downloadInteractionsCSV(), + fileUrl: this.interactionsFileUrl(), onGenerate: () => this.generateInteractions(), } } @@ -486,7 +488,7 @@ class RespondentIndex extends Component { const downloadButtonTooltip = fileExists ? t("Download file") : t("File not yet generated") const downloadButtonClass = fileExists ? "black-text" : "grey-text" - const downloadButtonOnClick = fileExists ? item.onDownload : null + const downloadButtonLink = fileExists ? item.fileUrl : null const createdAtLabel = fileExists ? : null const fileCreating = !!fileStatus?.creating @@ -498,7 +500,7 @@ class RespondentIndex extends Component { const downloadButton = !currentFile ? null : (
    - + get_app diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index 1cdb4b538..f12a605d8 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -364,8 +364,8 @@ defmodule Ask.SurveyResults do end defp write_to_file(file_type, survey, rows) do - File.mkdir_p!("./priv/static/" <> @target_dir) - target_file = "./priv/static/" <> file_path(survey, file_type, @target_dir) + File.mkdir_p!(@target_dir) + target_file = file_path(survey, file_type, @target_dir) if File.exists?(target_file), do: File.rm!(target_file) # Poor man's mktemp. We only want to avoid having the file living at the stable @@ -418,7 +418,7 @@ defmodule Ask.SurveyResults do defp file_type_symbol(type), do: type defp file_status(survey, file_type) do - path = "./priv/static/" <> file_path(survey, file_type) + path = file_path(survey, file_type) exists = File.exists?(path) %{ file_type: file_type_symbol(file_type), diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 48663938b..3dfa0b6f5 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -751,8 +751,8 @@ defmodule AskWeb.RespondentController do defp serve_file(conn, survey, file_type) do file_url = SurveyResults.file_path(survey, file_type) - # FIME: /priv/static is probably temporary - send_file(conn, 200, "./priv/static/" <> file_url) + conn + |> send_download({:file, file_url}) end def results_csv(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do diff --git a/lib/ask_web/router.ex b/lib/ask_web/router.ex index 498a325b6..0a34f3407 100644 --- a/lib/ask_web/router.ex +++ b/lib/ask_web/router.ex @@ -11,10 +11,6 @@ defmodule AskWeb.Router do plug :protect_from_forgery plug :put_secure_browser_headers plug Plug.Static, at: "files/", from: "assets/static/files/" - - # FIXME: this is temporary. We have to apply access control here: https://elixirforum.com/t/how-can-you-hide-certain-static-assets-behind-a-require-authenticated-user-path/53106/6 - plug Plug.Static, at: "generated_files/", from: "priv/static/generated_files/" - plug Coherence.Authentication.Session, db_model: Ask.User end diff --git a/test/ask/survey_results_test.exs b/test/ask/survey_results_test.exs index a51aca33d..ac0c33cad 100644 --- a/test/ask/survey_results_test.exs +++ b/test/ask/survey_results_test.exs @@ -23,7 +23,7 @@ defmodule Ask.SurveyResultsTest do test "generates empty interactions file" do survey = insert(:survey) assert {:noreply, _, _} = SurveyResults.handle_cast({:interactions, survey.id, nil}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :interactions) + path = SurveyResults.file_path(survey, :interactions) assert "ID,Respondent ID,Mode,Channel,Disposition,Action Type,Action Data,Timestamp\r\n" == File.read!(path) end @@ -136,7 +136,7 @@ defmodule Ask.SurveyResultsTest do end ]) - path = "./priv/static/" <> SurveyResults.file_path(survey, :interactions) + path = SurveyResults.file_path(survey, :interactions) lines = File.read!(path) |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) assert length(lines) == length(expected_list) assert lines == expected_list @@ -194,7 +194,7 @@ defmodule Ask.SurveyResultsTest do assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) + path = SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -313,7 +313,7 @@ defmodule Ask.SurveyResultsTest do assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) + path = SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -377,7 +377,7 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) + path = SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -503,7 +503,7 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) + path = SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -602,7 +602,7 @@ defmodule Ask.SurveyResultsTest do filter = %RespondentsFilter{disposition: :registered} assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, {:respondents_results, filter}) + path = SurveyResults.file_path(survey, {:respondents_results, filter}) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -676,7 +676,7 @@ defmodule Ask.SurveyResultsTest do filter = %RespondentsFilter{since: Timex.shift(Timex.now(), hours: 2)} assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, {:respondents_results, filter}) + path = SurveyResults.file_path(survey, {:respondents_results, filter}) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -748,7 +748,7 @@ defmodule Ask.SurveyResultsTest do filter = %RespondentsFilter{state: :completed} assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, {:respondents_results, filter}) + path = SurveyResults.file_path(survey, {:respondents_results, filter}) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -849,7 +849,7 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_4, field_name: "Exercises", value: "No") assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) + path = SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) assert !String.contains?(group_1.name, [" ", ",", "*", ":", "?", "\\", "|", "/", "<", ">"]) @@ -1005,7 +1005,7 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_2, field_name: "Smokes", value: "No") assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) + path = SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, line3, _] = csv |> String.split("\r\n") @@ -1107,7 +1107,7 @@ defmodule Ask.SurveyResultsTest do insert(:response, respondent: respondent_1, field_name: "language", value: "es") assert {:noreply, _, _} = SurveyResults.handle_cast({:respondents_results, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :respondents_results) + path = SurveyResults.file_path(survey, :respondents_results) csv = File.read!(path) [line1, line2, _] = csv |> String.split("\r\n") @@ -1177,7 +1177,7 @@ defmodule Ask.SurveyResultsTest do ) assert {:noreply, _, _} = SurveyResults.handle_cast({:disposition_history, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :disposition_history) + path = SurveyResults.file_path(survey, :disposition_history) csv = File.read!(path) lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) @@ -1239,7 +1239,7 @@ defmodule Ask.SurveyResultsTest do ) assert {:noreply, _, _} = SurveyResults.handle_cast({:incentives, survey.id, %RespondentsFilter{}}, nil) - path = "./priv/static/" <> SurveyResults.file_path(survey, :incentives) + path = SurveyResults.file_path(survey, :incentives) csv = File.read!(path) lines = csv |> String.split("\r\n") |> Enum.reject(fn x -> String.length(x) == 0 end) From b04f0650cdece1d5c1499daee4db1a26acf7e30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 13 Nov 2024 18:17:30 -0300 Subject: [PATCH 38/46] Move API routes concerns back to api.js Those URLs in the RespondentIndex.jsx smelled. See #2350 --- assets/js/api.js | 11 +++++++++++ .../components/respondents/RespondentIndex.jsx | 16 +++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/assets/js/api.js b/assets/js/api.js index 886b2b113..2dfe645d8 100644 --- a/assets/js/api.js +++ b/assets/js/api.js @@ -691,6 +691,17 @@ export const refreshDispositionHistoryLink = (projectId, surveyId) => { ) } +export const resultsFileUrl = (projectId, surveyId, q) => + `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/results_csv${ + (q && `?q=${encodeURIComponent(q)}`) || "" + }` +export const dispositionHistoryFileUrl = (projectId, surveyId) => + `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/disposition_history` +export const incentivesFileUrl = (projectId, surveyId) => + `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/incentives` +export const interactionsFileUrl = (projectId, surveyId) => + `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/interactions` + export const generateResults = (projectId, surveyId, filter) => apiPostJSON(`projects/${projectId}/surveys/${surveyId}/respondents/results`, null, { q: filter }) export const generateDispositionHistory = (projectId, surveyId) => diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 039f15273..572834edc 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -164,24 +164,22 @@ class RespondentIndex extends Component { resultsFileUrl(applyUserFilter = false) { const { projectId, surveyId, filter } = this.props const q = (applyUserFilter && filter) || null - return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/results_csv${ - (q && `?q=${encodeURIComponent(q)}`) || "" - }` + return api.resultsFileUrl(projectId, surveyId, q) } - + dispositionHistoryFileUrl() { const { projectId, surveyId } = this.props - return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/disposition_history` + return api.dispositionHistoryFileUrl(projectId, surveyId) } - + incentivesFileUrl() { const { projectId, surveyId } = this.props - return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/incentives` + return api.incentivesFileUrl(projectId, surveyId) } - + interactionsFileUrl() { const { projectId, surveyId } = this.props - return `/api/v1/projects/${projectId}/surveys/${surveyId}/respondents/interactions` + return api.interactionsFileUrl(projectId, surveyId) } sortBy(name) { From bb8a8acc92c7dc76f12a20de402b579efda3f45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 13 Nov 2024 18:45:08 -0300 Subject: [PATCH 39/46] Fix: filter was not updated in file status queries Due to the way the `q` variable was bound in the previous implementation of the `setInterval` call, we were always querying the file status using the filter (or lack thereof) that was in place in the first show of the modal. We now get the current value of `q` at the time in which the refresh function is called. See #2350 --- assets/js/components/respondents/RespondentIndex.jsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 572834edc..6c6e066a7 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -137,14 +137,18 @@ class RespondentIndex extends Component { } } - showDownloadsModal() { + fetchFilesStatus() { const { projectId, surveyId, filter } = this.props // FIXME: don't fetch if we're already fetching this.props.surveyActions.fetchRespondentsFilesStatus(projectId, surveyId, filter) + } + + showDownloadsModal() { + this.fetchFilesStatus() if (this.state.filesFetchTimer == null) { const filesFetchTimer = setInterval(() => { - this.props.surveyActions.fetchRespondentsFilesStatus(projectId, surveyId, filter) + this.fetchFilesStatus() }, 20_000); this.setState({ filesFetchTimer }) } From d5ceb9ad68b25fcf89d2380b89f47ed1ab76d937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 13 Nov 2024 19:13:26 -0300 Subject: [PATCH 40/46] Unskip survey links tests They were already working - not sure when they were fixed but that's fine. See #2350 --- test/ask_web/controllers/survey_link_controller_test.exs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/ask_web/controllers/survey_link_controller_test.exs b/test/ask_web/controllers/survey_link_controller_test.exs index d9699923b..9178f7f5c 100644 --- a/test/ask_web/controllers/survey_link_controller_test.exs +++ b/test/ask_web/controllers/survey_link_controller_test.exs @@ -62,7 +62,6 @@ defmodule AskWeb.SurveyLinkControllerTest do assert [] == ShortLink |> Repo.all() end - @tag :skip test "incentives link generation", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -110,7 +109,6 @@ defmodule AskWeb.SurveyLinkControllerTest do assert [] == ShortLink |> Repo.all() end - @tag :skip test "interactions link generation", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -158,7 +156,6 @@ defmodule AskWeb.SurveyLinkControllerTest do assert [] == ShortLink |> Repo.all() end - @tag :skip test "disposition_history link generation", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -362,7 +359,6 @@ defmodule AskWeb.SurveyLinkControllerTest do end end - @tag :skip test "allows editors to create some links", %{conn: conn, user: user} do project = create_project_for_user(user, level: "editor") survey = insert(:survey, project: project) @@ -387,7 +383,6 @@ defmodule AskWeb.SurveyLinkControllerTest do end end - @tag :skip test "allows editors to refresh some links", %{conn: conn, user: user} do project = create_project_for_user(user, level: "editor") survey = insert(:survey, project: project) @@ -414,7 +409,6 @@ defmodule AskWeb.SurveyLinkControllerTest do end end - @tag :skip test "forbids editor to delete some links", %{conn: conn, user: user} do project = create_project_for_user(user, level: "editor") survey = insert(:survey, project: project) @@ -501,7 +495,6 @@ defmodule AskWeb.SurveyLinkControllerTest do }) end - @tag :skip test "generates logs for incentives link", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -552,7 +545,6 @@ defmodule AskWeb.SurveyLinkControllerTest do }) end - @tag :skip test "generates logs for interactions link", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) @@ -603,7 +595,6 @@ defmodule AskWeb.SurveyLinkControllerTest do }) end - @tag :skip test "generates logs for disposition_history link", %{conn: conn, user: user} do project = create_project_for_user(user) survey = insert(:survey, project: project) From 6c1d42e52a2dd837a1b19c508962193389256a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 14 Nov 2024 16:32:52 -0300 Subject: [PATCH 41/46] Respond a 404 for files not yet generated We were raising an exception instead. See #2350 --- lib/ask_web/controllers/respondent_controller.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex index 3dfa0b6f5..ede16813f 100644 --- a/lib/ask_web/controllers/respondent_controller.ex +++ b/lib/ask_web/controllers/respondent_controller.ex @@ -749,12 +749,20 @@ defmodule AskWeb.RespondentController do end defp serve_file(conn, survey, file_type) do - file_url = SurveyResults.file_path(survey, file_type) + file_path = SurveyResults.file_path(survey, file_type) conn - |> send_download({:file, file_url}) + |> send_download_if_exists(file_path, File.exists?(file_path)) end + defp send_download_if_exists(conn, file_path, true), do: + conn + |> send_download({:file, file_path}) + + defp send_download_if_exists(conn, _file_path, false), do: + conn + |> send_resp(404, "File not found") + def results_csv(conn, %{"project_id" => project_id, "survey_id" => survey_id} = params) do project = load_project(conn, project_id) survey = load_survey(project, survey_id) From c9ba84ba8f4c84028608b557b229a769fc56d8eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 14 Nov 2024 17:34:09 -0300 Subject: [PATCH 42/46] Fix: short links point to the right file downloads The URLs where broken when splitting generation and download. See #2350 --- .../controllers/survey_link_controller.ex | 16 ++++------------ lib/ask_web/router.ex | 2 +- .../controllers/survey_link_controller_test.exs | 8 ++++---- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/lib/ask_web/controllers/survey_link_controller.ex b/lib/ask_web/controllers/survey_link_controller.ex index d98a55a95..a9ad0cfc0 100644 --- a/lib/ask_web/controllers/survey_link_controller.ex +++ b/lib/ask_web/controllers/survey_link_controller.ex @@ -16,13 +16,10 @@ defmodule AskWeb.SurveyLinkController do {name, target} = case target_name do - # FIXME: revisit this. We should remove _format "results" -> { Survey.link_name(survey, :results), - project_survey_get_respondents_results_path(conn, :results, project, survey, %{ - "_format" => "csv" - }) + project_survey_respondents_results_path(conn, :results_csv, project, survey) } "incentives" -> @@ -30,9 +27,7 @@ defmodule AskWeb.SurveyLinkController do { Survey.link_name(survey, :incentives), - project_survey_respondents_incentives_path(conn, :incentives, project, survey, %{ - "_format" => "csv" - }) + project_survey_respondents_incentives_path(conn, :incentives, project, survey) } "interactions" -> @@ -40,9 +35,7 @@ defmodule AskWeb.SurveyLinkController do { Survey.link_name(survey, :interactions), - project_survey_respondents_interactions_path(conn, :interactions, project, survey, %{ - "_format" => "csv" - }) + project_survey_respondents_interactions_path(conn, :interactions, project, survey) } "disposition_history" -> @@ -52,8 +45,7 @@ defmodule AskWeb.SurveyLinkController do conn, :disposition_history, project, - survey, - %{"_format" => "csv"} + survey ) } diff --git a/lib/ask_web/router.ex b/lib/ask_web/router.ex index 0a34f3407..291792d83 100644 --- a/lib/ask_web/router.ex +++ b/lib/ask_web/router.ex @@ -195,7 +195,7 @@ defmodule AskWeb.Router do get "/files", RespondentController, :files_status, as: :files get "/results", RespondentController, :results, as: :get_respondents_results - get "/results_csv", RespondentController, :results_csv + get "/results_csv", RespondentController, :results_csv, as: :respondents_results post "/results", RespondentController, :generate_results, as: :respondents_results get "/disposition_history", RespondentController, :disposition_history, as: :respondents_disposition_history diff --git a/test/ask_web/controllers/survey_link_controller_test.exs b/test/ask_web/controllers/survey_link_controller_test.exs index 9178f7f5c..842c1cfb7 100644 --- a/test/ask_web/controllers/survey_link_controller_test.exs +++ b/test/ask_web/controllers/survey_link_controller_test.exs @@ -32,7 +32,7 @@ defmodule AskWeb.SurveyLinkControllerTest do } assert link.target == - "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/results?_format=csv" + "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/results_csv" response = get(conn, project_survey_links_path(conn, :create, project, survey, "results")) @@ -87,7 +87,7 @@ defmodule AskWeb.SurveyLinkControllerTest do assert ShortLink |> Repo.all() |> length == 1 assert link.target == - "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/incentives?_format=csv" + "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/incentives" response = put(conn, project_survey_links_path(conn, :refresh, project, survey, "incentives")) @@ -134,7 +134,7 @@ defmodule AskWeb.SurveyLinkControllerTest do assert ShortLink |> Repo.all() |> length == 1 assert link.target == - "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/interactions?_format=csv" + "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/interactions" response = put(conn, project_survey_links_path(conn, :refresh, project, survey, "interactions")) @@ -174,7 +174,7 @@ defmodule AskWeb.SurveyLinkControllerTest do } assert link.target == - "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/disposition_history?_format=csv" + "/api/v1/projects/#{project.id}/surveys/#{survey.id}/respondents/disposition_history" response = get( From 63a3d361ff43f2a30d80186a06be6ec675770b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Wed, 4 Dec 2024 17:27:24 -0300 Subject: [PATCH 43/46] Honour results filter when generating file See #2350 --- lib/ask/survey_results.ex | 8 +++++++- test/ask/survey_results_test.exs | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index f12a605d8..2587f6747 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -427,6 +427,12 @@ defmodule Ask.SurveyResults do } end + defp filter_hash(filter) do + filter + |> :erlang.phash2 + |> to_string + end + defp file_prefix(:interactions), do: "respondents_interactions" defp file_prefix(:incentives), do: "respondents_incentives" defp file_prefix(:disposition_history), do: "disposition_history" @@ -435,7 +441,7 @@ defmodule Ask.SurveyResults do if RespondentsFilter.empty?(filter) do "respondents" else - "respondents_filtered" + "respondents_filtered_#{filter_hash(filter)}" end end diff --git a/test/ask/survey_results_test.exs b/test/ask/survey_results_test.exs index ac0c33cad..b807f1ba7 100644 --- a/test/ask/survey_results_test.exs +++ b/test/ask/survey_results_test.exs @@ -274,6 +274,32 @@ defmodule Ask.SurveyResultsTest do assert line_3_user_stopped == "true" end + test "generates different filtered results csv per filter" do + survey = insert(:survey) + + filter = RespondentsFilter.parse("") + + assert %{files: %{respondents_results: %{created_at: nil}}} = SurveyResults.files_status(survey, [{:respondents_results, filter}]) + + SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) + assert %{files: %{respondents_results: %{created_at: date}}} = SurveyResults.files_status(survey, [{:respondents_results, filter}]) + assert date != nil + + filter = RespondentsFilter.parse("disposition:queued") + assert %{files: %{respondents_filtered: %{created_at: nil}}} = SurveyResults.files_status(survey, [{:respondents_results, filter}]) + SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) + assert %{files: %{respondents_filtered: %{created_at: queued_date}}} = SurveyResults.files_status(survey, [{:respondents_results, filter}]) + assert queued_date != nil + + filter = RespondentsFilter.parse("mode:sms") + assert %{files: %{respondents_filtered: %{created_at: nil}}} = SurveyResults.files_status(survey, [{:respondents_results, filter}]) + SurveyResults.handle_cast({:respondents_results, survey.id, filter}, nil) + assert %{files: %{respondents_filtered: %{created_at: mode_date}}} = SurveyResults.files_status(survey, [{:respondents_results, filter}]) + assert mode_date != nil + + assert %{files: %{respondents_filtered: %{created_at: ^queued_date}}} = SurveyResults.files_status(survey, [{:respondents_results, RespondentsFilter.parse("disposition:queued")}]) + end + test "download results csv with non-started last call" do project = insert(:project) questionnaire = insert(:questionnaire, name: "test", project: project, steps: @dummy_steps) From b3414c32c47c58a1417fd0d47ea3c1a9aba47c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Thu, 5 Dec 2024 19:12:02 -0300 Subject: [PATCH 44/46] Nitpick: renames and comments from the PR See #2362 See #2350 --- assets/js/actions/survey.js | 1 - lib/ask/survey_results.ex | 33 +++++++++++++++------------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/assets/js/actions/survey.js b/assets/js/actions/survey.js index 5e142ead9..218b5a356 100644 --- a/assets/js/actions/survey.js +++ b/assets/js/actions/survey.js @@ -485,7 +485,6 @@ export const generateResultsFile = export const generateIncentivesFile = (projectId: number, surveyId: number) => (dispatch: Function) => { - // TODO: better handle when incentives download are disabled (due to sample with IDs) api.generateIncentives(projectId, surveyId).then((response) => { return dispatch(generatingFile("incentives")) }) diff --git a/lib/ask/survey_results.ex b/lib/ask/survey_results.ex index 2587f6747..f021dc3d8 100644 --- a/lib/ask/survey_results.ex +++ b/lib/ask/survey_results.ex @@ -353,14 +353,8 @@ defmodule Ask.SurveyResults do end def file_path(survey, file_type, target_dir \\ @target_dir) do - # FIXME: as a first iteration, generate a stable name and have a single file per type - # but we should probably include the date and respondent results filter in the name - prefix = file_prefix(file_type) - # name = survey.name || "survey_id_#{survey.id}" - # name = Regex.replace(~r/[^a-zA-Z0-9_]/, name, "_") - # current_time = Timex.format!(DateTime.utc_now(), "%Y-%m-%d-%H-%M-%S", :strftime) - # "#{target_dir}/#{name}_#{survey.state}-#{prefix}_#{current_time}.csv" - "#{target_dir}/survey_#{survey.id}-#{survey.state}-#{prefix}.csv" + suffix = file_suffix(file_type) + "#{target_dir}/survey_#{survey.id}-#{survey.state}-#{suffix}.csv" end defp write_to_file(file_type, survey, rows) do @@ -370,20 +364,20 @@ defmodule Ask.SurveyResults do # Poor man's mktemp. We only want to avoid having the file living at the stable # path while it's still being written to avoid partial downloads - temporal_file = for _ <- 1..10, into: "#{target_file}.tmp.", do: <> - file = File.open!(temporal_file, [:write, :utf8]) + temporal_file_name = for _ <- 1..10, into: "#{target_file}.tmp.", do: <> + temporal_file = File.open!(temporal_file_name, [:write, :utf8]) initial_datetime = Timex.now() rows |> CSV.encode() - |> Enum.each(&IO.write(file, &1)) + |> Enum.each(&IO.write(temporal_file, &1)) - File.rename!(temporal_file, target_file) + File.rename!(temporal_file_name, target_file) seconds_to_process_file = Timex.diff(Timex.now(), initial_datetime, :seconds) Logger.info( - "Generation of #{file_prefix(file_type)} file (survey_id: #{survey.id}) took #{seconds_to_process_file} seconds" + "Generation of #{file_suffix(file_type)} file (survey_id: #{survey.id}) took #{seconds_to_process_file} seconds" ) end @@ -433,14 +427,17 @@ defmodule Ask.SurveyResults do |> to_string end - defp file_prefix(:interactions), do: "respondents_interactions" - defp file_prefix(:incentives), do: "respondents_incentives" - defp file_prefix(:disposition_history), do: "disposition_history" - defp file_prefix(:respondents_results), do: "respondents" - defp file_prefix({:respondents_results, filter}) do + defp file_suffix(:interactions), do: "respondents_interactions" + defp file_suffix(:incentives), do: "respondents_incentives" + defp file_suffix(:disposition_history), do: "disposition_history" + defp file_suffix(:respondents_results), do: "respondents" + defp file_suffix({:respondents_results, filter}) do if RespondentsFilter.empty?(filter) do "respondents" else + # Hashing the filter object avoids dealing with the order of the params in the filter + # (ie, "disposition:queued mode:sms" vs "mode:sms disposition:queued"), and avoids `:` + # and ` ` characters in the file name "respondents_filtered_#{filter_hash(filter)}" end end From 6ad6f921c75c836d756d5c6fe02db5b5853f71f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Mon, 9 Dec 2024 18:24:34 -0300 Subject: [PATCH 45/46] Fix: avoid fetching file status after navigating Without invalidating the timer, navigating from one survey to another would keep the status for both surveys' files to be fetched, yielding bugs in the UI. See #2350 See #2362 --- assets/js/components/respondents/RespondentIndex.jsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/assets/js/components/respondents/RespondentIndex.jsx b/assets/js/components/respondents/RespondentIndex.jsx index 6c6e066a7..d5e63a705 100644 --- a/assets/js/components/respondents/RespondentIndex.jsx +++ b/assets/js/components/respondents/RespondentIndex.jsx @@ -102,6 +102,14 @@ class RespondentIndex extends Component { } } + componentWillUnmount() { + const timer = this.state.filesFetchTimer + if (timer) { + clearInterval(timer) + this.setState({filesFetchTimer: null}) + } + } + fetchRespondents(pageNumber = 1, overrideFilter = null) { const { projectId, surveyId, pageSize, filter, sortBy, sortAsc } = this.props const _filter = overrideFilter == null ? filter : overrideFilter From 9b2b9c54a97e53358d745b3581e933720335a066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Garc=C3=ADa=20Isa=C3=ADa?= Date: Mon, 9 Dec 2024 20:37:11 -0300 Subject: [PATCH 46/46] Mouse pointer on file generation button See #2350 --- assets/css/_card-modal.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/css/_card-modal.scss b/assets/css/_card-modal.scss index f74da9e9b..828b8ddaa 100644 --- a/assets/css/_card-modal.scss +++ b/assets/css/_card-modal.scss @@ -188,6 +188,7 @@ a { display: inline-flex; + cursor: pointer; } .material-icons {