From 8a397a20e15c69fe6699d4d40ae37f564b76d5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Wed, 11 Dec 2024 11:04:42 +0100 Subject: [PATCH] =?UTF-8?q?Validateur=20NeTEx=20:=20polling=20des=20r?= =?UTF-8?q?=C3=A9sultats=20(#4326)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "Temporaire : désactivation validation NeTEx (#4295)" This reverts commit f24f1e5898699b1be18904bc97e40ec4ce88d7b6. * Polling résultats validation enRoute * Commentaires de review * Evitons que le snooze ne retarde les retry légitimes * Lien vers de la doc Oban * Poller NeTEx: contrainte d'unicité --- apps/transport/lib/jobs/netex_poller_job.ex | 46 +++++ .../lib/jobs/on_demand_netex_poller_job.ex | 91 +++++++++ .../lib/jobs/on_demand_validation_helpers.ex | 41 ++++ .../lib/jobs/on_demand_validation_job.ex | 89 ++++----- .../controllers/resource_controller.ex | 3 +- .../lib/validators/netex_validator.ex | 172 ++++++++++------- .../lib/validators/validator_selection.ex | 2 +- .../enroute_chouette_valid_client_helpers.ex | 14 +- .../transport/jobs/netex_poller_job_test.exs | 175 ++++++++++++++++++ .../jobs/on_demand_netex_poller_job_test.exs | 156 ++++++++++++++++ .../jobs/on_demand_validation_job_test.exs | 40 +++- .../validators/netex_validator_test.exs | 86 ++++----- .../validators/validator_selection_test.exs | 4 +- .../controllers/resource_controller_test.exs | 95 +++++----- 14 files changed, 788 insertions(+), 226 deletions(-) create mode 100644 apps/transport/lib/jobs/netex_poller_job.ex create mode 100644 apps/transport/lib/jobs/on_demand_netex_poller_job.ex create mode 100644 apps/transport/lib/jobs/on_demand_validation_helpers.ex create mode 100644 apps/transport/test/transport/jobs/netex_poller_job_test.exs create mode 100644 apps/transport/test/transport/jobs/on_demand_netex_poller_job_test.exs diff --git a/apps/transport/lib/jobs/netex_poller_job.ex b/apps/transport/lib/jobs/netex_poller_job.ex new file mode 100644 index 0000000000..a08ee9d0fb --- /dev/null +++ b/apps/transport/lib/jobs/netex_poller_job.ex @@ -0,0 +1,46 @@ +defmodule Transport.Jobs.NeTExPollerJob do + @moduledoc """ + Companion module to the validator for NeTEx files, used to handle long + standing validations. + """ + + # Max attempts doesn't really matter here as it's useful for workers failing. + # Here we mostly poll and excepted network errors, the worker won't fail. + @max_attempts 3 + + use Oban.Worker, + tags: ["validation"], + max_attempts: @max_attempts, + queue: :resource_validation, + unique: [fields: [:args, :worker]] + + alias Transport.Validators.NeTEx + + # Override the backoff to play nice and avoiding falling in very slow retry + # after an important streak of snoozing (which increments the `attempt` + # counter). + # + # See https://hexdocs.pm/oban/Oban.Worker.html#module-snoozing-jobs. + @impl Worker + def backoff(%Oban.Job{} = job) do + corrected_attempt = @max_attempts - (job.max_attempts - job.attempt) + + Worker.backoff(%{job | attempt: corrected_attempt}) + end + + @impl Worker + def perform(%Oban.Job{ + args: %{ + "validation_id" => validation_id, + "resource_history_id" => resource_history_id + }, + attempt: attempt + }) do + NeTEx.poll_validation_results(validation_id, attempt) + |> NeTEx.handle_validation_results(resource_history_id, fn ^validation_id -> snooze_poller(attempt) end) + end + + def snooze_poller(attempt) do + {:snooze, NeTEx.poll_interval(attempt)} + end +end diff --git a/apps/transport/lib/jobs/on_demand_netex_poller_job.ex b/apps/transport/lib/jobs/on_demand_netex_poller_job.ex new file mode 100644 index 0000000000..ed94f499cc --- /dev/null +++ b/apps/transport/lib/jobs/on_demand_netex_poller_job.ex @@ -0,0 +1,91 @@ +defmodule Transport.Jobs.OnDemandNeTExPollerJob do + @moduledoc """ + Job in charge of polling validation results from enRoute Chouette Valid. + + Upon success it stores the result in the database. + """ + + # Max attempts doesn't really matter here as it's useful for workers failing. + # Here we mostly poll and excepted network errors, the worker won't fail. + @max_attempts 3 + + use Oban.Worker, + tags: ["validation"], + max_attempts: @max_attempts, + queue: :on_demand_validation, + unique: [fields: [:args, :worker]] + + alias Transport.Jobs.OnDemandValidationHelpers, as: Helpers + alias Transport.Validators.NeTEx + + # Override the backoff to play nice and avoiding falling in very slow retry + # after an important streak of snoozing (which increments the `attempt` + # counter). + # + # See https://hexdocs.pm/oban/Oban.Worker.html#module-snoozing-jobs. + @impl Worker + def backoff(%Oban.Job{} = job) do + corrected_attempt = @max_attempts - (job.max_attempts - job.attempt) + + Worker.backoff(%{job | attempt: corrected_attempt}) + end + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"id" => multivalidation_id} = args, attempt: attempt}) do + check_result(args, attempt) + |> Helpers.handle_validation_result(multivalidation_id) + end + + def later(validation_id, multivalidation_id, url) do + %{validation_id: validation_id, id: multivalidation_id, permanent_url: url} + |> new(schedule_in: {20, :seconds}) + |> Oban.insert() + + Helpers.delegated_state() + end + + def check_result(%{"permanent_url" => url, "validation_id" => validation_id}, attempt) do + case NeTEx.poll_validation(validation_id, attempt) do + {:error, error_result} -> handle_error(error_result) + {:ok, ok_result} -> handle_success(ok_result, url) + {:pending, _validation_id} -> handle_pending(attempt) + end + end + + def handle_error(error_result) do + error_result + |> build_error_validation_result() + |> Helpers.terminal_state() + end + + def handle_success(ok_result, url) do + ok_result + |> build_successful_validation_result(url) + |> Helpers.terminal_state() + end + + def handle_pending(attempt) do + attempt + |> NeTEx.poll_interval() + |> Helpers.snoozed_state() + end + + defp build_successful_validation_result(%{"validations" => validation, "metadata" => metadata}, url) do + %{ + result: validation, + metadata: metadata, + data_vis: nil, + validator: NeTEx.validator_name(), + validated_data_name: url, + max_error: NeTEx.get_max_severity_error(validation), + oban_args: Helpers.completed() + } + end + + defp build_error_validation_result(%{message: msg}) do + %{ + oban_args: Helpers.error(msg), + validator: NeTEx.validator_name() + } + end +end diff --git a/apps/transport/lib/jobs/on_demand_validation_helpers.ex b/apps/transport/lib/jobs/on_demand_validation_helpers.ex new file mode 100644 index 0000000000..383f1a915d --- /dev/null +++ b/apps/transport/lib/jobs/on_demand_validation_helpers.ex @@ -0,0 +1,41 @@ +defmodule Transport.Jobs.OnDemandValidationHelpers do + @moduledoc """ + Shared code for jobs implementing the On Demand validation. + """ + import Ecto.Changeset + import Ecto.Query + alias DB.{MultiValidation, Repo} + + def terminal_state(result), do: {:terminal, result} + def delegated_state, do: :delegated + def snoozed_state(duration_in_seconds), do: {:snooze, duration_in_seconds} + + def completed, do: %{"state" => "completed"} + + def error(error_message), do: %{"state" => "error", "error_reason" => error_message} + + def handle_validation_result(result, multivalidation_id) do + case result do + {:terminal, changes} -> update_multivalidation(multivalidation_id, changes) + :delegated -> :ok + {:snooze, _duration_in_seconds} -> result + end + end + + defp update_multivalidation(multivalidation_id, changes) do + validation = %{oban_args: oban_args} = MultiValidation |> preload(:metadata) |> Repo.get!(multivalidation_id) + + # update oban_args with validator output + oban_args = Map.merge(oban_args, Map.get(changes, :oban_args, %{})) + changes = changes |> Map.put(:oban_args, oban_args) + + {metadata, changes} = Map.pop(changes, :metadata) + + validation + |> change(changes) + |> put_assoc(:metadata, %{metadata: metadata}) + |> Repo.update!() + + :ok + end +end diff --git a/apps/transport/lib/jobs/on_demand_validation_job.ex b/apps/transport/lib/jobs/on_demand_validation_job.ex index d70fdde3bb..d855f1fca9 100644 --- a/apps/transport/lib/jobs/on_demand_validation_job.ex +++ b/apps/transport/lib/jobs/on_demand_validation_job.ex @@ -7,46 +7,31 @@ defmodule Transport.Jobs.OnDemandValidationJob do """ use Oban.Worker, tags: ["validation"], max_attempts: 5, queue: :on_demand_validation require Logger - import Ecto.Changeset - import Ecto.Query - alias DB.{MultiValidation, Repo} alias Shared.Validation.JSONSchemaValidator.Wrapper, as: JSONSchemaValidator alias Shared.Validation.TableSchemaValidator.Wrapper, as: TableSchemaValidator alias Transport.DataVisualization + alias Transport.Jobs.OnDemandNeTExPollerJob + alias Transport.Jobs.OnDemandValidationHelpers, as: Helpers alias Transport.Validators.GTFSRT alias Transport.Validators.GTFSTransport alias Transport.Validators.NeTEx + @download_timeout_ms 10_000 @impl Oban.Worker def perform(%Oban.Job{args: %{"id" => multivalidation_id, "state" => "waiting"} = payload}) do - changes = + result = try do perform_validation(payload) rescue - e -> %{oban_args: %{"state" => "error", "error_reason" => inspect(e)}} + e -> %{oban_args: Helpers.error(inspect(e))} |> Helpers.terminal_state() end - validation = %{oban_args: oban_args} = MultiValidation |> preload(:metadata) |> Repo.get!(multivalidation_id) - - # update oban_args with validator output - oban_args = Map.merge(oban_args, changes.oban_args) - changes = changes |> Map.put(:oban_args, oban_args) - - {metadata, changes} = Map.pop(changes, :metadata) - - validation - |> change(changes) - |> put_assoc(:metadata, %{ - metadata: metadata - }) - |> Repo.update!() - + Helpers.handle_validation_result(result, multivalidation_id) + after if Map.has_key?(payload, "filename") do Transport.S3.delete_object!(:on_demand_validation, payload["filename"]) end - - :ok end defp perform_validation(%{"type" => "gtfs", "permanent_url" => url}) do @@ -54,7 +39,8 @@ defmodule Transport.Jobs.OnDemandValidationJob do case GTFSTransport.validate(url) do {:error, msg} -> - %{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator} + %{oban_args: Helpers.error(msg), validator: validator} + |> Helpers.terminal_state() {:ok, %{"validations" => validation, "metadata" => metadata}} -> %{ @@ -65,32 +51,22 @@ defmodule Transport.Jobs.OnDemandValidationJob do command: GTFSTransport.command(url), validated_data_name: url, max_error: GTFSTransport.get_max_severity_error(validation), - oban_args: %{ - "state" => "completed" - } + oban_args: Helpers.completed() } + |> Helpers.terminal_state() end end - defp perform_validation(%{"type" => "netex", "permanent_url" => url}) do - validator = NeTEx.validator_name() + defp perform_validation(%{"type" => "netex", "id" => multivalidation_id, "permanent_url" => url}) do + case NeTEx.validate(url) do + {:error, error_result} -> + OnDemandNeTExPollerJob.handle_error(error_result) - case NeTEx.validate(url, []) do - {:error, %{message: msg}} -> - %{oban_args: %{"state" => "error", "error_reason" => msg}, validator: validator} + {:ok, ok_result} -> + OnDemandNeTExPollerJob.handle_success(ok_result, url) - {:ok, %{"validations" => validation, "metadata" => metadata}} -> - %{ - result: validation, - metadata: metadata, - data_vis: nil, - validator: validator, - validated_data_name: url, - max_error: NeTEx.get_max_severity_error(validation), - oban_args: %{ - "state" => "completed" - } - } + {:pending, validation_id} -> + OnDemandNeTExPollerJob.later(validation_id, multivalidation_id, url) end end @@ -103,12 +79,14 @@ defmodule Transport.Jobs.OnDemandValidationJob do case TableSchemaValidator.validate(schema_name, url) do nil -> - %{oban_args: %{"state" => "error", "error_reason" => "could not perform validation"}, validator: validator} + %{oban_args: Helpers.error("could not perform validation"), validator: validator} + |> Helpers.terminal_state() # https://github.com/etalab/transport-site/issues/2390 # validator name should come from validator module, when it is properly extracted validation -> - %{oban_args: %{"state" => "completed"}, result: validation, validator: validator} + %{oban_args: Helpers.completed(), result: validation, validator: validator} + |> Helpers.terminal_state() end end @@ -127,15 +105,14 @@ defmodule Transport.Jobs.OnDemandValidationJob do ) do nil -> %{ - oban_args: %{ - "state" => "error", - "error_reason" => "could not perform validation" - }, + oban_args: Helpers.error("could not perform validation"), validator: validator } + |> Helpers.terminal_state() validation -> - %{oban_args: %{"state" => "completed"}, result: validation, validator: validator} + %{oban_args: Helpers.completed(), result: validation, validator: validator} + |> Helpers.terminal_state() end end @@ -155,11 +132,12 @@ defmodule Transport.Jobs.OnDemandValidationJob do result |> Map.merge(%{validated_data_name: gtfs_rt_url, secondary_validated_data_name: gtfs_url}) + |> Helpers.terminal_state() end defp normalize_download(result) do case result do - {:error, reason} -> {:error, %{"state" => "error", "error_reason" => reason}} + {:error, reason} -> {:error, Helpers.error(reason)} {:ok, path, _} -> {:ok, path} end end @@ -191,7 +169,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do # https://github.com/etalab/transport-site/issues/2390 # to do: transport-tools version when available %{ - oban_args: %{"state" => "completed"}, + oban_args: Helpers.completed(), result: validation, validator: GTFSRT.validator_name(), command: inspect(validator_args) @@ -199,10 +177,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do :error -> %{ - oban_args: %{ - "state" => "error", - "error_reason" => "Could not run validator. Please provide a GTFS and a GTFS-RT." - } + oban_args: Helpers.error("Could not run validator. Please provide a GTFS and a GTFS-RT.") } end @@ -210,7 +185,7 @@ defmodule Transport.Jobs.OnDemandValidationJob do if not ignore_shapes and String.contains?(reason, "java.lang.OutOfMemoryError") do run_save_gtfs_rt_validation(gtfs_path, gtfs_rt_path, ignore_shapes: true) else - %{oban_args: %{"state" => "error", "error_reason" => inspect(reason)}} + %{oban_args: Helpers.error(inspect(reason))} end end end diff --git a/apps/transport/lib/transport_web/controllers/resource_controller.ex b/apps/transport/lib/transport_web/controllers/resource_controller.ex index d714fbca64..de5fc93cc5 100644 --- a/apps/transport/lib/transport_web/controllers/resource_controller.ex +++ b/apps/transport/lib/transport_web/controllers/resource_controller.ex @@ -12,7 +12,8 @@ defmodule TransportWeb.ResourceController do Transport.Validators.GTFSRT, Transport.Validators.GBFSValidator, Transport.Validators.TableSchema, - Transport.Validators.EXJSONSchema + Transport.Validators.EXJSONSchema, + Transport.Validators.NeTEx ]) def details(conn, %{"id" => id} = params) do diff --git a/apps/transport/lib/validators/netex_validator.ex b/apps/transport/lib/validators/netex_validator.ex index 85fdecea08..e050ada5ff 100644 --- a/apps/transport/lib/validators/netex_validator.ex +++ b/apps/transport/lib/validators/netex_validator.ex @@ -6,18 +6,38 @@ defmodule Transport.Validators.NeTEx do import TransportWeb.Gettext, only: [dgettext: 2, dngettext: 4] require Logger + alias Transport.Jobs.NeTExPollerJob, as: Poller + + @behaviour Transport.Validators.Validator @no_error "NoError" - @max_retries 100 + # 180 * 20 seconds = 1 hour + @max_attempts 180 @unknown_code "unknown-code" - @behaviour Transport.Validators.Validator + defmacro too_many_attempts(attempt) do + quote do + unquote(attempt) > unquote(@max_attempts) + end + end + + @doc """ + Poll interval to play nice with the tier. + + iex> 0..9 |> Enum.map(&poll_interval(&1)) + [10, 10, 10, 10, 10, 10, 20, 20, 20, 20] + """ + def poll_interval(nb_tries) when nb_tries < 6, do: 10 + def poll_interval(_), do: 20 @impl Transport.Validators.Validator def validator_name, do: "enroute-chouette-netex-validator" + # This will change with an actual versioning of the validator + def validator_version, do: "saas-production" + @impl Transport.Validators.Validator def validate_and_save(%DB.ResourceHistory{} = resource_history) do Logger.info("Validating NeTEx #{resource_history.id} with enRoute Chouette Valid") @@ -26,15 +46,32 @@ defmodule Transport.Validators.NeTEx do end def validate_resource_history(resource_history, filepath) do - case validate_with_enroute(filepath) do + validate_with_enroute(filepath) + |> handle_validation_results(resource_history.id, &enqueue_poller(resource_history.id, &1)) + end + + def enqueue_poller(resource_history_id, validation_id, attempt \\ 0) do + {:ok, _job} = + Poller.new(%{"validation_id" => validation_id, "resource_history_id" => resource_history_id}) + |> Oban.insert(schedule_in: _seconds = poll_interval(attempt)) + + :ok + end + + def handle_validation_results(validation_results, resource_history_id, on_pending) do + case validation_results do {:ok, %{url: result_url, elapsed_seconds: elapsed_seconds, retries: retries}} -> - insert_validation_results(resource_history.id, result_url, %{elapsed_seconds: elapsed_seconds, retries: retries}) + insert_validation_results( + resource_history_id, + result_url, + %{elapsed_seconds: elapsed_seconds, retries: retries} + ) :ok {:error, %{details: {result_url, errors}, elapsed_seconds: elapsed_seconds, retries: retries}} -> insert_validation_results( - resource_history.id, + resource_history_id, result_url, %{elapsed_seconds: elapsed_seconds, retries: retries}, demote_non_xsd_errors(errors) @@ -43,56 +80,81 @@ defmodule Transport.Validators.NeTEx do :ok {:error, :unexpected_validation_status} -> - Logger.error("Invalid API call to enRoute Chouette Valid") - {:error, "enRoute Chouette Valid: Unexpected validation status"} + Logger.error("Invalid API call to enRoute Chouette Valid (resource_history_id: #{resource_history_id})") + + :ok {:error, %{message: :timeout, retries: _retries}} -> - Logger.error("Timeout while fetching result on enRoute Chouette Valid") - {:error, "enRoute Chouette Valid: Timeout while fetching results"} + Logger.error( + "Timeout while fetching results on enRoute Chouette Valid (resource_history_id: #{resource_history_id})" + ) + + :ok + + {:pending, validation_id} -> + on_pending.(validation_id) end end - @type validate_options :: [{:graceful_retry, boolean()}] @type error_details :: %{:message => String.t(), optional(:retries) => integer()} + @type validation_id :: binary() + + @type validation_results :: {:ok, map()} | {:error, error_details()} | {:pending, validation_id()} + @doc """ Validate the resource from the given URL. - Options can be passed to tweak the behaviour: - - graceful_retry is a flag to skip the polling interval. Useful for testing - purposes mostly. Defaults to false. + Used by OnDemand job. """ - @spec validate(binary(), validate_options()) :: {:ok, map()} | {:error, error_details()} - def validate(url, opts \\ []) do + @spec validate(binary()) :: validation_results() + def validate(url) do with_url(url, fn filepath -> - case validate_with_enroute(filepath, opts) do - {:ok, %{url: result_url, elapsed_seconds: elapsed_seconds, retries: retries}} -> - # result_url in metadata? - Logger.info("Result URL: #{result_url}") - - {:ok, - %{"validations" => index_messages([]), "metadata" => %{elapsed_seconds: elapsed_seconds, retries: retries}}} - - {:error, %{details: {result_url, errors}, elapsed_seconds: elapsed_seconds, retries: retries}} -> - Logger.info("Result URL: #{result_url}") - # result_url in metadata? - {:ok, - %{ - "validations" => errors |> demote_non_xsd_errors() |> index_messages(), - "metadata" => %{elapsed_seconds: elapsed_seconds, retries: retries} - }} - - {:error, :unexpected_validation_status} -> - Logger.error("Invalid API call to enRoute Chouette Valid") - {:error, %{message: "enRoute Chouette Valid: Unexpected validation status"}} - - {:error, %{message: :timeout, retries: retries}} -> - Logger.error("Timeout while fetching result on enRoute Chouette Valid") - {:error, %{message: "enRoute Chouette Valid: Timeout while fetching results", retries: retries}} - end + validate_with_enroute(filepath) |> handle_validation_results_on_demand() end) end + @doc """ + Continuation for the validate function (when result was pending). + + Let the OnDemand job yield while waiting for results without having the OnDemand implementation leak here. + """ + @spec poll_validation(validation_id(), non_neg_integer()) :: validation_results() + def poll_validation(validation_id, retries) do + poll_validation_results(validation_id, retries) |> handle_validation_results_on_demand() + end + + defp handle_validation_results_on_demand(validation_results) do + case validation_results do + {:ok, %{url: result_url, elapsed_seconds: elapsed_seconds, retries: retries}} -> + # result_url in metadata? + Logger.info("Result URL: #{result_url}") + + {:ok, + %{"validations" => index_messages([]), "metadata" => %{elapsed_seconds: elapsed_seconds, retries: retries}}} + + {:error, %{details: {result_url, errors}, elapsed_seconds: elapsed_seconds, retries: retries}} -> + Logger.info("Result URL: #{result_url}") + # result_url in metadata? + {:ok, + %{ + "validations" => errors |> demote_non_xsd_errors() |> index_messages(), + "metadata" => %{elapsed_seconds: elapsed_seconds, retries: retries} + }} + + {:error, :unexpected_validation_status} -> + Logger.error("Invalid API call to enRoute Chouette Valid") + {:error, %{message: "enRoute Chouette Valid: Unexpected validation status"}} + + {:error, %{message: :timeout, retries: retries}} -> + Logger.error("Timeout while fetching results on enRoute Chouette Valid") + {:error, %{message: "enRoute Chouette Valid: Timeout while fetching results", retries: retries}} + + {:pending, validation_id} -> + {:pending, validation_id} + end + end + @spec with_resource_file(DB.ResourceHistory.t(), (Path.t() -> any())) :: any() def with_resource_file(resource_history, closure) do %DB.ResourceHistory{payload: %{"permanent_url" => permanent_url}} = resource_history @@ -231,21 +293,19 @@ defmodule Transport.Validators.NeTEx do def count_by_severity(_), do: %{} - defp validate_with_enroute(filepath, opts \\ []) do - client().create_a_validation(filepath) |> fetch_validation_results(0, opts) + defp validate_with_enroute(filepath) do + setup_validation(filepath) |> poll_validation_results(0) end - defp fetch_validation_results(validation_id, retries, opts) do + defp setup_validation(filepath), do: client().create_a_validation(filepath) + + def poll_validation_results(validation_id, retries) do case client().get_a_validation(validation_id) do - :pending when retries >= @max_retries -> + :pending when too_many_attempts(retries) -> {:error, %{message: :timeout, retries: retries}} :pending -> - if Keyword.get(opts, :graceful_retry, true) do - retries |> poll_interval() |> :timer.sleep() - end - - fetch_validation_results(validation_id, retries + 1, opts) + {:pending, validation_id} {:successful, url, elapsed_seconds} -> {:ok, %{url: url, elapsed_seconds: elapsed_seconds, retries: retries}} @@ -258,15 +318,6 @@ defmodule Transport.Validators.NeTEx do end end - @doc """ - Poll interval to play nice with the tier. - - iex> 0..9 |> Enum.map(&poll_interval(&1)) - [10000, 10000, 10000, 10000, 10000, 10000, 20000, 20000, 20000, 20000] - """ - def poll_interval(nb_tries) when nb_tries < 6, do: 10_000 - def poll_interval(_), do: 20_000 - @doc """ iex> index_messages([]) %{} @@ -283,9 +334,6 @@ defmodule Transport.Validators.NeTEx do defp get_code(%{"code" => code}), do: code defp get_code(%{}), do: @unknown_code - # This will change with an actual versioning of the validator - def validator_version, do: "saas-production" - @doc """ iex> validation_result = %{"uic-operating-period" => [%{"code" => "uic-operating-period", "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", "criticity" => "error"}], "valid-day-bits" => [%{"code" => "valid-day-bits", "message" => "Mandatory attribute valid_day_bits not found", "criticity" => "error"}], "frame-arret-resources" => [%{"code" => "frame-arret-resources", "message" => "Tag frame_id doesn't match ''", "criticity" => "warning"}]} iex> summary(validation_result) @@ -388,7 +436,7 @@ defmodule Transport.Validators.NeTEx do Transport.EnRouteChouetteValidClient.Wrapper.impl() end - defp demote_non_xsd_errors(errors), do: Enum.map(errors, &demote_non_xsd_error(&1)) + defp demote_non_xsd_errors(errors), do: Enum.map(errors, &demote_non_xsd_error/1) defp demote_non_xsd_error(error) do code = Map.get(error, "code", "") diff --git a/apps/transport/lib/validators/validator_selection.ex b/apps/transport/lib/validators/validator_selection.ex index fac6b53eac..a5ffd7fc6b 100644 --- a/apps/transport/lib/validators/validator_selection.ex +++ b/apps/transport/lib/validators/validator_selection.ex @@ -33,7 +33,7 @@ defmodule Transport.ValidatorsSelection.Impl do def validators(%{format: "GTFS"}), do: [Validators.GTFSTransport] def validators(%{format: "gtfs-rt"}), do: [Validators.GTFSRT] def validators(%{format: "gbfs"}), do: [Validators.GBFSValidator] - def validators(%{format: "NeTEx"}), do: [] + def validators(%{format: "NeTEx"}), do: [Validators.NeTEx] def validators(%{schema_name: schema_name}) when not is_nil(schema_name) do cond do diff --git a/apps/transport/test/support/enroute_chouette_valid_client_helpers.ex b/apps/transport/test/support/enroute_chouette_valid_client_helpers.ex index 43112901ac..f5a8d73958 100644 --- a/apps/transport/test/support/enroute_chouette_valid_client_helpers.ex +++ b/apps/transport/test/support/enroute_chouette_valid_client_helpers.ex @@ -5,28 +5,40 @@ defmodule Transport.Test.EnRouteChouetteValidClientHelpers do import Mox def expect_create_validation do - validation_id = Ecto.UUID.generate() + validation_id = with_running_validation() + expect(Transport.EnRouteChouetteValidClient.Mock, :create_a_validation, fn _ -> validation_id end) + validation_id end def expect_pending_validation(validation_id) do expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> :pending end) + + validation_id end def expect_successful_validation(validation_id, elapsed) do expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> {:successful, "http://localhost:9999/chouette-valid/#{validation_id}", elapsed} end) + + validation_id end def expect_failed_validation(validation_id, elapsed) do expect(Transport.EnRouteChouetteValidClient.Mock, :get_a_validation, fn ^validation_id -> {:failed, elapsed} end) + + validation_id end def expect_get_messages(validation_id, result) do expect(Transport.EnRouteChouetteValidClient.Mock, :get_messages, fn ^validation_id -> {"http://localhost:9999/chouette-valid/#{validation_id}/messages", result} end) + + validation_id end + + def with_running_validation, do: Ecto.UUID.generate() end diff --git a/apps/transport/test/transport/jobs/netex_poller_job_test.exs b/apps/transport/test/transport/jobs/netex_poller_job_test.exs new file mode 100644 index 0000000000..761f1bed04 --- /dev/null +++ b/apps/transport/test/transport/jobs/netex_poller_job_test.exs @@ -0,0 +1,175 @@ +defmodule Transport.Jobs.NeTExPollerJobTest do + use ExUnit.Case, async: true + use Oban.Testing, repo: DB.Repo + import DB.Factory + import ExUnit.CaptureLog + import Mox + import Transport.Test.EnRouteChouetteValidClientHelpers + + alias Transport.Validators.NeTEx + + setup do + Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo) + end + + setup :verify_on_exit! + + @sample_error_messages [ + %{ + "code" => "xsd-1871", + "criticity" => "error", + "message" => + "Element '{http://www.netex.org.uk/netex}OppositeDIrectionRef': This element is not expected. Expected is ( {http://www.netex.org.uk/netex}OppositeDirectionRef )." + }, + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "error" + }, + %{ + "code" => "valid-day-bits", + "message" => "Mandatory attribute valid_day_bits not found", + "criticity" => "error" + }, + %{ + "code" => "frame-arret-resources", + "message" => "Tag frame_id doesn't match ''", + "criticity" => "warning" + }, + %{ + "message" => "Reference MOBIITI:Quay:104325 doesn't match any existing Resource", + "criticity" => "error" + } + ] + + test "valid NeTEx" do + resource_history = generate_resource_url() |> mk_netex_resource() + + attempts = 5 + duration = 12 + + validation_id = with_running_validation() |> expect_successful_validation(duration) + + assert :ok == run_polling_job(resource_history, validation_id, attempts) + + multi_validation = load_multi_validation(resource_history.id) + + assert multi_validation.command == "http://localhost:9999/chouette-valid/#{validation_id}" + assert multi_validation.validator == "enroute-chouette-netex-validator" + assert multi_validation.validator_version == "saas-production" + assert multi_validation.result == %{} + assert multi_validation.metadata.metadata == %{"retries" => attempts, "elapsed_seconds" => duration} + end + + test "invalid NeTEx" do + resource_history = generate_resource_url() |> mk_netex_resource() + + attempts = 5 + duration = 12 + + validation_id = with_running_validation() |> expect_failed_validation(duration) + + expect_get_messages(validation_id, @sample_error_messages) + + assert :ok = run_polling_job(resource_history, validation_id, attempts) + + multi_validation = load_multi_validation(resource_history.id) + + assert multi_validation.command == "http://localhost:9999/chouette-valid/#{validation_id}/messages" + assert multi_validation.validator == "enroute-chouette-netex-validator" + assert multi_validation.validator_version == "saas-production" + assert multi_validation.metadata.metadata == %{"retries" => attempts, "elapsed_seconds" => duration} + + assert multi_validation.result == %{ + "xsd-1871" => [ + %{ + "code" => "xsd-1871", + "criticity" => "error", + "message" => + "Element '{http://www.netex.org.uk/netex}OppositeDIrectionRef': This element is not expected. Expected is ( {http://www.netex.org.uk/netex}OppositeDirectionRef )." + } + ], + "uic-operating-period" => [ + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "warning" + } + ], + "valid-day-bits" => [ + %{ + "code" => "valid-day-bits", + "message" => "Mandatory attribute valid_day_bits not found", + "criticity" => "warning" + } + ], + "frame-arret-resources" => [ + %{ + "code" => "frame-arret-resources", + "message" => "Tag frame_id doesn't match ''", + "criticity" => "warning" + } + ], + "unknown-code" => [ + %{ + "message" => "Reference MOBIITI:Quay:104325 doesn't match any existing Resource", + "criticity" => "warning" + } + ] + } + end + + test "pending validation" do + resource_history = generate_resource_url() |> mk_netex_resource() + + attempt = 5 + + validation_id = with_running_validation() |> expect_pending_validation() + + assert {:snooze, NeTEx.poll_interval(attempt)} == run_polling_job(resource_history, validation_id, attempt) + + assert nil == load_multi_validation(resource_history.id) + end + + test "too many attempts" do + resource_history = generate_resource_url() |> mk_netex_resource() + + attempt = 181 + + validation_id = with_running_validation() |> expect_pending_validation() + + capture_log([level: :error], fn -> + assert :ok = run_polling_job(resource_history, validation_id, attempt) + + assert nil == load_multi_validation(resource_history.id) + end) =~ "Timeout while fetching results on enRoute Chouette Valid (resource_history_id: #{resource_history.id})" + end + + defp load_multi_validation(resource_history_id) do + DB.MultiValidation + |> DB.Repo.get_by(resource_history_id: resource_history_id) + |> DB.Repo.preload(:metadata) + end + + defp mk_netex_resource(permanent_url) do + dataset = insert(:dataset) + + resource = insert(:resource, dataset_id: dataset.id, format: "NeTEx") + + insert(:resource_history, resource_id: resource.id, payload: %{"permanent_url" => permanent_url}) + end + + defp generate_resource_url do + "http://localhost:9999/netex-#{Ecto.UUID.generate()}.zip" + end + + defp run_polling_job(%DB.ResourceHistory{} = resource_history, validation_id, attempt) do + payload = + %{ + "resource_history_id" => resource_history.id, + "validation_id" => validation_id + } + + perform_job(Transport.Jobs.NeTExPollerJob, payload, attempt: attempt) + end +end diff --git a/apps/transport/test/transport/jobs/on_demand_netex_poller_job_test.exs b/apps/transport/test/transport/jobs/on_demand_netex_poller_job_test.exs new file mode 100644 index 0000000000..260b4a6656 --- /dev/null +++ b/apps/transport/test/transport/jobs/on_demand_netex_poller_job_test.exs @@ -0,0 +1,156 @@ +defmodule Transport.Test.Transport.Jobs.OnDemandNeTExPollerJobTest do + use ExUnit.Case, async: true + use Oban.Testing, repo: DB.Repo + import DB.Factory + import ExUnit.CaptureLog + import Mox + import Transport.Test.EnRouteChouetteValidClientHelpers + alias Transport.Validators.NeTEx + + setup :verify_on_exit! + + setup do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo) + end + + @filename "file.zip" + + test "still pending" do + attempt = 1 + validation = create_validation(%{"type" => "netex"}) + + validation_id = with_running_validation() |> expect_pending_validation() + + snooze_duration = NeTEx.poll_interval(attempt) + + assert {:snooze, ^snooze_duration} = run_polling_job(validation, validation_id, attempt) + + assert %{ + data_vis: nil, + max_error: nil, + metadata: nil, + oban_args: %{"state" => "waiting", "type" => "netex"}, + result: nil + } = validation |> DB.Repo.reload() |> DB.Repo.preload(:metadata) + end + + test "too many attempts" do + validation = create_validation(%{"type" => "netex"}) + + validation_id = with_running_validation() |> expect_pending_validation() + + assert capture_log([level: :error], fn -> + assert :ok == run_polling_job(validation, validation_id, 181) + + assert %{ + data_vis: nil, + metadata: %{}, + oban_args: %{ + "state" => "error", + "type" => "netex", + "error_reason" => "enRoute Chouette Valid: Timeout while fetching results" + }, + result: nil, + validation_timestamp: date + } = validation |> DB.Repo.reload() |> DB.Repo.preload(:metadata) + + assert DateTime.diff(date, DateTime.utc_now()) <= 1 + end) =~ "Timeout while fetching result" + end + + test "completed" do + validation = create_validation(%{"type" => "netex"}) + + validation_id = with_running_validation() |> expect_valid_netex() + + assert :ok == run_polling_job(validation, validation_id) + + assert %{ + data_vis: nil, + max_error: "NoError", + metadata: %{}, + oban_args: %{"state" => "completed", "type" => "netex"}, + result: %{}, + validation_timestamp: date + } = validation |> DB.Repo.reload() |> DB.Repo.preload(:metadata) + + assert DateTime.diff(date, DateTime.utc_now()) <= 1 + end + + test "error" do + validation = create_validation(%{"type" => "netex"}) + + errors = [ + %{ + "code" => "xsd-1871", + "criticity" => "error", + "message" => + "Element '{http://www.netex.org.uk/netex}OppositeDIrectionRef': This element is not expected. Expected is ( {http://www.netex.org.uk/netex}OppositeDirectionRef )." + }, + %{ + "code" => "uic-operating-period", + "message" => "Resource 23504000009 hasn't expected class but Netex::OperatingPeriod", + "criticity" => "error" + }, + %{ + "code" => "valid-day-bits", + "message" => "Mandatory attribute valid_day_bits not found", + "criticity" => "error" + }, + %{ + "code" => "frame-arret-resources", + "message" => "Tag frame_id doesn't match ''", + "criticity" => "warning" + } + ] + + validation_id = with_running_validation() |> expect_netex_with_errors(errors) + + assert :ok == run_polling_job(validation, validation_id) + + assert %{ + data_vis: nil, + max_error: "error", + metadata: %{}, + oban_args: %{"state" => "completed", "type" => "netex"}, + result: result, + validation_timestamp: date + } = validation |> DB.Repo.reload() |> DB.Repo.preload(:metadata) + + assert %{"xsd-1871" => a1, "uic-operating-period" => a2, "valid-day-bits" => a3, "frame-arret-resources" => a4} = + result + + assert length(a1) == 1 + assert length(a2) == 1 + assert length(a3) == 1 + assert length(a4) == 1 + + assert DateTime.diff(date, DateTime.utc_now()) <= 1 + end + + defp create_validation(details) do + oban_args = + details + |> Map.merge(%{"filename" => @filename, "permanent_url" => mk_url(), "state" => "waiting"}) + + insert(:multi_validation, oban_args: oban_args) + end + + defp mk_url, do: "http://localhost:9999/netex-#{Ecto.UUID.generate()}.zip" + + defp expect_valid_netex(validation_id), do: expect_successful_validation(validation_id, 180) + + defp expect_netex_with_errors(validation_id, messages) do + expect_failed_validation(validation_id, 10) + + expect_get_messages(validation_id, messages) + end + + defp run_polling_job(%DB.MultiValidation{} = validation, validation_id, attempt \\ 1) do + payload = + validation.oban_args + |> Map.merge(%{"id" => validation.id, "validation_id" => validation_id}) + + perform_job(Transport.Jobs.OnDemandNeTExPollerJob, payload, attempt: attempt) + end +end diff --git a/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs b/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs index 2ed35b6add..a59d52c31e 100644 --- a/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs +++ b/apps/transport/test/transport/jobs/on_demand_validation_job_test.exs @@ -391,7 +391,7 @@ defmodule Transport.Test.Transport.Jobs.OnDemandValidationJobTest do refute File.exists?(OnDemandValidationJob.gtfs_rt_result_path(gtfs_rt_path)) end - test "with a NeTEx" do + test "with a NeTEx with errors" do url = mk_raw_netex_resource() validation = create_validation(%{"type" => "netex"}, url) @@ -444,6 +444,40 @@ defmodule Transport.Test.Transport.Jobs.OnDemandValidationJobTest do assert DateTime.diff(date, DateTime.utc_now()) <= 1 end + + test "with a NeTEx long lasting" do + url = mk_raw_netex_resource() + validation = create_validation(%{"type" => "netex"}, url) + + validation_id = expect_netex_long_lasting() + + s3_mocks_delete_object(Transport.S3.bucket_name(:on_demand_validation), @filename) + + assert :ok == run_job(validation) + + validation = DB.Repo.reload(validation) + assert nil == validation.result + assert nil == validation.max_error + + assert %{ + "filename" => @filename, + "permanent_url" => url, + "type" => "netex", + "state" => "waiting" + } = validation.oban_args + + in_20_seconds = DateTime.utc_now() |> DateTime.add(20, :second) + + assert_enqueued( + worker: Transport.Jobs.OnDemandNeTExPollerJob, + args: %{ + "id" => validation.id, + "permanent_url" => url, + "validation_id" => validation_id + }, + scheduled_at: in_20_seconds + ) + end end defp create_validation(details, url \\ @url) do @@ -466,6 +500,10 @@ defmodule Transport.Test.Transport.Jobs.OnDemandValidationJobTest do expect_get_messages(validation_id, messages) end + def expect_netex_long_lasting do + expect_create_validation() |> expect_pending_validation() + end + defp run_job(%DB.MultiValidation{} = validation) do payload = Map.merge(%{"id" => validation.id}, validation.oban_args) perform_job(OnDemandValidationJob, payload) diff --git a/apps/transport/test/transport/validators/netex_validator_test.exs b/apps/transport/test/transport/validators/netex_validator_test.exs index 2a701d7088..abb2b438e0 100644 --- a/apps/transport/test/transport/validators/netex_validator_test.exs +++ b/apps/transport/test/transport/validators/netex_validator_test.exs @@ -1,8 +1,8 @@ defmodule Transport.Validators.NeTExTest do use ExUnit.Case, async: true + use Oban.Testing, repo: DB.Repo import DB.Factory import Mox - import ExUnit.CaptureLog import Transport.Test.EnRouteChouetteValidClientHelpers alias Transport.Validators.NeTEx @@ -43,14 +43,11 @@ defmodule Transport.Validators.NeTExTest do } ] - @sample_error_message Enum.take(@sample_error_messages, 1) - describe "existing resource" do test "valid NeTEx" do resource_history = mk_netex_resource() - validation_id = expect_create_validation() - expect_successful_validation(validation_id, 12) + validation_id = expect_create_validation() |> expect_successful_validation(12) assert :ok == NeTEx.validate_and_save(resource_history) @@ -63,11 +60,28 @@ defmodule Transport.Validators.NeTExTest do assert multi_validation.metadata.metadata == %{"retries" => 0, "elapsed_seconds" => 12} end + test "pending validation" do + resource_history = mk_netex_resource() + + validation_id = expect_create_validation() |> expect_pending_validation() + + assert :ok == NeTEx.validate_and_save(resource_history) + + assert_enqueued( + worker: Transport.Jobs.NeTExPollerJob, + args: %{ + "validation_id" => validation_id, + "resource_history_id" => resource_history.id + } + ) + + assert nil == load_multi_validation(resource_history.id) + end + test "invalid NeTEx" do resource_history = mk_netex_resource() - validation_id = expect_create_validation() - expect_failed_validation(validation_id, 31) + validation_id = expect_create_validation() |> expect_failed_validation(31) expect_get_messages(validation_id, @sample_error_messages) @@ -130,8 +144,7 @@ defmodule Transport.Validators.NeTExTest do test "valid NeTEx" do resource_url = mk_raw_netex_resource() - validation_id = expect_create_validation() - expect_successful_validation(validation_id, 9) + expect_create_validation() |> expect_successful_validation(9) assert {:ok, %{"validations" => %{}, "metadata" => %{retries: 0, elapsed_seconds: 9}}} == NeTEx.validate(resource_url) @@ -140,8 +153,7 @@ defmodule Transport.Validators.NeTExTest do test "invalid NeTEx" do resource_url = mk_raw_netex_resource() - validation_id = expect_create_validation() - expect_failed_validation(validation_id, 25) + validation_id = expect_create_validation() |> expect_failed_validation(25) expect_get_messages(validation_id, @sample_error_messages) @@ -187,47 +199,12 @@ defmodule Transport.Validators.NeTExTest do NeTEx.validate(resource_url) end - test "retries" do + test "pending" do resource_url = mk_raw_netex_resource() - validation_id = expect_create_validation() - expect_pending_validation(validation_id) - expect_pending_validation(validation_id) - expect_pending_validation(validation_id) - expect_failed_validation(validation_id, 35) - - expect_get_messages(validation_id, @sample_error_message) + validation_id = expect_create_validation() |> expect_pending_validation() - validation_result = %{ - "xsd-1871" => [ - %{ - "code" => "xsd-1871", - "criticity" => "error", - "message" => - "Element '{http://www.netex.org.uk/netex}OppositeDIrectionRef': This element is not expected. Expected is ( {http://www.netex.org.uk/netex}OppositeDirectionRef )." - } - ] - } - - # Let's disable graceful retry as we are mocking the API, otherwise the - # test would take almost a minute. - assert {:ok, %{"validations" => validation_result, "metadata" => %{retries: 3, elapsed_seconds: 35}}} == - NeTEx.validate(resource_url, graceful_retry: false) - end - - test "timeout" do - resource_url = mk_raw_netex_resource() - - validation_id = expect_create_validation() - - Enum.each(0..100, fn _i -> - expect_pending_validation(validation_id) - end) - - {result, log} = with_log(fn -> NeTEx.validate(resource_url, graceful_retry: false) end) - - assert result == {:error, %{message: "enRoute Chouette Valid: Timeout while fetching results", retries: 100}} - assert log =~ "[error] Timeout while fetching result on enRoute Chouette Valid" + assert {:pending, validation_id} == NeTEx.validate(resource_url) end end @@ -236,14 +213,11 @@ defmodule Transport.Validators.NeTExTest do resource = insert(:resource, dataset_id: dataset.id, format: "NeTEx") - resource_history = - insert(:resource_history, resource_id: resource.id, payload: %{"permanent_url" => mk_raw_netex_resource()}) - - resource_history + insert(:resource_history, resource_id: resource.id, payload: %{"permanent_url" => mk_raw_netex_resource()}) end defp mk_raw_netex_resource do - resource_url = "http://localhost:9999/netex-#{Ecto.UUID.generate()}.zip" + resource_url = generate_resource_url() expect(Transport.Req.Mock, :get!, 1, fn ^resource_url, [{:compressed, false}, {:into, _}] -> {:ok, %Req.Response{status: 200, body: %{"data" => "some_zip_file"}}} @@ -251,4 +225,8 @@ defmodule Transport.Validators.NeTExTest do resource_url end + + defp generate_resource_url do + "http://localhost:9999/netex-#{Ecto.UUID.generate()}.zip" + end end diff --git a/apps/transport/test/transport/validators/validator_selection_test.exs b/apps/transport/test/transport/validators/validator_selection_test.exs index df757ee142..bf4b518a5d 100644 --- a/apps/transport/test/transport/validators/validator_selection_test.exs +++ b/apps/transport/test/transport/validators/validator_selection_test.exs @@ -22,7 +22,9 @@ defmodule Transport.ValidatorsSelectionTest do resource_history = insert(:resource_history, payload: %{"format" => "NeTEx"}) - assert ValidatorsSelection.validators(resource_history) == [] + assert ValidatorsSelection.validators(resource_history) == [ + Transport.Validators.NeTEx + ] end test "for a ResourceHistory with a schema" do diff --git a/apps/transport/test/transport_web/controllers/resource_controller_test.exs b/apps/transport/test/transport_web/controllers/resource_controller_test.exs index b8ba94a589..b99ff6350f 100644 --- a/apps/transport/test/transport_web/controllers/resource_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/resource_controller_test.exs @@ -461,54 +461,53 @@ defmodule TransportWeb.ResourceControllerTest do |> html_response(200) =~ "couverture calendaire par réseau" end - # test "NeTEx validation is shown", %{conn: conn} do - # %{id: dataset_id} = insert(:dataset) - # - # %{id: resource_id} = - # insert(:resource, %{ - # dataset_id: dataset_id, - # format: "NeTEx", - # url: "https://example.com/file" - # }) - # - # conn1 = conn |> get(resource_path(conn, :details, resource_id)) - # assert conn1 |> html_response(200) =~ "Pas de validation disponible" - # - # %{id: resource_history_id} = - # insert(:resource_history, %{ - # resource_id: resource_id, - # payload: %{"permanent_url" => permanent_url = "https://example.com/#{Ecto.UUID.generate()}"} - # }) - # - # insert(:multi_validation, %{ - # resource_history_id: resource_history_id, - # validator: Transport.Validators.NeTEx.validator_name(), - # result: %{ - # "xsd-1871" => [ - # %{ - # "code" => "xsd-1871", - # "message" => - # "Element '{http://www.netex.org.uk/netex}OppositeDIrectionRef': This element is not expected. Expected is ( {http://www.netex.org.uk/netex}OppositeDirectionRef ).", - # "criticity" => "error" - # } - # ] - # }, - # max_error: "error", - # metadata: %DB.ResourceMetadata{ - # metadata: %{"elapsed_seconds" => 42}, - # modes: [], - # features: [] - # }, - # validation_timestamp: ~U[2022-10-28 14:12:29.041243Z] - # }) - # - # content = conn |> get(resource_path(conn, :details, resource_id)) |> html_response(200) - # assert content =~ "Rapport de validation" - # - # assert content =~ - # ~s{Validation effectuée en utilisant le - # fichier NeTEx en vigueur le 28/10/2022 à 16h12 Europe/Paris} - # end + test "NeTEx validation is shown", %{conn: conn} do + %{id: dataset_id} = insert(:dataset) + + %{id: resource_id} = + insert(:resource, %{ + dataset_id: dataset_id, + format: "NeTEx", + url: "https://example.com/file" + }) + + conn1 = conn |> get(resource_path(conn, :details, resource_id)) + assert conn1 |> html_response(200) =~ "Pas de validation disponible" + + %{id: resource_history_id} = + insert(:resource_history, %{ + resource_id: resource_id, + payload: %{"permanent_url" => permanent_url = "https://example.com/#{Ecto.UUID.generate()}"} + }) + + insert(:multi_validation, %{ + resource_history_id: resource_history_id, + validator: Transport.Validators.NeTEx.validator_name(), + result: %{ + "xsd-1871" => [ + %{ + "code" => "xsd-1871", + "message" => + "Element '{http://www.netex.org.uk/netex}OppositeDIrectionRef': This element is not expected. Expected is ( {http://www.netex.org.uk/netex}OppositeDirectionRef ).", + "criticity" => "error" + } + ] + }, + max_error: "error", + metadata: %DB.ResourceMetadata{ + metadata: %{"elapsed_seconds" => 42}, + modes: [], + features: [] + }, + validation_timestamp: ~U[2022-10-28 14:12:29.041243Z] + }) + + content = conn |> get(resource_path(conn, :details, resource_id)) |> html_response(200) + assert content =~ "Rapport de validation" + + assert content =~ + ~s{Validation effectuée en utilisant le fichier NeTEx en vigueur le 28/10/2022 à 16h12 Europe/Paris} + end test "GTFS-RT validation is shown", %{conn: conn} do %{id: dataset_id} = insert(:dataset)