From 4d61371ecac22ea383a1bb4d37e8eda17b321b15 Mon Sep 17 00:00:00 2001 From: Matthew Somerville <matthew@mysociety.org> Date: Tue, 8 Aug 2023 16:26:30 +0100 Subject: [PATCH 01/11] [Brent] Clinical collection tickboxes. --- perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm b/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm index 034b7f070..59b9406ea 100644 --- a/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm +++ b/perllib/Open311/Endpoint/Integration/UK/Brent/Echo.pm @@ -28,6 +28,9 @@ around process_service_request_args => sub { } elsif ($service_id == 317) { $args->{attributes}{"Garden_BIN"} = 1; $args->{attributes}{"Garden_BAG"} = 1; + } elsif ($service_id == 274) { + $args->{attributes}{"Clinical_BIN"} = 1; + $args->{attributes}{"Clinical_BOX"} = 1; } } elsif ($request->{event_type} == 1159) { if ($args->{attributes}{Paid_Collection_Container_Type} == 2) { From 51d08d1cda002164163331ca920c9a177b904429 Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Tue, 20 Jun 2023 14:42:18 +0100 Subject: [PATCH 02/11] [Central Beds] Move to a multi integration. --- .../Integration/UK/CentralBedfordshire.pm | 81 +++--------------- .../UK/CentralBedfordshire/Symology.pm | 84 +++++++++++++++++++ ...shire.t => centralbedfordshire_symology.t} | 10 +-- t/open311/endpoint/uk.t | 5 +- 4 files changed, 103 insertions(+), 77 deletions(-) create mode 100644 perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm rename t/open311/endpoint/{centralbedfordshire.t => centralbedfordshire_symology.t} (97%) diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire.pm index 01963f8c0..d78b131e2 100644 --- a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire.pm +++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire.pm @@ -1,84 +1,23 @@ package Open311::Endpoint::Integration::UK::CentralBedfordshire; -# use SOAP::Lite +trace => [ qw/method debug/ ]; - use Moo; -extends 'Open311::Endpoint::Integration::Symology'; +extends 'Open311::Endpoint::Integration::Multi'; -use Open311::Endpoint::Service::UKCouncil::Symology::CentralBedfordshire; +use Module::Pluggable + search_path => ['Open311::Endpoint::Integration::UK::CentralBedfordshire'], + instantiate => 'new'; has jurisdiction_id => ( is => 'ro', + # Using 'centralbedfordshire_symology' to be consistent with jurisdiction + # before becoming a multi integration. + # Will want to migrate to just 'centralbedfordshire' at some point. default => 'centralbedfordshire_symology', ); -has service_class => ( +has integration_without_prefix => ( is => 'ro', - default => 'Open311::Endpoint::Service::UKCouncil::Symology::CentralBedfordshire' + default => 'Symology', ); -sub process_service_request_args { - my $self = shift; - - my $location = (delete $_[0]->{attributes}->{title}) || ''; - delete $_[0]->{attributes}->{description}; - delete $_[0]->{attributes}->{report_url}; - - my $area_code = (delete $_[0]->{attributes}->{area_code}) || ''; - my @args = $self->SUPER::process_service_request_args(@_); - my $response = $args[0]; - - my $lookup = $self->endpoint_config->{area_to_username}; - $response->{NextActionUserName} ||= $lookup->{$area_code}; - - $response->{Location} = $location; - - # Send the first photo to the appropriate field in Symology - # (these values are specific to Central Beds and were divined by - # inspecting the GetRequestAdditionalGroup output for an existing - # enquiry that had a photo.) - if ( my $photo_url = $_[0]->{media_url}->[0] ) { - $args[2] = [ - [ FieldLine => 15, ValueType => 8, DataValue => $photo_url ], - ]; - } - - return @args; -} - -# Unlike Bexley, the CSV from the SFTP doesn't have everything we need to -# build the ServiceRequestUpdates. We can get the full picture from the -# Symology API by calling the GetRequestAdditionalGroup method for each -# enquiry mentioned in the CSVs and looking at the history entries there. -sub post_process_files { - my ($self, $updates) = @_; - - my @updates; - push(@updates, @{ $self->_updates_for_crno($_) }) for @$updates; - - @$updates = @updates; -} - -sub _process_csv_row { - my ($self, $row, $dt) = @_; - return ($row->{CRNo}, $row->{CRNo}); -} - -sub _updates_for_crno { - my ($self, $crno) = @_; - - my $response = $self->get_integration->get_request( - "SERV", - $crno - ); - - if (($response->{StatusCode}//-1) != 0) { - my $error = $response->{StatusMessage}; - $self->logger->warn("Couldn't call GetRequestAdditionalGroup for CRNo $crno: $error"); - return []; - } - - return $self->_process_request_history($response, 'full'); -} - -1; +__PACKAGE__->run_if_script; diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm new file mode 100644 index 000000000..0b64beaa5 --- /dev/null +++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Symology.pm @@ -0,0 +1,84 @@ +package Open311::Endpoint::Integration::UK::CentralBedfordshire::Symology; + +# use SOAP::Lite +trace => [ qw/method debug/ ]; + +use Moo; +extends 'Open311::Endpoint::Integration::Symology'; + +use Open311::Endpoint::Service::UKCouncil::Symology::CentralBedfordshire; + +has jurisdiction_id => ( + is => 'ro', + default => 'centralbedfordshire_symology', +); + +has service_class => ( + is => 'ro', + default => 'Open311::Endpoint::Service::UKCouncil::Symology::CentralBedfordshire' +); + +sub process_service_request_args { + my $self = shift; + + my $location = (delete $_[0]->{attributes}->{title}) || ''; + delete $_[0]->{attributes}->{description}; + delete $_[0]->{attributes}->{report_url}; + + my $area_code = (delete $_[0]->{attributes}->{area_code}) || ''; + my @args = $self->SUPER::process_service_request_args(@_); + my $response = $args[0]; + + my $lookup = $self->endpoint_config->{area_to_username}; + $response->{NextActionUserName} ||= $lookup->{$area_code}; + + $response->{Location} = $location; + + # Send the first photo to the appropriate field in Symology + # (these values are specific to Central Beds and were divined by + # inspecting the GetRequestAdditionalGroup output for an existing + # enquiry that had a photo.) + if ( my $photo_url = $_[0]->{media_url}->[0] ) { + $args[2] = [ + [ FieldLine => 15, ValueType => 8, DataValue => $photo_url ], + ]; + } + + return @args; +} + +# Unlike Bexley, the CSV from the SFTP doesn't have everything we need to +# build the ServiceRequestUpdates. We can get the full picture from the +# Symology API by calling the GetRequestAdditionalGroup method for each +# enquiry mentioned in the CSVs and looking at the history entries there. +sub post_process_files { + my ($self, $updates) = @_; + + my @updates; + push(@updates, @{ $self->_updates_for_crno($_) }) for @$updates; + + @$updates = @updates; +} + +sub _process_csv_row { + my ($self, $row, $dt) = @_; + return ($row->{CRNo}, $row->{CRNo}); +} + +sub _updates_for_crno { + my ($self, $crno) = @_; + + my $response = $self->get_integration->get_request( + "SERV", + $crno + ); + + if (($response->{StatusCode}//-1) != 0) { + my $error = $response->{StatusMessage}; + $self->logger->warn("Couldn't call GetRequestAdditionalGroup for CRNo $crno: $error"); + return []; + } + + return $self->_process_request_history($response, 'full'); +} + +1; diff --git a/t/open311/endpoint/centralbedfordshire.t b/t/open311/endpoint/centralbedfordshire_symology.t similarity index 97% rename from t/open311/endpoint/centralbedfordshire.t rename to t/open311/endpoint/centralbedfordshire_symology.t index 9f12d2c01..f2980b81e 100644 --- a/t/open311/endpoint/centralbedfordshire.t +++ b/t/open311/endpoint/centralbedfordshire_symology.t @@ -121,15 +121,15 @@ $soap_lite->mock(call => sub { } }); -my $centralbeds_integ = Test::MockModule->new('Integrations::Symology'); -$centralbeds_integ->mock(config => sub { +my $centralbeds_symology_integ = Test::MockModule->new('Integrations::Symology'); +$centralbeds_symology_integ->mock(config => sub { { endpoint_url => 'http://www.example.org/', } }); -my $centralbeds_end = Test::MockModule->new('Open311::Endpoint::Integration::UK::CentralBedfordshire'); -$centralbeds_end->mock(endpoint_config => sub { +my $centralbeds_symology_end = Test::MockModule->new('Open311::Endpoint::Integration::UK::CentralBedfordshire::Symology'); +$centralbeds_symology_end->mock(endpoint_config => sub { { username => 'FMS', nsgref_to_action => {}, @@ -182,7 +182,7 @@ $centralbeds_end->mock(endpoint_config => sub { external_id_prefix => "FMS", } }); -$centralbeds_end->mock(config_file => sub { path('t')->absolute }); +$centralbeds_symology_end->mock(config_file => sub { path('t')->absolute }); use Open311::Endpoint::Integration::UK::CentralBedfordshire; diff --git a/t/open311/endpoint/uk.t b/t/open311/endpoint/uk.t index a02d0964f..c10316452 100644 --- a/t/open311/endpoint/uk.t +++ b/t/open311/endpoint/uk.t @@ -10,7 +10,6 @@ test_multi(0, 'Open311::Endpoint::Integration::UK', 'Open311::Endpoint::Integration::UK::Buckinghamshire' => 'buckinghamshire_confirm', 'Open311::Endpoint::Integration::UK::BuckinghamshireAlloy' => 'buckinghamshire_alloy', 'Open311::Endpoint::Integration::UK::Camden' => 'camden_symology', - 'Open311::Endpoint::Integration::UK::CentralBedfordshire' => 'centralbedfordshire_symology', 'Open311::Endpoint::Integration::UK::CheshireEast' => 'cheshireeast_confirm', 'Open311::Endpoint::Integration::UK::EastSussex' => 'eastsussex_salesforce', 'Open311::Endpoint::Integration::UK::Hounslow' => 'hounslow_confirm', @@ -58,6 +57,10 @@ test_multi(1, 'Open311::Endpoint::Integration::UK::Hackney', 'Open311::Endpoint::Integration::UK::Hackney::Environment' => 'hackney_environment_alloy_v2', ); +test_multi(1, 'Open311::Endpoint::Integration::UK::CentralBedfordshire', + 'Open311::Endpoint::Integration::UK::CentralBedfordshire::Symology' => 'centralbedfordshire_symology', +); + done_testing; sub test_multi { From c579b36b39138f757c23d75ca6e728e667894e92 Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Mon, 26 Jun 2023 18:02:20 +0100 Subject: [PATCH 03/11] Fix multivaluelist attribute handling. --- perllib/Open311/Endpoint.pm | 17 ++++++++++++++++- perllib/Open311/Endpoint/Service/Attribute.pm | 3 ++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/perllib/Open311/Endpoint.pm b/perllib/Open311/Endpoint.pm index e3066de16..ee0f01b03 100644 --- a/perllib/Open311/Endpoint.pm +++ b/perllib/Open311/Endpoint.pm @@ -302,8 +302,23 @@ sub dispatch_request { $self->call_api( GET_Service_Definition => $service_id, $args ); }, - sub (POST + /requests + %:@media_url~&* + **) { + sub (POST + /requests + %:@media_url~&@* + **) { my ($self, $args, $uploads) = @_; + + # We use '@*' to capture parameters as multiple in the hashref to handle attributes with multiple values. + # This puts every value in array, regardless of whether it's singleton or empty, which we don't want. + # The following block 'un-arrays' the value in these cases. + + while (my ($key, $value) = each %$args) { + next if $key eq 'media_url'; + my $length = scalar @$value; + if ($length == 1) { + $args->{$key} = $value->[0]; + } elsif ($length == 0) { + $args->{$key} = ""; + } + } + my @files = grep { $_->is_upload } values %$uploads; diff --git a/perllib/Open311/Endpoint/Service/Attribute.pm b/perllib/Open311/Endpoint/Service/Attribute.pm index 9fef6ce55..9745912cb 100644 --- a/perllib/Open311/Endpoint/Service/Attribute.pm +++ b/perllib/Open311/Endpoint/Service/Attribute.pm @@ -95,7 +95,8 @@ sub schema_definition { datetime => '//str', # TODO text => '//str', singlevaluelist => { type => '//any', of => [@values] }, - multivaluelist => { type => '//arr', of => [@values] }, + # Either a single value or a non-empty list of values. + multivaluelist => { type => '//any', of => [{ type => '//any', of => [@values]}, { type => '//arr', contents => { type => '//any', of => [@values] }, length => { min => 1 } }] }, ); return $schema_types{ $self->datatype }; From f6e9574bbf6ff07425ae9534e684774c208921a6 Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Thu, 22 Jun 2023 11:21:05 +0100 Subject: [PATCH 04/11] [Central Beds] Add Jadu endpoint. --- bin/centralbedfordshire/jadu/gather_updates | 16 + .../jadu/init_update_gathering_files | 28 + ...uncil-centralbedfordshire_jadu.yml-example | 22 + cpanfile | 2 + cpanfile.snapshot | 554 ++++++++++++++++++ perllib/Integrations/Jadu.pm | 171 ++++++ .../UK/CentralBedfordshire/Jadu.pm | 506 ++++++++++++++++ .../CentralBedfordshireFlytipping.pm | 125 ++++ t/open311/endpoint/centralbedfordshire_jadu.t | 280 +++++++++ .../endpoint/centralbedfordshire_symology.t | 2 + t/open311/endpoint/uk.t | 1 + 11 files changed, 1707 insertions(+) create mode 100755 bin/centralbedfordshire/jadu/gather_updates create mode 100755 bin/centralbedfordshire/jadu/init_update_gathering_files create mode 100644 conf/council-centralbedfordshire_jadu.yml-example create mode 100644 perllib/Integrations/Jadu.pm create mode 100644 perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm create mode 100644 perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm create mode 100644 t/open311/endpoint/centralbedfordshire_jadu.t diff --git a/bin/centralbedfordshire/jadu/gather_updates b/bin/centralbedfordshire/jadu/gather_updates new file mode 100755 index 000000000..3a7ff9d3e --- /dev/null +++ b/bin/centralbedfordshire/jadu/gather_updates @@ -0,0 +1,16 @@ +#!/usr/bin/env perl + +# Gathers status updates from Jadu for Central Bedfordshire and stores +# these in a file ready for use in future 'GET service request updates' calls. +# See 'gather_updates' in the centralbedfordshire/Jadu endpoint for more information. + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../../setenv.pl"; +} + +use Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu; +my $endpoint = Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu->new(); +$endpoint->gather_updates; diff --git a/bin/centralbedfordshire/jadu/init_update_gathering_files b/bin/centralbedfordshire/jadu/init_update_gathering_files new file mode 100755 index 000000000..029ac829b --- /dev/null +++ b/bin/centralbedfordshire/jadu/init_update_gathering_files @@ -0,0 +1,28 @@ +#!/usr/bin/env perl + +# Initialises the files needed for the 'gather_updates' script. +# Requires a number of lookback-days to be specied which is the number of days +# in the past to start update collection from. +# See 'init_update_gathering_files' in the centralbedfordshire/Jadu endpoint for more information. + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../../setenv.pl"; +} + +use DateTime; +use Getopt::Long; +use Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu; + +my $lookback_days; +GetOptions("lookback-days=i" => \$lookback_days); +if (!$lookback_days) { + print "Please specify a number of days to look back for updates via --lookback-days"; + exit 1; +} + +my $start_from = DateTime->now->subtract(days => $lookback_days); +my $endpoint = Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu->new(); +$endpoint->init_update_gathering_files($start_from); diff --git a/conf/council-centralbedfordshire_jadu.yml-example b/conf/council-centralbedfordshire_jadu.yml-example new file mode 100644 index 000000000..f804eb0a4 --- /dev/null +++ b/conf/council-centralbedfordshire_jadu.yml-example @@ -0,0 +1,22 @@ +api_base_url: '' +api_key: '' +username: '' +password: '' + +case_type: '' +sys_channel: '' + +most_recently_updated_cases_filter: '' +case_status_to_fms_status: + 'case status': 'fms_status' +case_status_to_fms_status_timed: + 'case status 2': + fms_status: 'fms_status' + days_to_wait: 1 +case_status_tracking_file: '' +case_status_tracking_max_age_days: 365 +update_storage_file: '' +update_storage_max_age_days: 365 + +town_to_officer: + 'Town': 'officer' diff --git a/cpanfile b/cpanfile index 559c2a4d3..6dd1b93b7 100644 --- a/cpanfile +++ b/cpanfile @@ -13,6 +13,7 @@ requires 'Crypt::JWT'; requires 'Data::Dumper'; requires 'Data::Rx'; requires 'DateTime', '1.38'; +requires 'DateTime::Format::ISO8601'; requires 'DateTime::Format::Oracle'; requires 'DateTime::Format::W3CDTF', '>= 0.07'; # requires 'DBD::Oracle'; @@ -45,6 +46,7 @@ requires 'Module::Loaded'; requires 'Object::Tiny'; requires 'Test::Exception'; requires 'Test::LongString'; +requires 'Test::MockFile'; requires 'Test::MockModule'; requires 'Test::MockTime'; requires 'Test::More'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 1d924cb04..e08f8a610 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -494,6 +494,13 @@ DISTRIBUTIONS parent 0 perl 5.012 warnings 0 + Data-UUID-1.226 + pathname: R/RJ/RJBS/Data-UUID-1.226.tar.gz + provides: + Data::UUID 1.226 + requirements: + Digest::MD5 0 + ExtUtils::MakeMaker 0 DateTime-1.59 pathname: D/DR/DROLSKY/DateTime-1.59.tar.gz provides: @@ -553,6 +560,25 @@ DISTRIBUTIONS parent 0 strict 0 warnings 0 + DateTime-Format-ISO8601-0.16 + pathname: D/DR/DROLSKY/DateTime-Format-ISO8601-0.16.tar.gz + provides: + DateTime::Format::ISO8601 0.16 + DateTime::Format::ISO8601::Types 0.16 + requirements: + Carp 0 + DateTime 1.45 + DateTime::Format::Builder 0.77 + ExtUtils::MakeMaker 0 + Params::ValidationCompiler 0.26 + Specio 0.18 + Specio::Declare 0 + Specio::Exporter 0 + Specio::Library::Builtins 0 + namespace::autoclean 0 + parent 0 + strict 0 + warnings 0 DateTime-Format-Oracle-0.06 pathname: K/KO/KOLIBRIE/DateTime-Format-Oracle-0.06.tar.gz provides: @@ -1214,6 +1240,20 @@ DISTRIBUTIONS perl 5.006 strict 0 warnings 0 + File-Slurper-0.014 + pathname: L/LE/LEONT/File-Slurper-0.014.tar.gz + provides: + File::Slurper 0.014 + requirements: + Carp 0 + Encode 2.11 + Exporter 5.57 + ExtUtils::MakeMaker 0 + PerlIO::encoding 0 + constant 0 + perl 5.008 + strict 0 + warnings 0 Filesys-Notify-Simple-0.14 pathname: M/MI/MIYAGAWA/Filesys-Notify-Simple-0.14.tar.gz provides: @@ -1438,6 +1478,13 @@ DISTRIBUTIONS Mozilla::CA 0 Net::SSLeay 1.46 Scalar::Util 0 + Importer-0.026 + pathname: E/EX/EXODIST/Importer-0.026.tar.gz + provides: + Importer 0.026 + requirements: + ExtUtils::MakeMaker 0 + perl 5.008001 JSON-4.10 pathname: I/IS/ISHIGAKI/JSON-4.10.tar.gz provides: @@ -1624,6 +1671,13 @@ DISTRIBUTIONS File::Path 2.07 File::Spec 0.82 perl 5.006 + Long-Jump-0.000001 + pathname: E/EX/EXODIST/Long-Jump-0.000001.tar.gz + provides: + Long::Jump 0.000001 + requirements: + ExtUtils::MakeMaker 0 + perl 5.008001 MRO-Compat-0.15 pathname: H/HA/HAARG/MRO-Compat-0.15.tar.gz provides: @@ -1891,6 +1945,13 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 perl 5.006 strict 0 + Overload-FileCheck-0.013 + pathname: A/AT/ATOOMIC/Overload-FileCheck-0.013.tar.gz + provides: + Overload::FileCheck 0.013 + requirements: + ExtUtils::MakeMaker 0 + perl 5.010 POSIX-strftime-Compiler-0.44 pathname: K/KA/KAZEBURO/POSIX-strftime-Compiler-0.44.tar.gz provides: @@ -2217,6 +2278,14 @@ DISTRIBUTIONS Sub::Identify 0.03 Test::Simple 0.61 perl v5.6.2 + Scope-Guard-0.21 + pathname: C/CH/CHOCOLATE/Scope-Guard-0.21.tar.gz + provides: + Scope::Guard 0.21 + requirements: + ExtUtils::MakeMaker 0 + Test::More 0 + perl 5.006001 Specio-0.48 pathname: D/DR/DROLSKY/Specio-0.48.tar.gz provides: @@ -2458,6 +2527,23 @@ DISTRIBUTIONS File::Spec 0.8 File::Temp 0.12 Scalar::Util 0 + Term-Table-0.016 + pathname: E/EX/EXODIST/Term-Table-0.016.tar.gz + provides: + Term::Table 0.016 + Term::Table::Cell 0.016 + Term::Table::CellStack 0.016 + Term::Table::HashBase 0.016 + Term::Table::LineBreak 0.016 + Term::Table::Spacer 0.016 + Term::Table::Util 0.016 + requirements: + Carp 0 + ExtUtils::MakeMaker 0 + Importer 0.024 + List::Util 0 + Scalar::Util 0 + perl 5.008001 Test-Deep-1.204 pathname: R/RJ/RJBS/Test-Deep-1.204.tar.gz provides: @@ -2555,6 +2641,29 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 Test::Builder 0.12 Test::Builder::Tester 1.04 + Test-MockFile-0.035 + pathname: T/TO/TODDR/Test-MockFile-0.035.tar.gz + provides: + Test::MockFile 0.035 + Test::MockFile::DirHandle 0.035 + Test::MockFile::FileHandle 0.035 + Test::MockFile::Plugin 0.035 + Test::MockFile::Plugin::FileTemp 0.035 + Test::MockFile::Plugins 0.035 + requirements: + ExtUtils::MakeMaker 0 + File::Basename 0 + File::Slurper 0 + File::Temp 0 + Overload::FileCheck 0.013 + Test2::Bundle::Extended 0.000084 + Test2::Harness::Util::IPC 0 + Test2::Plugin::NoWarnings 0 + Test2::Tools::Explain 0 + Test::MockModule 0 + Test::More 1.302133 + Text::Glob 0 + perl 5.006 Test-MockModule-v0.177.0 pathname: G/GF/GFRANKS/Test-MockModule-v0.177.0.tar.gz provides: @@ -2598,6 +2707,89 @@ DISTRIBUTIONS Test::Builder::Module 0 Test::More 0.88 perl 5.008_001 + Test-Simple-1.302195 + pathname: E/EX/EXODIST/Test-Simple-1.302195.tar.gz + provides: + Test2 1.302195 + Test2::API 1.302195 + Test2::API::Breakage 1.302195 + Test2::API::Context 1.302195 + Test2::API::Instance 1.302195 + Test2::API::InterceptResult 1.302195 + Test2::API::InterceptResult::Event 1.302195 + Test2::API::InterceptResult::Facet 1.302195 + Test2::API::InterceptResult::Hub 1.302195 + Test2::API::InterceptResult::Squasher 1.302195 + Test2::API::Stack 1.302195 + Test2::Event 1.302195 + Test2::Event::Bail 1.302195 + Test2::Event::Diag 1.302195 + Test2::Event::Encoding 1.302195 + Test2::Event::Exception 1.302195 + Test2::Event::Fail 1.302195 + Test2::Event::Generic 1.302195 + Test2::Event::Note 1.302195 + Test2::Event::Ok 1.302195 + Test2::Event::Pass 1.302195 + Test2::Event::Plan 1.302195 + Test2::Event::Skip 1.302195 + Test2::Event::Subtest 1.302195 + Test2::Event::TAP::Version 1.302195 + Test2::Event::V2 1.302195 + Test2::Event::Waiting 1.302195 + Test2::EventFacet 1.302195 + Test2::EventFacet::About 1.302195 + Test2::EventFacet::Amnesty 1.302195 + Test2::EventFacet::Assert 1.302195 + Test2::EventFacet::Control 1.302195 + Test2::EventFacet::Error 1.302195 + Test2::EventFacet::Hub 1.302195 + Test2::EventFacet::Info 1.302195 + Test2::EventFacet::Info::Table 1.302195 + Test2::EventFacet::Meta 1.302195 + Test2::EventFacet::Parent 1.302195 + Test2::EventFacet::Plan 1.302195 + Test2::EventFacet::Render 1.302195 + Test2::EventFacet::Trace 1.302195 + Test2::Formatter 1.302195 + Test2::Formatter::TAP 1.302195 + Test2::Hub 1.302195 + Test2::Hub::Interceptor 1.302195 + Test2::Hub::Interceptor::Terminator 1.302195 + Test2::Hub::Subtest 1.302195 + Test2::IPC 1.302195 + Test2::IPC::Driver 1.302195 + Test2::IPC::Driver::Files 1.302195 + Test2::Tools::Tiny 1.302195 + Test2::Util 1.302195 + Test2::Util::ExternalMeta 1.302195 + Test2::Util::Facets2Legacy 1.302195 + Test2::Util::HashBase 1.302195 + Test2::Util::Trace 1.302195 + Test::Builder 1.302195 + Test::Builder::Formatter 1.302195 + Test::Builder::IO::Scalar 2.114 + Test::Builder::Module 1.302195 + Test::Builder::Tester 1.302195 + Test::Builder::Tester::Color 1.302195 + Test::Builder::Tester::Tie 1.302195 + Test::Builder::TodoDiag 1.302195 + Test::More 1.302195 + Test::Simple 1.302195 + Test::Tester 1.302195 + Test::Tester::Capture 1.302195 + Test::Tester::CaptureRunner 1.302195 + Test::Tester::Delegate 1.302195 + Test::use::ok 1.302195 + ok 1.302195 + requirements: + ExtUtils::MakeMaker 0 + File::Spec 0 + File::Temp 0 + Scalar::Util 1.13 + Storable 0 + perl 5.006002 + utf8 0 Test-TCP-2.22 pathname: M/MI/MIYAGAWA/Test-TCP-2.22.tar.gz provides: @@ -2623,6 +2815,346 @@ DISTRIBUTIONS Test::Builder 0.13 Test::Builder::Tester 1.02 perl 5.006 + Test2-Harness-1.000152 + pathname: E/EX/EXODIST/Test2-Harness-1.000152.tar.gz + provides: + App::Yath 1.000152 + App::Yath::Command 1.000152 + App::Yath::Command::abort 1.000152 + App::Yath::Command::auditor 1.000152 + App::Yath::Command::collector 1.000152 + App::Yath::Command::do 1.000152 + App::Yath::Command::failed 1.000152 + App::Yath::Command::help 1.000152 + App::Yath::Command::init 1.000152 + App::Yath::Command::kill 1.000152 + App::Yath::Command::projects 1.000152 + App::Yath::Command::ps 1.000152 + App::Yath::Command::reload 1.000152 + App::Yath::Command::replay 1.000152 + App::Yath::Command::resources 1.000152 + App::Yath::Command::run 1.000152 + App::Yath::Command::runner 1.000152 + App::Yath::Command::spawn 1.000152 + App::Yath::Command::speedtag 1.000152 + App::Yath::Command::start 1.000152 + App::Yath::Command::status 1.000152 + App::Yath::Command::stop 1.000152 + App::Yath::Command::test 1.000152 + App::Yath::Command::times 1.000152 + App::Yath::Command::watch 1.000152 + App::Yath::Command::which 1.000152 + App::Yath::Converting 1.000152 + App::Yath::Option 1.000152 + App::Yath::Options 1.000152 + App::Yath::Options::Collector 1.000152 + App::Yath::Options::Debug 1.000152 + App::Yath::Options::Display 1.000152 + App::Yath::Options::Finder 1.000152 + App::Yath::Options::Logging 1.000152 + App::Yath::Options::Persist 1.000152 + App::Yath::Options::PreCommand 1.000152 + App::Yath::Options::Run 1.000152 + App::Yath::Options::Runner 1.000152 + App::Yath::Options::Workspace 1.000152 + App::Yath::Plugin 1.000152 + App::Yath::Plugin::Cover 1.000152 + App::Yath::Plugin::Git 1.000152 + App::Yath::Plugin::Notify 1.000152 + App::Yath::Plugin::SelfTest undef + App::Yath::Plugin::SysInfo 1.000152 + App::Yath::Plugin::YathUI 1.000152 + App::Yath::Tester 1.000152 + App::Yath::Util 1.000152 + Test2::Formatter::QVF 1.000152 + Test2::Formatter::Stream 1.000152 + Test2::Formatter::Test2 1.000152 + Test2::Formatter::Test2::Composer 1.000152 + Test2::Harness 1.000152 + Test2::Harness::Auditor 1.000152 + Test2::Harness::Auditor::TimeTracker 1.000152 + Test2::Harness::Auditor::Watcher 1.000152 + Test2::Harness::Collector 1.000152 + Test2::Harness::Collector::JobDir 1.000152 + Test2::Harness::Collector::TapParser 1.000152 + Test2::Harness::Event 1.000152 + Test2::Harness::Finder 1.000152 + Test2::Harness::IPC 1.000152 + Test2::Harness::IPC::Process 1.000152 + Test2::Harness::Log 1.000152 + Test2::Harness::Log::CoverageAggregator 1.000152 + Test2::Harness::Log::CoverageAggregator::ByRun 1.000152 + Test2::Harness::Log::CoverageAggregator::ByTest 1.000152 + Test2::Harness::Plugin 1.000152 + Test2::Harness::Renderer 1.000152 + Test2::Harness::Renderer::Formatter 1.000152 + Test2::Harness::Run 1.000152 + Test2::Harness::Runner 1.000152 + Test2::Harness::Runner::Constants 1.000152 + Test2::Harness::Runner::DepTracer 1.000152 + Test2::Harness::Runner::Job 1.000152 + Test2::Harness::Runner::Preload 1.000152 + Test2::Harness::Runner::Preload::Stage 1.000152 + Test2::Harness::Runner::Preloader 1.000152 + Test2::Harness::Runner::Preloader::Stage 1.000152 + Test2::Harness::Runner::Reloader 1.000152 + Test2::Harness::Runner::Resource 1.000152 + Test2::Harness::Runner::Resource::JobCount 1.000152 + Test2::Harness::Runner::Resource::SharedJobSlots 1.000152 + Test2::Harness::Runner::Resource::SharedJobSlots::Config 1.000152 + Test2::Harness::Runner::Resource::SharedJobSlots::State 1.000152 + Test2::Harness::Runner::Run 1.000152 + Test2::Harness::Runner::Spawn 1.000152 + Test2::Harness::Runner::Spawn::Run 1.000152 + Test2::Harness::Runner::State 1.000152 + Test2::Harness::Settings 1.000152 + Test2::Harness::Settings::Prefix 1.000152 + Test2::Harness::TestFile 1.000152 + Test2::Harness::Util 1.000152 + Test2::Harness::Util::File 1.000152 + Test2::Harness::Util::File::JSON 1.000152 + Test2::Harness::Util::File::JSONL 1.000152 + Test2::Harness::Util::File::Stream 1.000152 + Test2::Harness::Util::File::Value 1.000152 + Test2::Harness::Util::HashBase 1.000152 + Test2::Harness::Util::IPC 1.000152 + Test2::Harness::Util::JSON 1.000152 + Test2::Harness::Util::Queue 1.000152 + Test2::Harness::Util::Term 1.000152 + Test2::Harness::Util::UUID 1.000152 + Test2::Tools::HarnessTester 1.000152 + requirements: + Carp 0 + Config 0 + Cwd 0 + Data::Dumper 0 + Data::UUID 0 + Exporter 0 + ExtUtils::MakeMaker 0 + Fcntl 0 + File::Find 0 + File::Path 2.11 + File::Spec 0 + File::Temp 0 + Filter::Util::Call 0 + IO::Compress::Bzip2 0 + IO::Compress::Gzip 0 + IO::Handle 1.27 + IO::Uncompress::Bunzip2 0 + IO::Uncompress::Gunzip 0 + IPC::Cmd 0 + Importer 0.025 + JSON::PP 0 + List::Util 1.44 + Long::Jump 0.000001 + POSIX 0 + Scalar::Util 0 + Scope::Guard 0 + Symbol 0 + Sys::Hostname 0 + Term::Table 0.015 + Test2 1.302170 + Test2::API 1.302170 + Test2::Bundle::Extended 0.000127 + Test2::Event 1.302170 + Test2::Event::V2 1.302170 + Test2::Formatter 1.302170 + Test2::Plugin::MemUsage 0.002003 + Test2::Plugin::UUID 0.002001 + Test2::Tools::AsyncSubtest 0.000127 + Test2::Tools::Subtest 0.000127 + Test2::Util 1.302170 + Test2::Util::Term 0.000127 + Test2::V0 0.000127 + Test::Builder 1.302170 + Test::Builder::Formatter 1.302170 + Test::More 1.302170 + Text::ParseWords 0 + Time::HiRes 0 + YAML::Tiny 0 + base 0 + constant 0 + goto::file 0.005 + parent 0 + perl 5.010000 + Test2-Plugin-MemUsage-0.002003 + pathname: E/EX/EXODIST/Test2-Plugin-MemUsage-0.002003.tar.gz + provides: + Test2::Plugin::MemUsage 0.002003 + requirements: + ExtUtils::MakeMaker 0 + Test2::API 1.302165 + Test2::Event::V2 1.302165 + perl 5.008009 + Test2-Plugin-NoWarnings-0.09 + pathname: D/DR/DROLSKY/Test2-Plugin-NoWarnings-0.09.tar.gz + provides: + Test2::Event::Warning 0.09 + Test2::Plugin::NoWarnings 0.09 + requirements: + Carp 0 + ExtUtils::MakeMaker 0 + Test2 1.302167 + Test2::API 0 + Test2::Event 0 + Test2::Util::HashBase 0 + parent 0 + strict 0 + warnings 0 + Test2-Plugin-UUID-0.002001 + pathname: E/EX/EXODIST/Test2-Plugin-UUID-0.002001.tar.gz + provides: + Test2::Plugin::UUID 0.002001 + requirements: + Data::UUID 1.148 + ExtUtils::MakeMaker 0 + Test2::API 1.302165 + perl 5.008009 + Test2-Suite-0.000155 + pathname: E/EX/EXODIST/Test2-Suite-0.000155.tar.gz + provides: + Test2::AsyncSubtest 0.000155 + Test2::AsyncSubtest::Event::Attach 0.000155 + Test2::AsyncSubtest::Event::Detach 0.000155 + Test2::AsyncSubtest::Formatter 0.000155 + Test2::AsyncSubtest::Hub 0.000155 + Test2::Bundle 0.000155 + Test2::Bundle::Extended 0.000155 + Test2::Bundle::More 0.000155 + Test2::Bundle::Simple 0.000155 + Test2::Compare 0.000155 + Test2::Compare::Array 0.000155 + Test2::Compare::Bag 0.000155 + Test2::Compare::Base 0.000155 + Test2::Compare::Bool 0.000155 + Test2::Compare::Custom 0.000155 + Test2::Compare::DeepRef 0.000155 + Test2::Compare::Delta 0.000155 + Test2::Compare::Event 0.000155 + Test2::Compare::EventMeta 0.000155 + Test2::Compare::Float 0.000155 + Test2::Compare::Hash 0.000155 + Test2::Compare::Isa 0.000155 + Test2::Compare::Meta 0.000155 + Test2::Compare::Negatable 0.000155 + Test2::Compare::Number 0.000155 + Test2::Compare::Object 0.000155 + Test2::Compare::OrderedSubset 0.000155 + Test2::Compare::Pattern 0.000155 + Test2::Compare::Ref 0.000155 + Test2::Compare::Regex 0.000155 + Test2::Compare::Scalar 0.000155 + Test2::Compare::Set 0.000155 + Test2::Compare::String 0.000155 + Test2::Compare::Undef 0.000155 + Test2::Compare::Wildcard 0.000155 + Test2::Manual 0.000155 + Test2::Manual::Anatomy 0.000155 + Test2::Manual::Anatomy::API 0.000155 + Test2::Manual::Anatomy::Context 0.000155 + Test2::Manual::Anatomy::EndToEnd 0.000155 + Test2::Manual::Anatomy::Event 0.000155 + Test2::Manual::Anatomy::Hubs 0.000155 + Test2::Manual::Anatomy::IPC 0.000155 + Test2::Manual::Anatomy::Utilities 0.000155 + Test2::Manual::Concurrency 0.000155 + Test2::Manual::Contributing 0.000155 + Test2::Manual::Testing 0.000155 + Test2::Manual::Testing::Introduction 0.000155 + Test2::Manual::Testing::Migrating 0.000155 + Test2::Manual::Testing::Planning 0.000155 + Test2::Manual::Testing::Todo 0.000155 + Test2::Manual::Tooling 0.000155 + Test2::Manual::Tooling::FirstTool 0.000155 + Test2::Manual::Tooling::Formatter 0.000155 + Test2::Manual::Tooling::Nesting 0.000155 + Test2::Manual::Tooling::Plugin::TestExit 0.000155 + Test2::Manual::Tooling::Plugin::TestingDone 0.000155 + Test2::Manual::Tooling::Plugin::ToolCompletes 0.000155 + Test2::Manual::Tooling::Plugin::ToolStarts 0.000155 + Test2::Manual::Tooling::Subtest 0.000155 + Test2::Manual::Tooling::TestBuilder 0.000155 + Test2::Manual::Tooling::Testing 0.000155 + Test2::Mock 0.000155 + Test2::Plugin 0.000155 + Test2::Plugin::BailOnFail 0.000155 + Test2::Plugin::DieOnFail 0.000155 + Test2::Plugin::ExitSummary 0.000155 + Test2::Plugin::SRand 0.000155 + Test2::Plugin::Times 0.000155 + Test2::Plugin::UTF8 0.000155 + Test2::Require 0.000155 + Test2::Require::AuthorTesting 0.000155 + Test2::Require::EnvVar 0.000155 + Test2::Require::Fork 0.000155 + Test2::Require::Module 0.000155 + Test2::Require::Perl 0.000155 + Test2::Require::RealFork 0.000155 + Test2::Require::Threads 0.000155 + Test2::Suite 0.000155 + Test2::Todo 0.000155 + Test2::Tools 0.000155 + Test2::Tools::AsyncSubtest 0.000155 + Test2::Tools::Basic 0.000155 + Test2::Tools::Class 0.000155 + Test2::Tools::ClassicCompare 0.000155 + Test2::Tools::Compare 0.000155 + Test2::Tools::Defer 0.000155 + Test2::Tools::Encoding 0.000155 + Test2::Tools::Event 0.000155 + Test2::Tools::Exception 0.000155 + Test2::Tools::Exports 0.000155 + Test2::Tools::GenTemp 0.000155 + Test2::Tools::Grab 0.000155 + Test2::Tools::Mock 0.000155 + Test2::Tools::Ref 0.000155 + Test2::Tools::Refcount 0.000155 + Test2::Tools::Spec 0.000155 + Test2::Tools::Subtest 0.000155 + Test2::Tools::Target 0.000155 + Test2::Tools::Tester 0.000155 + Test2::Tools::Warnings 0.000155 + Test2::Util::Grabber 0.000155 + Test2::Util::Guard 0.000155 + Test2::Util::Importer 0.000155 + Test2::Util::Ref 0.000155 + Test2::Util::Stash 0.000155 + Test2::Util::Sub 0.000155 + Test2::Util::Table 0.000155 + Test2::Util::Table::Cell 0.000155 + Test2::Util::Table::LineBreak 0.000155 + Test2::Util::Term 0.000155 + Test2::Util::Times 0.000155 + Test2::V0 0.000155 + Test2::Workflow 0.000155 + Test2::Workflow::BlockBase 0.000155 + Test2::Workflow::Build 0.000155 + Test2::Workflow::Runner 0.000155 + Test2::Workflow::Task 0.000155 + Test2::Workflow::Task::Action 0.000155 + Test2::Workflow::Task::Group 0.000155 + requirements: + B 0 + Carp 0 + Data::Dumper 0 + Exporter 0 + ExtUtils::MakeMaker 0 + Scalar::Util 0 + Term::Table 0.013 + Test2::API 1.302176 + Time::HiRes 0 + overload 0 + perl 5.008001 + utf8 0 + Test2-Tools-Explain-0.02 + pathname: P/PE/PETDANCE/Test2-Tools-Explain-0.02.tar.gz + provides: + Test2::Tools::Explain 0.02 + requirements: + ExtUtils::MakeMaker 0 + Test2::Suite 0 + parent 0 + perl 5.008001 Text-CSV-2.02 pathname: I/IS/ISHIGAKI/Text-CSV-2.02.tar.gz provides: @@ -3028,6 +3560,20 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 Test::More 0 perl 5.006 + YAML-Tiny-1.74 + pathname: E/ET/ETHER/YAML-Tiny-1.74.tar.gz + provides: + YAML::Tiny 1.74 + requirements: + B 0 + Carp 0 + Exporter 0 + ExtUtils::MakeMaker 0 + Fcntl 0 + Scalar::Util 0 + perl 5.008001 + strict 0 + warnings 0 bareword-filehandles-0.007 pathname: I/IL/ILMARI/bareword-filehandles-0.007.tar.gz provides: @@ -3042,6 +3588,14 @@ DISTRIBUTIONS perl 5.008001 strict 0 warnings 0 + goto-file-0.005 + pathname: E/EX/EXODIST/goto-file-0.005.tar.gz + provides: + goto::file 0.005 + requirements: + ExtUtils::MakeMaker 0 + Filter::Util::Call 0 + perl 5.008001 indirect-0.39 pathname: V/VP/VPIT/indirect-0.39.tar.gz provides: diff --git a/perllib/Integrations/Jadu.pm b/perllib/Integrations/Jadu.pm new file mode 100644 index 000000000..f56700b05 --- /dev/null +++ b/perllib/Integrations/Jadu.pm @@ -0,0 +1,171 @@ +package Integrations::Jadu; + +use v5.14; +use warnings; + +use Data::Dumper; +use HTTP::Request; +use HTTP::Request::Common; +use JSON::MaybeXS qw(decode_json encode_json); +use LWP::UserAgent; +use Moo; + +with 'Role::Config'; +with 'Role::Logger'; +with 'Role::Memcached'; + +has base_url => ( + is => 'lazy', + default => sub { $_[0]->config->{api_base_url} } +); + +has api_key => ( + is => 'lazy', + default => sub { $_[0]->config->{api_key} } +); + +has username => ( + is => 'lazy', + default => sub { $_[0]->config->{username} } +); + +has password => ( + is => 'lazy', + default => sub { $_[0]->config->{password} } +); + +has ua => ( + is => 'lazy', + default => sub { LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter") } +); + +has api_token => ( + is => 'lazy', + default => sub { + my $self = shift; + my $token = $self->memcache->get('jadu_api_token'); + unless ($token) { + $token = $self->sign_in_and_get_token; + $self->memcache->set('jadu_api_token', $token, time() + $self->api_token_expiry_seconds); + } + return $token; + }, +); + +has api_token_expiry_seconds => ( + is => 'ro', + default => sub { 60 * 30 }, +); + +has api_key_header_name => ( + is => 'ro', + default => 'X-API-KEY', +); + +has api_token_header_name => ( + is => 'ro', + default => 'X-API-TOKEN', +); + + +sub _fail { + my ($self, $request, $response, $message) = @_; + my $log = sprintf( + "%s + Sent: %s + Got: %s", + $message, Dumper($request), Dumper($response) + ); + $self->logger->error($log); + die $message; +} + +sub sign_in_and_get_token { + my $self = shift; + my $request = HTTP::Request->new( + 'POST', + $self->base_url . "sign-in", + [ + $self->api_key_header_name => $self->api_key, + "Content-Type" => "application/json", + ], + encode_json({ + username => $self->username, + password => $self->password, + }) + ); + my $response = $self->ua->request($request); + if (!$response->is_success) { + $self->_fail($request, $response, "sign-in failed"); + } + my $response_json = decode_json($response->content); + if (!$response_json->{token}) { + $self->_fail($request, $response, "token not found in sign-in response"); + } + return $response_json->{token}; +} + +sub create_case_and_get_reference { + my ($self, $case_type, $payload) = @_; + my $url = $self->base_url . $case_type . "/case/create"; + my $request = HTTP::Request::Common::POST( + $url, + $self->api_key_header_name => $self->api_key, + $self->api_token_header_name => $self->api_token, + "Content-Type" => "application/json", + Content => encode_json($payload) + ); + + $self->logger->debug($url . " sending:\n" . Dumper($payload)); + my $response = $self->ua->request($request); + if (!$response->is_success) { + $self->_fail($request, $response, "create case failed"); + } + my $response_json = decode_json($response->content); + if (!$response_json->{reference}) { + $self->_fail($request, $response, "case reference not found in create case response"); + } + $self->logger->debug($url . " got:\n" . Dumper($response_json)); + return $response_json->{reference}; +} + +sub attach_file_to_case { + my ($self, $case_type, $case_reference, $filepath, $filename) = @_; + my $url = $self->base_url . "case/" . $case_reference . "/attach"; + my $content = [ + json => encode_json({name => $filename}), + file => [$filepath] + ]; + my $request = HTTP::Request::Common::POST( + $url, + $self->api_key_header_name => $self->api_key, + $self->api_token_header_name => $self->api_token, + "Content-Type" => "form-data", + Content => $content + ); + $self->logger->debug($url . " sending:\n" . Dumper($content)); + my $response = $self->ua->request($request); + if (!$response->is_success) { + $self->_fail($request, $response, "failed to attach file to case"); + } + $self->logger->debug($url . " got:\n" . Dumper(decode_json($response->content))); +} + +sub get_case_summaries_by_filter { + my ($self, $case_type, $filter_name, $page_number) = @_; + my $url = $self->base_url . "filters/" . $filter_name . "/summaries?page=" . $page_number; + my $request = HTTP::Request::Common::GET( + $url, + $self->api_key_header_name => $self->api_key, + $self->api_token_header_name => $self->api_token, + ); + my $response = $self->ua->request($request); + if (!$response->is_success) { + $self->_fail($request, $response, "failed to get case summaries by filter"); + } + my $response_json = decode_json($response->content); + $self->logger->debug($url . " got: " . Dumper($response_json)); + return $response_json; +} + +1; diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm new file mode 100644 index 000000000..ce8865901 --- /dev/null +++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm @@ -0,0 +1,506 @@ +package Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu; + +=head1 NAME + +Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu - +A Jadu integration specifically for Central Bedfordshire's Fly Tipping service. + +=head1 SYNOPSIS + +This integration provides a 'Fly Tipping' service. +Posted service requests have cases created in Jadu. +FMS relevant case status changes in Jadu are returned as service request updates. +Posted updates are not sent to Jadu. + +=cut + +use v5.14; +use warnings; + +use Moo; + +extends 'Open311::Endpoint'; +with 'Open311::Endpoint::Role::mySociety'; +with 'Role::EndpointConfig'; +with 'Role::Logger'; + +use DateTime::Format::ISO8601; +use DateTime::Format::W3CDTF; +use Fcntl qw(:flock); +use File::Temp qw(tempfile); +use Integrations::Jadu; +use JSON::MaybeXS qw(decode_json encode_json); +use LWP::Simple; +use Open311::Endpoint::Service::Attribute; +use Open311::Endpoint::Service::UKCouncil::CentralBedfordshireFlytipping; +use Open311::Endpoint::Service::Request::Update::mySociety; +use Path::Tiny; +use Try::Tiny; + +=head1 CONFIGURATION + +=cut + +has jurisdiction_id => ( + is => 'ro', + default => 'centralbedfordshire_jadu', +); + +has jadu => ( + is => 'lazy', + default => sub { Integrations::Jadu->new(config_filename => $_[0]->jurisdiction_id) } +); + +has flytipping_service => ( + is => 'lazy', + default => sub { + Open311::Endpoint::Service::UKCouncil::CentralBedfordshireFlytipping->new( + # TODO: Will be renamed to just "Fly Tipping" when ready to replace existing category. + service_name => "Fly Tipping (Jadu)", + group => "Flytipping, Bins and Graffiti", + service_code => "fly-tipping", + description => "Fly Tipping", + ); + } +); + +sub get_integration { + return $_[0]->jadu; +} + +=head2 sys_channel + +This is the value to set for the 'sys-channel' field when creating a new Fly Tipping case. + +=cut + +has sys_channel => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{sys_channel} } +); + +=head2 case_type + +This is the name of the type of case to use when creating a new Fly Tipping case. + +=cut + +has case_type => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{case_type} } +); + +=head2 town_to_officer + +This is a mapping from any town associated with an address in Central Bedfordshire to +the value that should be set in the 'eso-officer' field when creating a new Fly Tipping case. + +=cut + +has town_to_officer => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{town_to_officer} } +); + +=head2 most_recently_updated_cases_filter + +This is the name of a filter in Jadu that has been configured to return the most recently +updated Fly Tipping cases. + +=cut + +has most_recently_updated_cases_filter => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{most_recently_updated_cases_filter} } +); + +=head2 case_status_to_fms_status + +This is a mapping from Jadu Fly Tipping case status labels to a corresponding status in FMS. + +=cut + +has case_status_to_fms_status => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{case_status_to_fms_status} } +); + +=head2 case_status_to_fms_status_timed + +Similiar to C<case_status_to_fms_status> but additionally specifies a number of days to wait +before the FMS status is transitioned. +Maps from the Jadu case status to a mapping including 'fms_status' and 'days_to_wait'. + +=cut + +has case_status_to_fms_status_timed => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{case_status_to_fms_status_timed} } +); + +=head2 case_status_tracking_file + +The path to a file used by C<gather_updates> to track current case statuses. + +=cut + +has case_status_tracking_file => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{case_status_tracking_file} } +); + +=head2 update_storage_file + +The path to a file containing a sorted JSON list of updates, maintained by C<gather_updates> and consumed by C<get_service_request_updates>. + +=cut + +has update_storage_file => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{update_storage_file} } +); + +=head2 case_status_tracking_max_age_days + +The maximum age of a case to track status updates for. Used by C<gather_updates>. + +=cut + +has case_status_tracking_max_age_days => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{case_status_tracking_max_age_days} } +); + +=head2 update_storage_max_age_days + +The maximum age of an update to to keep in C<update_storage_file>. Used by C<gather_updates>. + +=cut + +has update_storage_max_age_days => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{update_storage_max_age_days} } +); + + +=head1 DESCRIPTION + +=cut + +sub services { + my $self = shift; + return ($self->flytipping_service,); +} + +=head2 post_service_request + +A new Fly Tipping case is created using C<case_type>. +The town is looked up in C<town_to_officer> to determine which value to set for the 'eso-officer' field. +Files provided as uploads or URLs are added as attachments to the created case. + +=cut + +sub post_service_request { + my ($self, $service, $args) = @_; + my $attributes = $args->{attributes}; + + my $officer = $self->town_to_officer->{$attributes->{town}}; + if (!$officer) { + die "No officer found for town " . $attributes->{town}; + } + + my $google_street_view_url = sprintf( + "https://google.com/maps/@?api=1&map_action=pano&viewpoint=%s,%s", + $args->{lat}, $args->{long} + ); + + my $type_of_waste = $attributes->{type_of_waste}; + if (ref $type_of_waste eq 'ARRAY') { + $type_of_waste = join ",", @$type_of_waste; + } + + my $fly_tip_datetime = DateTime::Format::ISO8601->parse_datetime($attributes->{fly_tip_date_and_time}) if $attributes->{fly_tip_date_and_time}; + + my %payload = ( + 'coordinates' => $args->{lat} . ',' . $args->{long}, + 'ens-latitude' => $args->{lat}, + 'ens-longitude' => $args->{long}, + 'ens-google-street-view-url' => $google_street_view_url, + 'usrn' => $attributes->{usrn}, + 'ens-street' => $attributes->{street}, + 'sys-town' => $attributes->{town}, + 'eso-officer' => $officer, + 'ens-location-description' => $attributes->{title}, + 'ens-land-type' => $attributes->{land_type}, + 'ens-type-of-waste-fly-tipped' => $type_of_waste, + 'ens-description-of-fly-tipped-waste' => $attributes->{description}, + 'ens-fly-tip-witnessed' => $attributes->{fly_tip_witnessed}, + 'ens-description-of-alleged-offender' => $attributes->{description_of_alleged_offender}, + 'sys-first-name' => $args->{first_name}, + 'sys-last-name' => $args->{last_name}, + 'sys-email-address' => $args->{email}, + 'sys-telephone-number' => $args->{phone}, + 'fms-reference' => $attributes->{report_url}, + 'sys-channel' => $self->sys_channel + ); + $payload{'ens-fly-tip-date'} = $fly_tip_datetime->ymd if $fly_tip_datetime; + $payload{'ens-fly-tip-time'} = $fly_tip_datetime->strftime('%H:%M') if $fly_tip_datetime; + + my $case_reference = $self->jadu->create_case_and_get_reference($self->case_type, \%payload); + + foreach my $url (@{$args->{media_url}}) { + + # Capture file extension from URL. + $url =~ /(\.\w+)(?:\?.*)?$/; + my $file_ext = $1; + + my (undef, $tmp_file) = tempfile( SUFFIX => $file_ext ); + + my $code = getstore($url, $tmp_file); + + if ($code != 200) { + $self->logger->warn("Unable to download file from " . $url . " with code " . $code . "."); + next; + } + + try { + $self->jadu->attach_file_to_case($self->case_type, $case_reference, $tmp_file, $url); + } catch { + $self->logger->warn("Failed to attach file from " . $url . " to case."); + } + } + + foreach my $file (@{$args->{uploads}}) { + try { + $self->jadu->attach_file_to_case($self->case_type, $case_reference, $file->{tempname}, $file->{filename}); + } catch { + $self->logger->warn("Failed to attach uploaded file " . $file->{filename} . " to case."); + } + } + + return $self->new_request( + service_request_id => $case_reference + ); +} + +sub get_service_requests { + my ($self, $args) = @_; + die "uninmplemented"; +} + +sub get_service_request { + my ($self, $service_request_id, $args) = @_; + die "uninmplemented"; +} + +=head2 post_service_request_update + +This is not supported but is implemented as blank to avoid errors when called as part of the Multi integration. + +=cut + +sub post_service_request_update {} + +=head2 init_update_gathering_files + +Initialises C<case_status_tracking_file> and C<update_storage_file> ready for use by C<gather_updates>. + +=cut + +sub init_update_gathering_files { + my ($self, $start_from) = @_; + if (-e $self->case_status_tracking_file && ! -z $self->case_status_tracking_file) { + die $self->case_status_tracking_file . " already exists and is not empty. Aborting."; + } + if (-e $self->update_storage_file && ! -z $self->update_storage_file) { + die $self->update_storage_file . " already exists and is not empty. Aborting."; + } + + path($self->case_status_tracking_file)->spew_raw(encode_json({ + latest_update_seen_time => $start_from->epoch, + cases => {} + })); + + path($self->update_storage_file)->spew_raw('[]'); +} + +=head2 gather_updates + +Uses C<most_recently_updated_cases_filter>, C<case_status_to_fms_status> and state tracking in C<case_status_tracking_file> +to calculate updates to report, which are stored in C<update_storage_file> for consumption by C<get_service_request_updates>. + +Also applies C<case_status_to_fms_status_timed> for timed transitions e.g. "closed if in state A for 10 days." + +Limits size of tracking files via C<update_storage_max_age_days> and C<case_status_tracking_max_age_days>. + +=cut + +sub gather_updates { + my $self = shift; + + # Keeping this lock for whole duration to prevent interleaving runs. + my $case_status_tracking_fh = path($self->case_status_tracking_file)->openrw_raw({ locked => 1 }); + read $case_status_tracking_fh, my $tracked_case_statuses_raw, -s $case_status_tracking_fh; + my $tracked_case_statuses = decode_json($tracked_case_statuses_raw); + + my $update_storage_raw = path($self->update_storage_file)->slurp_raw; + my $updates = decode_json($update_storage_raw); + + my $case_cutoff = DateTime->now->subtract(days => $self->case_status_tracking_max_age_days)->epoch; + $self->_delete_old_cases($tracked_case_statuses, $case_cutoff); + + my $update_cutoff = DateTime->now->subtract(days => $self->update_storage_max_age_days)->epoch; + $self->_delete_old_updates($updates, $update_cutoff); + + push @$updates, @{$self->_apply_time_based_transitions($tracked_case_statuses)}; + push @$updates, @{$self->_fetch_and_apply_updated_cases_info($tracked_case_statuses, $case_cutoff)}; + + # Descending time. + @$updates = sort { $b->{time} <=> $a->{time} } @$updates; + + my $new_tracked_case_statuses_raw = encode_json($tracked_case_statuses); + my $new_updates_storage_raw = encode_json($updates); + + path($self->update_storage_file)->spew_raw($new_updates_storage_raw); + + seek $case_status_tracking_fh, 0, 0; + truncate $case_status_tracking_fh, 0; + print $case_status_tracking_fh $new_tracked_case_statuses_raw; + close $case_status_tracking_fh; +} + +sub _delete_old_cases { + my ($self, $tracked_case_statuses, $cutoff) = @_; + while (my ($case_reference, $state) = each %{$tracked_case_statuses->{cases}}) { + if ($state->{created_time} < $cutoff) { + delete $tracked_case_statuses->{cases}{$case_reference}; + } + } +} + +sub _delete_old_updates { + my ($self, $updates, $cutoff) = @_; + @$updates = grep { $_->{time} > $cutoff } @$updates; +} + +sub _apply_time_based_transitions { + my ($self, $tracked_case_statuses) = @_; + my @updates; + + while (my ($case_reference, $state) = each %{$tracked_case_statuses->{cases}}) { + my $timed_status_mapping = $self->case_status_to_fms_status_timed->{$state->{jadu_status}}; + if ($timed_status_mapping && $timed_status_mapping->{fms_status} ne $state->{fms_status}) { + my $cutoff = DateTime->now()->subtract(days => $timed_status_mapping->{days_to_wait})->epoch; + if ($state->{jadu_status_update_time} < $cutoff) { + push @updates, { + fms_status => $timed_status_mapping->{fms_status}, + jadu_status => $state->{jadu_status}, + case_reference => $case_reference, + time => $state->{jadu_status_update_time} + 60 * 60 * 24 * $timed_status_mapping->{days_to_wait}, + }; + $state->{fms_status} = $timed_status_mapping->{fms_status}; + } + } + } + return \@updates; +} + +sub _fetch_and_apply_updated_cases_info { + my ($self, $tracked_case_statuses, $cutoff) = @_; + + my @case_summaries_in_descending_time; + + my $new_latest_update_seen_time = $tracked_case_statuses->{latest_update_seen_time}; + my $all_new_updates_seen = 0; + my $page_number = 1; + my $page_items = 1; + + # Case summaries are given in descending time order (i.e most recent first). + # Iterate through these and collect the ones we need to process. + while (!$all_new_updates_seen && $page_items > 0) { + + my $case_summaries = $self->jadu->get_case_summaries_by_filter($self->case_type, $self->most_recently_updated_cases_filter, $page_number); + $page_items = $case_summaries->{num_items}; + + foreach my $case_summary (@{ $case_summaries->{items} }) { + my $update_time = DateTime::Format::ISO8601->parse_datetime($case_summary->{updated_at})->epoch; + if ($update_time < $tracked_case_statuses->{latest_update_seen_time}) { + $all_new_updates_seen = 1; + last + } + if ($update_time > $new_latest_update_seen_time) { + $new_latest_update_seen_time = $update_time; + } + push @case_summaries_in_descending_time, $case_summary; + } + $page_number++; + } + $tracked_case_statuses->{latest_update_seen_time} = $new_latest_update_seen_time; + + # Process the collected case summaries in ascending time order (i.e. chronologically). + my @updates; + while (my $case_summary = pop @case_summaries_in_descending_time) { + my $update_time = DateTime::Format::ISO8601->parse_datetime($case_summary->{updated_at})->epoch; + my $case_created_time = DateTime::Format::ISO8601->parse_datetime($case_summary->{created_at})->epoch; + next if $case_created_time < $cutoff; + + my $case_reference = $case_summary->{reference}; + my $jadu_status = $case_summary->{status}->{title}; + my $fms_status = $self->case_status_to_fms_status->{$jadu_status}; + my $existing_state = $tracked_case_statuses->{cases}->{$case_reference}; + + next if $existing_state && $jadu_status eq $existing_state->{jadu_status}; + + if ($fms_status && ($existing_state && $fms_status ne $existing_state->{fms_status} || !$existing_state)) { + push @updates, { + fms_status => $fms_status, + jadu_status => $jadu_status, + case_reference => $case_reference, + time => $update_time + }; + } + $tracked_case_statuses->{cases}->{$case_reference} = { + fms_status => $fms_status ? $fms_status : ( $existing_state ? $existing_state->{fms_status} : '' ), + jadu_status => $jadu_status, + jadu_status_update_time => $update_time, + created_time => $case_created_time + }; + } + + return \@updates; +} + +=head2 get_service_request_updates + +Reads updates from C<update_storage_file>. + +=cut + +sub get_service_request_updates { + my ($self, $args) = @_; + my $w3c = DateTime::Format::W3CDTF->new; + my $start_time = $w3c->parse_datetime($args->{start_date})->epoch; + my $end_time = $w3c->parse_datetime($args->{end_date})->epoch; + + my $update_storage_raw = path($self->update_storage_file)->slurp_raw; + my $updates = decode_json($update_storage_raw); + my @updates_to_send; + + foreach my $update (@$updates) { + next if $update->{time} > $end_time; + # Assuming in descending time order. + last if $update->{time} < $start_time; + my %args = ( + status => $update->{fms_status}, + external_status_code => $update->{jadu_status}, + update_id => $update->{case_reference} . '_' . $update->{time}, + service_request_id => $update->{case_reference}, + description => "", + updated_datetime => DateTime->from_epoch(epoch => $update->{time}), + ); + push @updates_to_send, Open311::Endpoint::Service::Request::Update::mySociety->new( %args ); + } + return @updates_to_send; +} + +1; diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm new file mode 100644 index 000000000..b4a2f1393 --- /dev/null +++ b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm @@ -0,0 +1,125 @@ +package Open311::Endpoint::Service::UKCouncil::CentralBedfordshireFlytipping; +use Moo; +extends 'Open311::Endpoint::Service'; + +use Open311::Endpoint::Service::Attribute; + +sub _build_attributes { + my $self = shift; + + my @attributes = ( + @{ $self->SUPER::_build_attributes() }, + + Open311::Endpoint::Service::Attribute->new( + code => "title", + description => "Title", + datatype => "string", + required => 1, + automated => 'server_set', + ), + Open311::Endpoint::Service::Attribute->new( + code => "description", + description => "Description", + datatype => "text", + required => 1, + automated => 'server_set', + ), + Open311::Endpoint::Service::Attribute->new( + code => "report_url", + required => 1, + datatype => "string", + description => "Report URL", + automated => 'server_set' + ), + Open311::Endpoint::Service::Attribute->new( + code => "usrn", + description => "USRN", + datatype => "string", + required => 1, + automated => 'server_set' + ), + Open311::Endpoint::Service::Attribute->new( + code => "street", + description => "Street", + datatype => "string", + required => 1, + automated => 'server_set' + ), + Open311::Endpoint::Service::Attribute->new( + code => "town", + description => "Town", + datatype => "string", + required => 1, + automated => 'server_set' + ), + Open311::Endpoint::Service::Attribute->new( + code => "land_type", + variable => 1, + required => 1, + datatype => "singlevaluelist", + description => "The flytip is located on:", + "values" => { + "Roadside / verge" => "Roadside / verge", + "Footpath" => "Footpath", + "Private land" => "Private land", + "Public land" => "Public land", + } + ), + Open311::Endpoint::Service::Attribute->new( + code => "type_of_waste", + variable => 1, + required => 1, + datatype => "multivaluelist", + description => "What type of waste is it?", + "values" => { + "Asbestos" => "Asbestos", + "Black bags" => "Black bags", + "Building materials" => "Building materials", + "Chemical / oil drums" => "Chemical / oil drums", + "Construction waste" => "Construction waste", + "Electricals" => "Electricals", + "Fly posting" => "Fly posting", + "Furniture" => "Furniture", + "Green / garden waste" => "Green / garden waste", + "Household waste / black bin bags" => "Household waste / black bin bags", + "Mattress or bed base" => "Mattress or bed base", + "Trolleys" => "Trolleys", + "Tyres" => "Tyres", + "Vehicle parts" => "Vehicle parts", + "White goods - fridge, freezer, washing matchine etc" => "White goods - fridge, freezer, washing matchine etc", + "Other" => "Other", + } + ), + Open311::Endpoint::Service::Attribute->new( + code => "fly_tip_witnessed", + variable => 1, + required => 1, + datatype => "singlevaluelist", + description => "Did you observe this taking place?", + "values" => { + "Yes" => "Yes", + "No" => "No", + } + ), + Open311::Endpoint::Service::Attribute->new( + code => "fly_tip_date_and_time", + variable => 1, + # Only required when fly tip witnessed, expecting client to enforce this. + required => 0, + datatype => "datetime", + description => "When did this take place?" + ), + Open311::Endpoint::Service::Attribute->new( + code => "description_of_alleged_offender", + variable => 1, + # Only required when fly tip witnessed, expecting client to enforce this. + required => 0, + datatype => "text", + description => "Please provide any futher information which may help identify the alleged offender" + ), + ); + + return \@attributes; +} + +1; diff --git a/t/open311/endpoint/centralbedfordshire_jadu.t b/t/open311/endpoint/centralbedfordshire_jadu.t new file mode 100644 index 000000000..420af9f1e --- /dev/null +++ b/t/open311/endpoint/centralbedfordshire_jadu.t @@ -0,0 +1,280 @@ +use strict; +use warnings; + +use Test::MockFile qw< nostrict >; +use Test::MockModule; +use Test::MockTime qw( :all ); + +use DateTime; +use File::Temp qw(tempfile); +use JSON::MaybeXS; +use Test::More; +use Moo; +use Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu; +use Path::Tiny; + +BEGIN { $ENV{TEST_MODE} = 1; } + +my (undef, $status_tracking_file) = tempfile(EXLOCK => 0); +my (undef, $update_storage_file) = tempfile(EXLOCK => 0); + +my $config_string = ' + sys_channel: "test_sys_channel" + case_status_to_fms_status: + "Action Scheduled 1": "action_scheduled" + "Action Scheduled 2": "action_scheduled" + "Investigating": "investigating" + case_status_to_fms_status_timed: + "Closed": + fms_status: "closed" + days_to_wait: 1 + case_status_tracking_file: "%s" + case_status_tracking_max_age_days: 365 + update_storage_file: "%s" + update_storage_max_age_days: 365 + town_to_officer: + "Chicksands": "area_1"'; + +my $config = Test::MockFile->file( "/config.yml", sprintf($config_string, $status_tracking_file, $update_storage_file)); + +my $integration = Test::MockModule->new('Integrations::Jadu'); + +my $centralbedfordshire_jadu = Test::MockModule->new('Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu'); +$centralbedfordshire_jadu->mock('_build_config_file', sub { '/config.yml' }); + +my $endpoint = Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu->new; + +subtest "POST service request" => sub { + my @sent_payloads; + $integration->mock('create_case_and_get_reference', sub { + my ($self, undef, $payload) = @_; + push @sent_payloads, $payload; + return "test_case_reference"; + }); + + my $res = $endpoint->run_test_request( + POST => '/requests.json', + jurisdiction_id => 'centralbedfordshire_jadu', + api_key => 'test', + service_code => 'fly-tipping', + first_name => 'John', + last_name => 'Smith', + email => 'john.smith@example.com', + lat => 52.035536, + long => -0.360673, + phone => '07700900077', + 'attribute[title]' => 'Near the junction.', + 'attribute[description]' => 'Black bags of asbestos.', + 'attribute[easting]' => 12548, + 'attribute[northing]' => 238727, + 'attribute[fixmystreet_id]' => 1, + 'attribute[report_url]' => 'http://fixmystreet.com/reports/1', + 'attribute[land_type]' => 'Footpath', + 'attribute[type_of_waste]' => 'Asbestos', + 'attribute[type_of_waste]' => 'Black bags', + 'attribute[fly_tip_witnessed]' => 'Yes', + 'attribute[fly_tip_date_and_time]' => '2023-07-03T14:30:36Z', + 'attribute[description_of_alleged_offender]' => 'Stealthy.', + 'attribute[usrn]' => '25202550', + 'attribute[street]' => "Monk's Walk", + 'attribute[town]' => "Chicksands", + ); + + ok $res->is_success, 'valid request' + or diag $res->content; + + my $sent_payload = pop @sent_payloads; + + my $expected_payload = { + 'sys-channel' => 'test_sys_channel', + 'ens-latitude' => '52.035536', + 'ens-longitude' => '-0.360673', + 'coordinates' => '52.035536,-0.360673', + 'usrn' => '25202550', + 'sys-town' => 'Chicksands', + 'eso-officer' => 'area_1', + 'ens-street' => "Monk's Walk", + 'sys-first-name' => 'John', + 'sys-last-name' => 'Smith', + 'sys-email-address' => 'john.smith@example.com', + 'sys-telephone-number' => '07700900077', + 'ens-google-street-view-url' => 'https://google.com/maps/@?api=1&map_action=pano&viewpoint=52.035536,-0.360673', + 'ens-location-description' => 'Near the junction.', + 'ens-land-type' => 'Footpath', + 'ens-type-of-waste-fly-tipped' => 'Asbestos,Black bags', + 'ens-description-of-fly-tipped-waste' => 'Black bags of asbestos.', + 'ens-fly-tip-date' => '2023-07-03', + 'ens-fly-tip-time' => '14:30', + 'ens-description-of-alleged-offender' => 'Stealthy.', + 'fms-reference' => 'http://fixmystreet.com/reports/1', + 'ens-fly-tip-witnessed' => 'Yes', + }; + is_deeply $sent_payload, $expected_payload; + + is_deeply decode_json($res->content), + [ { + "service_request_id" => "test_case_reference" + } ], 'correct response returned'; +}; + +subtest "GET service request updates" => sub { + set_fixed_time('2023-01-31T00:00:00Z'); + + my $start_from = DateTime->now()->subtract(days => 1); + $endpoint->init_update_gathering_files($start_from); + + $integration->mock('get_case_summaries_by_filter', sub { + my ($self, undef, undef, $page_number) = @_; + if ($page_number == 1) { + return { + num_items => 3, + items => [ + { # Should only result in a closed update after a day. + reference => "case_1", + updated_at => "2023-01-31T00:00:00+0000", + created_at => "2023-01-30T16:00:00+0000", + status => { + title => 'Closed' + }, + }, + { # Shouldn't result in an update as already in action_scheduled. + reference => "case_1", + updated_at => "2023-01-30T18:00:00+0000", + created_at => "2023-01-30T16:00:00+0000", + status => { + title => 'Action Scheduled 2' + }, + }, + { # Should result in an action_scheduled update. + reference => "case_1", + updated_at => "2023-01-30T17:00:00+0000", + created_at => "2023-01-30T16:00:00+0000", + status => { + title => 'Action Scheduled 1' + }, + }, + { # Should result in an investigating update. + reference => "case_1", + updated_at => "2023-01-30T16:00:00+0000", + created_at => "2023-01-30T16:00:00+0000", + status => { + title => 'Investigating' + }, + }, + { # Shouldn't result in an update - unmapped case status. + reference => "case_unmapped_status", + updated_at => "2023-01-30T15:00:00+0000", + created_at => "2023-01-30T00:00:00+0000", + status => { + title => 'Unmapped' + }, + }, + { # Shouldn't result in an update - case too old. + reference => "case_too_old", + updated_at => "2023-01-30T14:00:00+0000", + created_at => "2022-01-30T00:00:00+0000", + status => { + title => 'Open' + }, + } + ] + }; + } elsif ($page_number == 2) { + return { + num_items => 1, + items => [ + { # Update too old, should stop querying. + reference => "case_update_too_old", + updated_at => "2023-01-29T23:59:59+0000", + status => { + title => 'Closed' + }, + }, + ] + } + } + die "unexpected page number: " . $page_number; + }); + + $endpoint->gather_updates(); + + my $start_date = '2023-01-01T00:00:00Z'; + my $end_date = '2023-02-28T00:00:00Z'; + + my $res = $endpoint->run_test_request( + GET => sprintf('/servicerequestupdates.json?start_date=%s&end_date=%s', $start_date, $end_date), + jurisdiction_id => 'centralbedfordshire_jadu', + api_key => 'test', + service_code => 'fly-tipping', + ); + + ok $res->is_success, 'valid request' + or diag $res->content; + + my $expected_updates = [ + { + 'service_request_id' => 'case_1', + 'description' => '', + 'external_status_code' => 'Action Scheduled 1', + 'status' => 'action_scheduled', + 'media_url' => '', + 'updated_datetime' => '2023-01-30T17:00:00Z', + 'update_id' => 'case_1_1675098000' + }, + { + 'description' => '', + 'service_request_id' => 'case_1', + 'external_status_code' => 'Investigating', + 'status' => 'investigating', + 'update_id' => 'case_1_1675094400', + 'updated_datetime' => '2023-01-30T16:00:00Z', + 'media_url' => '' + } + ]; + + is_deeply decode_json($res->content), $expected_updates, "correct updates returned"; + + # Two days later... + set_fixed_time('2023-02-02T00:00:00Z'); + + $endpoint->gather_updates(); + + $integration->mock('get_case_summaries_by_filter', sub { + my ($self, undef, undef, $page_number) = @_; + if ($page_number == 1) { + return { + num_items => 0, + items => [] + }; + } + die "unexpected page number: " . $page_number; + }); + + my $next_day_expected_updates = [ + { + 'service_request_id' => 'case_1', + 'description' => '', + 'external_status_code' => 'Closed', + 'status' => 'closed', + 'media_url' => '', + 'updated_datetime' => '2023-02-01T00:00:00Z', + 'update_id' => 'case_1_1675209600' + }, + ]; + push @$next_day_expected_updates, @$expected_updates; + + + $res = $endpoint->run_test_request( + GET => sprintf('/servicerequestupdates.json?start_date=%s&end_date=%s', $start_date, $end_date), + jurisdiction_id => 'centralbedfordshire_jadu', + api_key => 'test', + service_code => 'fly-tipping', + ); + + ok $res->is_success, 'valid request' + or diag $res->content; + + is_deeply decode_json($res->content), $next_day_expected_updates, "correct updates returned"; +}; + +done_testing; diff --git a/t/open311/endpoint/centralbedfordshire_symology.t b/t/open311/endpoint/centralbedfordshire_symology.t index f2980b81e..be6f71bde 100644 --- a/t/open311/endpoint/centralbedfordshire_symology.t +++ b/t/open311/endpoint/centralbedfordshire_symology.t @@ -197,6 +197,8 @@ subtest "GET services" => sub { my $content = decode_json($res->content); my $services = [ sort { $a->{service_code} cmp $b->{service_code} } @$content ]; + # Filter out services belonging to the Jadu integration. + @$services = grep { $_->{service_code} !~ m/Jadu/ } @$services; is_deeply $services, [ { diff --git a/t/open311/endpoint/uk.t b/t/open311/endpoint/uk.t index c10316452..2aa76a9a2 100644 --- a/t/open311/endpoint/uk.t +++ b/t/open311/endpoint/uk.t @@ -59,6 +59,7 @@ test_multi(1, 'Open311::Endpoint::Integration::UK::Hackney', test_multi(1, 'Open311::Endpoint::Integration::UK::CentralBedfordshire', 'Open311::Endpoint::Integration::UK::CentralBedfordshire::Symology' => 'centralbedfordshire_symology', + 'Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu' => 'centralbedfordshire_jadu', ); done_testing; From 2ee6ad192baddca1422f10e918cc2a4ebc8a3cca Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Thu, 13 Jul 2023 13:23:39 +0100 Subject: [PATCH 05/11] [Central Beds] Use Singlepoint to find the nearest address. --- ...uncil-centralbedfordshire_jadu.yml-example | 12 ++- perllib/Geocode/SinglePoint.pm | 79 +++++++++++++++++++ perllib/Integrations/Jadu.pm | 8 +- .../UK/CentralBedfordshire/Jadu.pm | 66 ++++++++++++++-- .../CentralBedfordshireFlytipping.pm | 23 +----- ...earch_by_easting_and_northing_response.xml | 39 +++++++++ t/geocode/singlepoint.t | 39 +++++++++ t/open311/endpoint/centralbedfordshire_jadu.t | 16 +++- 8 files changed, 242 insertions(+), 40 deletions(-) create mode 100644 perllib/Geocode/SinglePoint.pm create mode 100644 t/geocode/files/singlepoint/spatial_radial_search_by_easting_and_northing_response.xml create mode 100644 t/geocode/singlepoint.t diff --git a/conf/council-centralbedfordshire_jadu.yml-example b/conf/council-centralbedfordshire_jadu.yml-example index f804eb0a4..b0df3921b 100644 --- a/conf/council-centralbedfordshire_jadu.yml-example +++ b/conf/council-centralbedfordshire_jadu.yml-example @@ -1,7 +1,11 @@ -api_base_url: '' -api_key: '' -username: '' -password: '' +jadu_api_base_url: '' +jadu_api_key: '' +jadu_username: '' +jadu_password: '' + +singlepoint_api_base_url: '' +singlepoint_api_key: '' +reverse_geocode_radius_meters: 100 case_type: '' sys_channel: '' diff --git a/perllib/Geocode/SinglePoint.pm b/perllib/Geocode/SinglePoint.pm new file mode 100644 index 000000000..8043b7631 --- /dev/null +++ b/perllib/Geocode/SinglePoint.pm @@ -0,0 +1,79 @@ +package Geocode::SinglePoint; + +use v5.14; +use warnings; + +use Data::Dumper; +use LWP::UserAgent; +use Moo; +use XML::Simple qw(:strict); + +with 'Role::Config'; +with 'Role::Logger'; + +has base_url => ( + is => 'lazy', + default => sub { $_[0]->config->{singlepoint_api_base_url} } +); + +has api_key => ( + is => 'lazy', + default => sub { $_[0]->config->{singlepoint_api_key} } +); + +has ua => ( + is => 'lazy', + default => sub { LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter") } +); + +sub _fail { + my ($self, $message, $url, $response) = @_; + my $log = sprintf( + "%s + Requested: %s + Got: %s", + $message, $url, $response->content + ); + $self->logger->error($log); + die $message; +} + +sub _get_field_value_for_tag { + my ($self, $dom, $tag) = @_; + return $self->xml_path_context->findvalue('./x:FieldItems/x:FieldInfo/x:Value[../x:Tag="'. $tag . '"]', $dom); +} + +sub get_nearest_addresses { + my ($self, $easting, $northing, $radius_meters, $address_field_tags) = @_; + my $url = sprintf( + "%sSpatialRadialSearchByEastingNorthing?apiKey=%s&adapterName=LLPG&easting=%s&northing=%s&unit=Meter&distance=%s", + $self->base_url, + $self->api_key, + $easting, + $northing, + $radius_meters, + ); + my $response = $self->ua->get($url); + if (!$response->is_success) { + $self->_fail("Request failed.", $url, $response); + } + my $x = XML::Simple->new(ForceArray => ["SearchResultItem", "FieldInfo"], NoAttr => 1, SuppressEmpty => "", KeyAttr => ["Tag"]); + my $xml = $x->XMLin($response->content); + my $results = $xml->{Results}{Items}{SearchResultItem}; + + my @addresses; + # Results are already ordered nearest-first. + foreach my $result (@$results) { + + my %address; + foreach my $address_field_tag (@$address_field_tags) { + my $field = $result->{FieldItems}{FieldInfo}{$address_field_tag}; + $address{$address_field_tag} = $field->{Value} if $field; + } + push @addresses, \%address if %address; + } + + return \@addresses; +} + +1; diff --git a/perllib/Integrations/Jadu.pm b/perllib/Integrations/Jadu.pm index f56700b05..6194a54e6 100644 --- a/perllib/Integrations/Jadu.pm +++ b/perllib/Integrations/Jadu.pm @@ -16,22 +16,22 @@ with 'Role::Memcached'; has base_url => ( is => 'lazy', - default => sub { $_[0]->config->{api_base_url} } + default => sub { $_[0]->config->{jadu_api_base_url} } ); has api_key => ( is => 'lazy', - default => sub { $_[0]->config->{api_key} } + default => sub { $_[0]->config->{jadu_api_key} } ); has username => ( is => 'lazy', - default => sub { $_[0]->config->{username} } + default => sub { $_[0]->config->{jadu_username} } ); has password => ( is => 'lazy', - default => sub { $_[0]->config->{password} } + default => sub { $_[0]->config->{jadu_password} } ); has ua => ( diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm index ce8865901..319ecc8ea 100644 --- a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm +++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm @@ -28,6 +28,7 @@ use DateTime::Format::ISO8601; use DateTime::Format::W3CDTF; use Fcntl qw(:flock); use File::Temp qw(tempfile); +use Geocode::SinglePoint; use Integrations::Jadu; use JSON::MaybeXS qw(decode_json encode_json); use LWP::Simple; @@ -46,6 +47,11 @@ has jurisdiction_id => ( default => 'centralbedfordshire_jadu', ); +has singlepoint => ( + is => 'lazy', + default => sub { Geocode::SinglePoint->new(config_filename => $_[0]->jurisdiction_id) } +); + has jadu => ( is => 'lazy', default => sub { Integrations::Jadu->new(config_filename => $_[0]->jurisdiction_id) } @@ -68,6 +74,17 @@ sub get_integration { return $_[0]->jadu; } +=head2 reverse_geocode_radius_meters + +This is the radius in meters of the area around the report location to search for addresses. + +=cut + +has reverse_geocode_radius_meters => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{reverse_geocode_radius_meters} } +); + =head2 sys_channel This is the value to set for the 'sys-channel' field when creating a new Fly Tipping case. @@ -94,6 +111,7 @@ has case_type => ( This is a mapping from any town associated with an address in Central Bedfordshire to the value that should be set in the 'eso-officer' field when creating a new Fly Tipping case. +Towns must be specified in lowercase. =cut @@ -204,9 +222,45 @@ sub post_service_request { my ($self, $service, $args) = @_; my $attributes = $args->{attributes}; - my $officer = $self->town_to_officer->{$attributes->{town}}; - if (!$officer) { - die "No officer found for town " . $attributes->{town}; + my $addresses = $self->singlepoint->get_nearest_addresses( + $attributes->{easting}, + $attributes->{northing}, + $self->reverse_geocode_radius_meters, + ['STREET', 'TOWN', 'USRN'], + ); + + if ($addresses == 0) { + die sprintf( + "No addresses found within %dm of easting: %d northing: %d", + $self->reverse_geocode_radius_meters, + $attributes->{easting}, + $attributes->{northing}, + ); + } + + my $officer; + my $nearest_valid_address; + foreach my $address (@$addresses) { + my $usrn = $address->{USRN}; + my $street = $address->{STREET}; + my $town = $address->{TOWN}; + + unless ($usrn && $street && $town) { + $self->logger->warn("Skipping address missing one or more of USRN, STREET and TOWN"); + next; + } + + $officer = $self->town_to_officer->{lc $town}; + if (!$officer) { + $self->logger->warn("Skipping address with unmapped town: " . $town); + next; + } + $nearest_valid_address = $address; + last; + } + + if (!$nearest_valid_address) { + die "None of the addresses found were valid."; } my $google_street_view_url = sprintf( @@ -226,9 +280,9 @@ sub post_service_request { 'ens-latitude' => $args->{lat}, 'ens-longitude' => $args->{long}, 'ens-google-street-view-url' => $google_street_view_url, - 'usrn' => $attributes->{usrn}, - 'ens-street' => $attributes->{street}, - 'sys-town' => $attributes->{town}, + 'usrn' => $nearest_valid_address->{USRN}, + 'ens-street' => $nearest_valid_address->{STREET}, + 'sys-town' => $nearest_valid_address->{TOWN}, 'eso-officer' => $officer, 'ens-location-description' => $attributes->{title}, 'ens-land-type' => $attributes->{land_type}, diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm index b4a2f1393..4a1c5be1f 100644 --- a/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm +++ b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm @@ -1,6 +1,6 @@ package Open311::Endpoint::Service::UKCouncil::CentralBedfordshireFlytipping; use Moo; -extends 'Open311::Endpoint::Service'; +extends 'Open311::Endpoint::Service::UKCouncil'; use Open311::Endpoint::Service::Attribute; @@ -31,27 +31,6 @@ sub _build_attributes { description => "Report URL", automated => 'server_set' ), - Open311::Endpoint::Service::Attribute->new( - code => "usrn", - description => "USRN", - datatype => "string", - required => 1, - automated => 'server_set' - ), - Open311::Endpoint::Service::Attribute->new( - code => "street", - description => "Street", - datatype => "string", - required => 1, - automated => 'server_set' - ), - Open311::Endpoint::Service::Attribute->new( - code => "town", - description => "Town", - datatype => "string", - required => 1, - automated => 'server_set' - ), Open311::Endpoint::Service::Attribute->new( code => "land_type", variable => 1, diff --git a/t/geocode/files/singlepoint/spatial_radial_search_by_easting_and_northing_response.xml b/t/geocode/files/singlepoint/spatial_radial_search_by_easting_and_northing_response.xml new file mode 100644 index 000000000..5f2bed5d4 --- /dev/null +++ b/t/geocode/files/singlepoint/spatial_radial_search_by_easting_and_northing_response.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="utf-8"?> +<SearchResultData xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.aligned-assets.co.uk/SinglePoint/Search/WebServices/V2/SearchService"> + <Results> + <Items> + <SearchResultItem> + <FieldItems> + <FieldInfo> + <Tag>USRN</Tag> + <Value>25202550</Value> + </FieldInfo> + <FieldInfo> + <Tag>STREET</Tag> + <Value>Monks Walk</Value> + </FieldInfo> + <FieldInfo> + <Tag>TOWN</Tag> + <Value>Chicksands</Value> + </FieldInfo> + </FieldItems> + </SearchResultItem> + <SearchResultItem> + <FieldItems> + <FieldInfo> + <Tag>USRN</Tag> + <Value>25201736</Value> + </FieldInfo> + <FieldInfo> + <Tag>STREET</Tag> + <Value>Northwood End Road</Value> + </FieldInfo> + <FieldInfo> + <Tag>TOWN</Tag> + <Value>Haynes</Value> + </FieldInfo> + </FieldItems> + </SearchResultItem> + </Items> + </Results> +</SearchResultData> diff --git a/t/geocode/singlepoint.t b/t/geocode/singlepoint.t new file mode 100644 index 000000000..ecdb48d94 --- /dev/null +++ b/t/geocode/singlepoint.t @@ -0,0 +1,39 @@ +use strict; +use warnings; + +use Test::MockModule; +use Test::More; + +use Geocode::SinglePoint; +use LWP::UserAgent; +use Moo; +use Path::Tiny; + +BEGIN { $ENV{TEST_MODE} = 1; } + +subtest "get_nearest_addresses" => sub { + my $ua = Test::MockModule->new('LWP::UserAgent'); + $ua->mock('get', sub { + my $resp = HTTP::Response->new(200); + $resp->content(path(__FILE__)->sibling("files/singlepoint/spatial_radial_search_by_easting_and_northing_response.xml")->slurp); + return $resp; + }); + my $singlepoint = Geocode::SinglePoint->new(); + # Call with arbitrary easting, northing and radius. + my $addresses = $singlepoint->get_nearest_addresses(0, 0, 0, ["STREET", "USRN", "TOWN"]); + + is_deeply $addresses, [ + { + 'TOWN' => 'Chicksands', + 'STREET' => 'Monks Walk', + 'USRN' => '25202550' + }, + { + 'TOWN' => 'Haynes', + 'STREET' => 'Northwood End Road', + 'USRN' => '25201736' + }, + ]; +}; + +done_testing; diff --git a/t/open311/endpoint/centralbedfordshire_jadu.t b/t/open311/endpoint/centralbedfordshire_jadu.t index 420af9f1e..b23e21d54 100644 --- a/t/open311/endpoint/centralbedfordshire_jadu.t +++ b/t/open311/endpoint/centralbedfordshire_jadu.t @@ -33,11 +33,12 @@ my $config_string = ' update_storage_file: "%s" update_storage_max_age_days: 365 town_to_officer: - "Chicksands": "area_1"'; + "chicksands": "area_1"'; my $config = Test::MockFile->file( "/config.yml", sprintf($config_string, $status_tracking_file, $update_storage_file)); my $integration = Test::MockModule->new('Integrations::Jadu'); +my $geocode = Test::MockModule->new('Geocode::SinglePoint'); my $centralbedfordshire_jadu = Test::MockModule->new('Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu'); $centralbedfordshire_jadu->mock('_build_config_file', sub { '/config.yml' }); @@ -52,6 +53,16 @@ subtest "POST service request" => sub { return "test_case_reference"; }); + $geocode->mock('get_nearest_addresses', sub { + return [ + { + USRN => 25202550, + STREET => "Monk's Walk", + TOWN => "Chicksands", + } + ] + }); + my $res = $endpoint->run_test_request( POST => '/requests.json', jurisdiction_id => 'centralbedfordshire_jadu', @@ -75,9 +86,6 @@ subtest "POST service request" => sub { 'attribute[fly_tip_witnessed]' => 'Yes', 'attribute[fly_tip_date_and_time]' => '2023-07-03T14:30:36Z', 'attribute[description_of_alleged_offender]' => 'Stealthy.', - 'attribute[usrn]' => '25202550', - 'attribute[street]' => "Monk's Walk", - 'attribute[town]' => "Chicksands", ); ok $res->is_success, 'valid request' From a16f4599c6176d2a6e71164d569c8384ed7208b4 Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Tue, 18 Jul 2023 14:41:50 +0100 Subject: [PATCH 06/11] [Central Beds] Rename Fly Tipping (Jadu) to just Fly Tipping. --- .../Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm index 319ecc8ea..a5c4c8e42 100644 --- a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm +++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm @@ -61,8 +61,7 @@ has flytipping_service => ( is => 'lazy', default => sub { Open311::Endpoint::Service::UKCouncil::CentralBedfordshireFlytipping->new( - # TODO: Will be renamed to just "Fly Tipping" when ready to replace existing category. - service_name => "Fly Tipping (Jadu)", + service_name => "Fly Tipping", group => "Flytipping, Bins and Graffiti", service_code => "fly-tipping", description => "Fly Tipping", From c77a70bf0b28f8e770451c7a533052548f634681 Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Tue, 1 Aug 2023 16:31:22 +0100 Subject: [PATCH 07/11] [Central Beds] Vary Jadu case sys_channel depending on staff vs self-report. --- ...uncil-centralbedfordshire_jadu.yml-example | 3 ++- .../UK/CentralBedfordshire/Jadu.pm | 23 +++++++++++++++---- .../CentralBedfordshireFlytipping.pm | 11 +++++++++ t/open311/endpoint/centralbedfordshire_jadu.t | 6 +++-- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/conf/council-centralbedfordshire_jadu.yml-example b/conf/council-centralbedfordshire_jadu.yml-example index b0df3921b..7e5bb4d84 100644 --- a/conf/council-centralbedfordshire_jadu.yml-example +++ b/conf/council-centralbedfordshire_jadu.yml-example @@ -8,7 +8,8 @@ singlepoint_api_key: '' reverse_geocode_radius_meters: 100 case_type: '' -sys_channel: '' +sys_channel_self_reported: '' +sys_channel_reported_by_staff: '' most_recently_updated_cases_filter: '' case_status_to_fms_status: diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm index a5c4c8e42..bba5e4b54 100644 --- a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm +++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm @@ -84,15 +84,26 @@ has reverse_geocode_radius_meters => ( default => sub { $_[0]->endpoint_config->{reverse_geocode_radius_meters} } ); -=head2 sys_channel +=head2 sys_channel_self_reported -This is the value to set for the 'sys-channel' field when creating a new Fly Tipping case. +This is the value to set for the 'sys-channel' field when creating a new self-reported Fly Tipping case. =cut -has sys_channel => ( +has sys_channel_self_reported => ( is => 'lazy', - default => sub { $_[0]->endpoint_config->{sys_channel} } + default => sub { $_[0]->endpoint_config->{sys_channel_self_reported} } +); + +=head2 sys_channel_reported_by_staff + +This is the value to set for the 'sys-channel' field when creating a new Fly Tipping case reported by staff. + +=cut + +has sys_channel_reported_by_staff => ( + is => 'lazy', + default => sub { $_[0]->endpoint_config->{sys_channel_reported_by_staff} } ); =head2 case_type @@ -274,6 +285,8 @@ sub post_service_request { my $fly_tip_datetime = DateTime::Format::ISO8601->parse_datetime($attributes->{fly_tip_date_and_time}) if $attributes->{fly_tip_date_and_time}; + my $sys_channel = $attributes->{reported_by_staff} eq 'Yes' ? $self->sys_channel_reported_by_staff : $self->sys_channel_self_reported; + my %payload = ( 'coordinates' => $args->{lat} . ',' . $args->{long}, 'ens-latitude' => $args->{lat}, @@ -294,7 +307,7 @@ sub post_service_request { 'sys-email-address' => $args->{email}, 'sys-telephone-number' => $args->{phone}, 'fms-reference' => $attributes->{report_url}, - 'sys-channel' => $self->sys_channel + 'sys-channel' => $sys_channel, ); $payload{'ens-fly-tip-date'} = $fly_tip_datetime->ymd if $fly_tip_datetime; $payload{'ens-fly-tip-time'} = $fly_tip_datetime->strftime('%H:%M') if $fly_tip_datetime; diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm index 4a1c5be1f..c36e42430 100644 --- a/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm +++ b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm @@ -31,6 +31,17 @@ sub _build_attributes { description => "Report URL", automated => 'server_set' ), + Open311::Endpoint::Service::Attribute->new( + code => "reported_by_staff", + required => 1, + datatype => "singlevaluelist", + description => "Reported by staff", + automated => 'server_set', + "values" => { + "Yes" => "Yes", + "No" => "No", + } + ), Open311::Endpoint::Service::Attribute->new( code => "land_type", variable => 1, diff --git a/t/open311/endpoint/centralbedfordshire_jadu.t b/t/open311/endpoint/centralbedfordshire_jadu.t index b23e21d54..b897c1141 100644 --- a/t/open311/endpoint/centralbedfordshire_jadu.t +++ b/t/open311/endpoint/centralbedfordshire_jadu.t @@ -19,7 +19,8 @@ my (undef, $status_tracking_file) = tempfile(EXLOCK => 0); my (undef, $update_storage_file) = tempfile(EXLOCK => 0); my $config_string = ' - sys_channel: "test_sys_channel" + sys_channel_reported_by_staff: "sys_channel_reported_by_staff" + sys_channel_self_reported: "sys_channel_self_reported" case_status_to_fms_status: "Action Scheduled 1": "action_scheduled" "Action Scheduled 2": "action_scheduled" @@ -80,6 +81,7 @@ subtest "POST service request" => sub { 'attribute[northing]' => 238727, 'attribute[fixmystreet_id]' => 1, 'attribute[report_url]' => 'http://fixmystreet.com/reports/1', + 'attribute[reported_by_staff]' => 'Yes', 'attribute[land_type]' => 'Footpath', 'attribute[type_of_waste]' => 'Asbestos', 'attribute[type_of_waste]' => 'Black bags', @@ -94,7 +96,7 @@ subtest "POST service request" => sub { my $sent_payload = pop @sent_payloads; my $expected_payload = { - 'sys-channel' => 'test_sys_channel', + 'sys-channel' => 'sys_channel_reported_by_staff', 'ens-latitude' => '52.035536', 'ens-longitude' => '-0.360673', 'coordinates' => '52.035536,-0.360673', From b76c79b536461eb94352491f91515e9a79fa79be Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Wed, 2 Aug 2023 15:10:04 +0100 Subject: [PATCH 08/11] [Central Beds] Fix typo in fly tipping waste type. --- .../Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm index c36e42430..ddf29eec2 100644 --- a/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm +++ b/perllib/Open311/Endpoint/Service/UKCouncil/CentralBedfordshireFlytipping.pm @@ -76,7 +76,7 @@ sub _build_attributes { "Trolleys" => "Trolleys", "Tyres" => "Tyres", "Vehicle parts" => "Vehicle parts", - "White goods - fridge, freezer, washing matchine etc" => "White goods - fridge, freezer, washing matchine etc", + "White goods - fridge, freezer, washing machine etc" => "White goods - fridge, freezer, washing machine etc", "Other" => "Other", } ), From d523d5fcb0002a7297b1f2d73ef28f3220443112 Mon Sep 17 00:00:00 2001 From: Nik Gupta <vngupta77@gmail.com> Date: Mon, 14 Aug 2023 12:44:40 +0100 Subject: [PATCH 09/11] [open311-adapter] [Central Beds] Allow zero lookback-days when initialising Jadu state gathering files. --- bin/centralbedfordshire/jadu/init_update_gathering_files | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/centralbedfordshire/jadu/init_update_gathering_files b/bin/centralbedfordshire/jadu/init_update_gathering_files index 029ac829b..5022d41e2 100755 --- a/bin/centralbedfordshire/jadu/init_update_gathering_files +++ b/bin/centralbedfordshire/jadu/init_update_gathering_files @@ -18,7 +18,7 @@ use Open311::Endpoint::Integration::UK::CentralBedfordshire::Jadu; my $lookback_days; GetOptions("lookback-days=i" => \$lookback_days); -if (!$lookback_days) { +if (!defined($lookback_days)) { print "Please specify a number of days to look back for updates via --lookback-days"; exit 1; } From 6c0886088ccc9b255c7fbde215f19af8e2f436ef Mon Sep 17 00:00:00 2001 From: Matthew Somerville <matthew@mysociety.org> Date: Wed, 16 Aug 2023 11:59:24 +0100 Subject: [PATCH 10/11] [Central Beds] Couple of test warnfixes. --- .../Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm | 2 ++ t/geocode/singlepoint.t | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm index bba5e4b54..c3f3fa160 100644 --- a/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm +++ b/perllib/Open311/Endpoint/Integration/UK/CentralBedfordshire/Jadu.pm @@ -548,6 +548,8 @@ sub get_service_request_updates { my $start_time = $w3c->parse_datetime($args->{start_date})->epoch; my $end_time = $w3c->parse_datetime($args->{end_date})->epoch; + return unless $self->update_storage_file; + my $update_storage_raw = path($self->update_storage_file)->slurp_raw; my $updates = decode_json($update_storage_raw); my @updates_to_send; diff --git a/t/geocode/singlepoint.t b/t/geocode/singlepoint.t index ecdb48d94..f9c8421d0 100644 --- a/t/geocode/singlepoint.t +++ b/t/geocode/singlepoint.t @@ -18,7 +18,7 @@ subtest "get_nearest_addresses" => sub { $resp->content(path(__FILE__)->sibling("files/singlepoint/spatial_radial_search_by_easting_and_northing_response.xml")->slurp); return $resp; }); - my $singlepoint = Geocode::SinglePoint->new(); + my $singlepoint = Geocode::SinglePoint->new( base_url => '', api_key => '' ); # Call with arbitrary easting, northing and radius. my $addresses = $singlepoint->get_nearest_addresses(0, 0, 0, ["STREET", "USRN", "TOWN"]); From dc0ab1ca3720580731a88879e52815c902a8ee4e Mon Sep 17 00:00:00 2001 From: Matthew Somerville <matthew@mysociety.org> Date: Wed, 16 Aug 2023 11:59:45 +0100 Subject: [PATCH 11/11] [SLWP] Store cheque payment type separately. --- perllib/Open311/Endpoint/Integration/UK/Kingston.pm | 1 + perllib/Open311/Endpoint/Integration/UK/Sutton.pm | 1 + t/open311/endpoint/sutton.t | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/perllib/Open311/Endpoint/Integration/UK/Kingston.pm b/perllib/Open311/Endpoint/Integration/UK/Kingston.pm index 068e9c266..dfa88d56f 100644 --- a/perllib/Open311/Endpoint/Integration/UK/Kingston.pm +++ b/perllib/Open311/Endpoint/Integration/UK/Kingston.pm @@ -23,6 +23,7 @@ around check_for_data_value => sub { my $method = $args->{attributes}{LastPayMethod} || ''; return 2 if $name eq 'Payment Type' && $method eq 3; # DD + return 3 if $name eq 'Payment Type' && $method eq 4; # 'cheque' (or phone) return $class->$orig($name, $args, $request, $parent_name); }; diff --git a/perllib/Open311/Endpoint/Integration/UK/Sutton.pm b/perllib/Open311/Endpoint/Integration/UK/Sutton.pm index 2e33da706..17853ba27 100644 --- a/perllib/Open311/Endpoint/Integration/UK/Sutton.pm +++ b/perllib/Open311/Endpoint/Integration/UK/Sutton.pm @@ -23,6 +23,7 @@ around check_for_data_value => sub { my $method = $args->{attributes}{LastPayMethod} || ''; return 2 if $name eq 'Payment Type' && $method eq 3; # DD + return 3 if $name eq 'Payment Type' && $method eq 4; # 'cheque' (or phone) return $class->$orig($name, $args, $request, $parent_name); }; diff --git a/t/open311/endpoint/sutton.t b/t/open311/endpoint/sutton.t index 7229e1443..fedd51042 100644 --- a/t/open311/endpoint/sutton.t +++ b/t/open311/endpoint/sutton.t @@ -70,6 +70,13 @@ $soap_lite->mock(call => sub { my @type = ${$data[1]->value}->value; is $type[0]->value, 1009; is $type[1]->value, 1; + } elsif ($client_ref eq 'LBS-2000123') { + is $event_type, EVENT_TYPE_SUBSCRIBE; + is $service_id, 409; + my @data = ${$params[0]->value}->value->value; + my @paper = ${$data[0]->value}->value; + is $paper[0]->value, 1009; + is $paper[1]->value, 3; } return SOAP::Result->new(result => { EventGuid => '1234', @@ -116,6 +123,7 @@ subtest "POST subscription request OK" => sub { 'attribute[Subscription_Details_Containers]' => 26, # Garden Bin 'attribute[Subscription_Details_Quantity]' => 1, 'attribute[Request_Type]' => 1, + 'attribute[LastPayMethod]' => 4, ); ok $res->is_success, 'valid request' or diag $res->content;