From 2c5e19c3fd95ed4b2f0014e6734ec9270111d37a Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Wed, 12 Jun 2024 14:47:11 +0100
Subject: [PATCH 01/19] [Surrey] Add WIP Boomi integration

---
 conf/council-surrey_boomi.yml-example         |   5 +
 perllib/Integrations/Surrey/Boomi.pm          | 123 +++++++++++++++
 perllib/Open311/Endpoint/Integration/Boomi.pm | 119 ++++++++++++++
 .../Open311/Endpoint/Integration/UK/Surrey.pm |  12 ++
 .../Endpoint/Service/UKCouncil/Boomi.pm       |  52 +++++++
 t/open311/endpoint/surrey_boomi.t             | 146 ++++++++++++++++++
 t/open311/endpoint/surrey_boomi.yml           |   5 +
 t/open311/endpoint/uk.t                       |   1 +
 8 files changed, 463 insertions(+)
 create mode 100644 conf/council-surrey_boomi.yml-example
 create mode 100644 perllib/Integrations/Surrey/Boomi.pm
 create mode 100644 perllib/Open311/Endpoint/Integration/Boomi.pm
 create mode 100644 perllib/Open311/Endpoint/Integration/UK/Surrey.pm
 create mode 100644 perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm
 create mode 100644 t/open311/endpoint/surrey_boomi.t
 create mode 100644 t/open311/endpoint/surrey_boomi.yml

diff --git a/conf/council-surrey_boomi.yml-example b/conf/council-surrey_boomi.yml-example
new file mode 100644
index 000000000..e6723681d
--- /dev/null
+++ b/conf/council-surrey_boomi.yml-example
@@ -0,0 +1,5 @@
+username:
+password:
+api_url: https://boomi.example.local/api/
+
+integrationId: example.01
diff --git a/perllib/Integrations/Surrey/Boomi.pm b/perllib/Integrations/Surrey/Boomi.pm
new file mode 100644
index 000000000..8a3e6cc77
--- /dev/null
+++ b/perllib/Integrations/Surrey/Boomi.pm
@@ -0,0 +1,123 @@
+=head1 NAME
+
+Integrations::Surrey::Boomi - Interface to the Surrey Boomi REST API
+
+=cut
+
+package Integrations::Surrey::Boomi;
+
+use strict;
+use warnings;
+
+use Moo;
+use LWP::UserAgent;
+use JSON::MaybeXS;
+use Try::Tiny;
+use MIME::Base64 qw(encode_base64);
+
+
+with 'Role::Logger';
+with 'Role::Config';
+
+=head1 ATTRIBUTES
+
+
+=head2 ua
+
+The LWP::UserAgent object used to make requests to the Boomi API.
+
+=cut
+
+has ua => (
+    is  => 'rw',
+    default => sub {
+        my $self = shift;
+        my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
+        my $hash = encode_base64($self->config->{username} . ':' . $self->config->{password}, "");
+        $ua->default_header('Authorization' => "Basic $hash");
+        $ua->default_header('Content-Type' => "application/json");
+        return $ua;
+    },
+);
+
+=head1 METHODS
+
+=head2 upsertHighwaysTicket
+
+Create or update a ticket in the Surrey Boomi system.
+
+Returns the ID of the created or updated ticket.
+
+=cut
+
+sub upsertHighwaysTicket {
+    my ($self, $ticket) = @_;
+
+    my $resp = $self->post('upsertHighwaysTicket', $ticket);
+    if (my $errors = $resp->{errors}) {
+        $self->logger->error("[Boomi] Error upserting ticket:");
+        $self->logger->error($_->{error} . ": " . $_->{details}) for @$errors;
+        die;
+    }
+    if (my $warnings = $resp->{warnings}) {
+        $self->logger->warn("[Boomi] Warnings when upserting ticket: ");
+        $self->logger->warn($_->{warning} . ": " . $_->{details}) for @$warnings;
+    }
+    if (my $ticket = $resp->{ticket}) {
+        return $ticket->{system} . "_" . $ticket->{id};
+    }
+    $self->error("Couldn't determine ID from response: " . encode_json($resp));
+}
+
+
+=head2 post
+
+Make a POST request to the Boomi API with the given path and data.
+
+Returns the decoded JSON from the response, if possible, otherwise logs an
+error and dies.
+
+=cut
+
+sub post {
+    my ($self, $path, $data) = @_;
+
+    my $url = $self->config->{api_url} . $path;
+    my $content = encode_json($data);
+    $self->logger->debug("[Boomi] Request URL: $url");
+    $self->logger->debug("[Boomi] Request content: $content");
+
+    my $response = $self->ua->post(
+        $url,
+        Content => $content,
+    );
+
+    $self->logger->debug("[Boomi] Response status: " . $response->status_line);
+    $self->logger->debug("[Boomi] Response content: " . $response->decoded_content);
+
+    if (!$response->is_success) {
+        $self->error("[Boomi] POST request failed: " . $response->status_line);
+    }
+
+    try {
+        my $content = $response->decoded_content;
+        return decode_json($content);
+    } catch {
+        $self->error("[Boomi] Error parsing JSON: $_");
+    };
+}
+
+=head2 error
+
+Log an error and die.
+
+=cut
+
+sub error {
+    my ($self, $message) = @_;
+
+    $self->logger->error($message);
+    die $message;
+}
+
+1;
diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
new file mode 100644
index 000000000..3d73dfeb6
--- /dev/null
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -0,0 +1,119 @@
+package Open311::Endpoint::Integration::Boomi;
+
+use Moo;
+extends 'Open311::Endpoint';
+with 'Open311::Endpoint::Role::mySociety';
+with 'Role::EndpointConfig';
+with 'Role::Logger';
+
+use POSIX qw(strftime);
+use MIME::Base64 qw(encode_base64);
+use Open311::Endpoint::Service::UKCouncil::Boomi;
+use Integrations::Surrey::Boomi;
+use JSON::MaybeXS;
+use DateTime::Format::W3CDTF;
+use Path::Tiny;
+use Try::Tiny;
+
+has jurisdiction_id => (
+    is => 'ro',
+    required => 1,
+);
+
+has boomi => (
+    is => 'lazy',
+    default => sub { $_[0]->integration_class->new(config_filename => $_[0]->jurisdiction_id) }
+);
+
+has integration_class => (
+    is => 'ro',
+    default => 'Integrations::Surrey::Boomi',
+);
+
+
+
+sub service {
+    my ($self, $id, $args) = @_;
+
+    my $service = Open311::Endpoint::Service::UKCouncil::Boomi->new(
+        service_name => $id,
+        service_code => $id,
+        description => $id,
+        type => 'realtime',
+        keywords => [qw/ /],
+        allow_any_attributes => 1,
+    );
+
+    return $service;
+}
+
+
+sub services {
+    my ($self) = @_;
+
+    # Boomi doesn't provide a list of services; they're just created as
+    # contacts in the FMS admin.
+
+    return ();
+}
+
+sub post_service_request {
+    my ($self, $service, $args) = @_;
+
+    die "Args must be a hashref" unless ref $args eq 'HASH';
+
+    $self->logger->info("[Boomi] Creating issue");
+    $self->logger->debug("[Boomi] POST service request args: " . encode_json($args));
+
+    my $ticket = {
+        integrationId => $self->endpoint_config->{integrationId},
+        subject => $args->{attributes}->{title},
+        status => 'open',
+        description => $args->{attributes}->{description},
+        location => {
+            latitude => $args->{lat},
+            longitude => $args->{long},
+            easting => $args->{attributes}->{easting},
+            northing => $args->{attributes}->{northing},
+        },
+        requester => {
+            fullName => $args->{first_name} . " " . $args->{last_name},
+            email => $args->{email},
+            phone => $args->{attributes}->{phone},
+        },
+        customFields => [
+            {
+                id => 'category',
+                values => [ $args->{attributes}->{group} ],
+            },
+            {
+                id => 'subCategory',
+                values => [ $args->{attributes}->{category} ],
+            },
+            {
+                id => 'fixmystreet_id',
+                values => [ $args->{attributes}->{fixmystreet_id} ],
+            },
+        ],
+    };
+    my $service_request_id = $self->boomi->upsertHighwaysTicket($ticket);
+
+    return $self->new_request(
+        service_request_id => $service_request_id,
+    )
+}
+
+
+
+sub get_service_request_updates {
+    my ($self, $args) = @_;
+
+    # TBC
+
+    return ();
+
+}
+
+
+
+1;
diff --git a/perllib/Open311/Endpoint/Integration/UK/Surrey.pm b/perllib/Open311/Endpoint/Integration/UK/Surrey.pm
new file mode 100644
index 000000000..11e3f1acd
--- /dev/null
+++ b/perllib/Open311/Endpoint/Integration/UK/Surrey.pm
@@ -0,0 +1,12 @@
+package Open311::Endpoint::Integration::UK::Surrey;
+
+use Moo;
+extends 'Open311::Endpoint::Integration::Boomi';
+
+around BUILDARGS => sub {
+    my ( $orig, $class, %args ) = @_;
+    $args{jurisdiction_id} = 'surrey_boomi';
+    return $class->$orig(%args);
+};
+
+1;
diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm b/perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm
new file mode 100644
index 000000000..fef78dc47
--- /dev/null
+++ b/perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm
@@ -0,0 +1,52 @@
+package Open311::Endpoint::Service::UKCouncil::Boomi;
+use Moo;
+extends 'Open311::Endpoint::Service::UKCouncil';
+
+use Open311::Endpoint::Service::Attribute;
+
+sub _build_attributes {
+    my $self = shift;
+
+    my @attributes = (
+        @{ $self->SUPER::_build_attributes() },
+        Open311::Endpoint::Service::Attribute->new(
+            code => "report_url",
+            description => "Report URL",
+            datatype => "string",
+            required => 1,
+            automated => 'server_set',
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "title",
+            description => "Title",
+            datatype => "string",
+            required => 1,
+            automated => 'server_set',
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "description",
+            description => "Description",
+            datatype => "text",
+            required => 1,
+            automated => 'server_set',
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "group",
+            description => "Group",
+            datatype => "string",
+            required => 0,
+            automated => 'server_set',
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "category",
+            description => "Category",
+            datatype => "string",
+            required => 1,
+            automated => 'server_set',
+        ),
+    );
+
+    return \@attributes;
+}
+
+1;
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
new file mode 100644
index 000000000..4ac1267e0
--- /dev/null
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -0,0 +1,146 @@
+package Integrations::Surrey::Boomi::Dummy;
+use Path::Tiny;
+use Moo;
+use HTTP::Response;
+use HTTP::Headers;
+use Test::More;
+use Test::MockModule;
+use JSON::MaybeXS qw(encode_json decode_json);
+use URI;
+
+extends 'Integrations::Surrey::Boomi';
+
+my $lwp = Test::MockModule->new('LWP::UserAgent');
+my $lwp_counter = 0;
+
+$lwp->mock(request => sub {
+    my ($ua, $req) = @_;
+
+    if ($req->uri =~ /upsertHighwaysTicket$/) {
+        is $req->method, 'POST', "Correct method used";
+        is $req->uri, 'http://localhost/ws/simple/upsertHighwaysTicket';
+        my $content = decode_json($req->content);
+        is_deeply $content, {
+        "location" => {
+            "longitude" => "0.1",
+            "northing" => "2",
+            "latitude" => "50",
+            "easting" => "1"
+        },
+        "subject" => "Pot hole on road",
+        "description" => "Big hole in the road",
+        "status" => "open",
+        "integrationId" => "Integration.1",
+        "requester" => {
+            "email" => 'test@example.com',
+            "fullName" => "Bob Mould",
+            "phone" => undef
+        },
+        "customFields" => [
+            {
+                "values" => [
+                    "Roads"
+                ],
+                "id" => "category"
+            },
+            {
+                "values" => [
+                    "Pothole"
+                ],
+                "id" => "subCategory"
+            },
+            {
+                "id" => "fixmystreet_id",
+                "values" => [
+                    "1"
+                ]
+            }
+        ]
+        };
+        return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
+    }
+});
+
+sub _build_config_file { path(__FILE__)->sibling("surrey_boomi.yml")->stringify };
+
+package Open311::Endpoint::Integration::Boomi::Dummy;
+use Path::Tiny;
+use Moo;
+use HTTP::Response;
+use HTTP::Headers;
+
+extends 'Open311::Endpoint::Integration::Boomi';
+
+has integration_class => (
+    is => 'ro',
+    default => 'Integrations::Surrey::Boomi::Dummy',
+);
+
+package main;
+
+use strict; use warnings;
+
+use utf8;
+
+use Test::More;
+use Test::MockTime ':all';
+
+use Open311::Endpoint;
+use Path::Tiny;
+use Open311::Endpoint::Integration::Boomi;
+use Integrations::Surrey::Boomi;
+use Open311::Endpoint::Service::UKCouncil;
+use LWP::UserAgent;
+use JSON::MaybeXS qw(encode_json decode_json);
+
+BEGIN { $ENV{TEST_MODE} = 1; }
+
+my $surrey_endpoint = Open311::Endpoint::Integration::Boomi::Dummy->new(
+    jurisdiction_id => 'surrey_boomi',
+    config_file => path(__FILE__)->sibling("surrey_boomi.yml")->stringify,
+);
+
+subtest "Check empty services structure" => sub {
+    my @services = $surrey_endpoint->services;
+
+    is scalar @services, 0, "Zero categories found";
+};
+
+subtest "GET Service List" => sub {
+    my $res = $surrey_endpoint->run_test_request( GET => '/services.json' );
+    ok $res->is_success, 'json success';
+    is_deeply decode_json($res->content), [];
+};
+
+subtest "POST report" => sub {
+    set_fixed_time('2023-05-01T12:00:00Z');
+    my $res = $surrey_endpoint->run_test_request(
+        POST => '/requests.json',
+        jurisdiction_id => 'surrey_boomi',
+        api_key => 'api-key',
+        service_code => 'potholes',
+        address_string => '22 Acacia Avenue',
+        first_name => 'Bob',
+        last_name => 'Mould',
+        email => 'test@example.com',
+        description => 'title: Pothole on road detail: Big hole in the road',
+        media_url => ['https://localhost/one.jpeg?123', 'https://localhost/two.jpeg?123'],
+        lat => '50',
+        long => '0.1',
+        'attribute[description]' => 'Big hole in the road',
+        'attribute[title]' => 'Pot hole on road',
+        'attribute[report_url]' => 'http://localhost/1',
+        'attribute[easting]' => 1,
+        'attribute[northing]' => 2,
+        'attribute[category]' => 'Pothole',
+        'attribute[group]' => 'Roads',
+        'attribute[fixmystreet_id]' => 1,
+        );
+    is $res->code, 200;
+    is_deeply decode_json($res->content), [{
+        service_request_id => 'Zendesk_1234',
+    }];
+    restore_time();
+};
+
+done_testing;
diff --git a/t/open311/endpoint/surrey_boomi.yml b/t/open311/endpoint/surrey_boomi.yml
new file mode 100644
index 000000000..97586af10
--- /dev/null
+++ b/t/open311/endpoint/surrey_boomi.yml
@@ -0,0 +1,5 @@
+username: "my_surrey_username"
+password: "my_surrey_password"
+api_url: http://localhost/ws/simple/
+
+integrationId: "Integration.1"
diff --git a/t/open311/endpoint/uk.t b/t/open311/endpoint/uk.t
index a11bd1365..1327eae3e 100644
--- a/t/open311/endpoint/uk.t
+++ b/t/open311/endpoint/uk.t
@@ -19,6 +19,7 @@ test_multi(0, 'Open311::Endpoint::Integration::UK',
     'Open311::Endpoint::Integration::UK::Rutland' => 'rutland',
     'Open311::Endpoint::Integration::UK::Shropshire' => 'shropshire_confirm',
     'Open311::Endpoint::Integration::UK::Southwark' => 'southwark_confirm',
+    'Open311::Endpoint::Integration::UK::Surrey' => 'surrey_boomi',
     'Open311::Endpoint::Integration::UK::Sutton' => 'sutton_echo',
     'Open311::Endpoint::Integration::UK::Hampshire' => 'hampshire_confirm',
 );

From ffd2fa2b2f99e998b0c5a3fbdb4cbc45125e8cd8 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Fri, 28 Jun 2024 14:28:22 +0100
Subject: [PATCH 02/19] [Surrey] Add fetching of updates from Boomi

---
 perllib/Integrations/Surrey/Boomi.pm          | 93 ++++++++++++++++++-
 perllib/Open311/Endpoint/Integration/Boomi.pm | 29 +++++-
 t/open311/endpoint/surrey_boomi.t             | 69 +++++++++++++-
 t/open311/endpoint/surrey_boomi.yml           |  4 +-
 t/open311/endpoint/uk.t                       |  1 -
 5 files changed, 189 insertions(+), 7 deletions(-)

diff --git a/perllib/Integrations/Surrey/Boomi.pm b/perllib/Integrations/Surrey/Boomi.pm
index 8a3e6cc77..33870c7dd 100644
--- a/perllib/Integrations/Surrey/Boomi.pm
+++ b/perllib/Integrations/Surrey/Boomi.pm
@@ -10,6 +10,7 @@ use strict;
 use warnings;
 
 use Moo;
+use URI;
 use LWP::UserAgent;
 use JSON::MaybeXS;
 use Try::Tiny;
@@ -33,6 +34,7 @@ has ua => (
     default => sub {
         my $self = shift;
         my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
+        $ua->ssl_opts(SSL_cipher_list => 'DEFAULT:!DH'); # Disable DH ciphers, server's key is too small apparently
         my $hash = encode_base64($self->config->{username} . ':' . $self->config->{password}, "");
         $ua->default_header('Authorization' => "Basic $hash");
         $ua->default_header('Content-Type' => "application/json");
@@ -69,6 +71,36 @@ sub upsertHighwaysTicket {
     $self->error("Couldn't determine ID from response: " . encode_json($resp));
 }
 
+=head2 getHighwaysTicketUpdates
+
+Create or update a ticket in the Surrey Boomi system.
+
+Returns the ID of the created or updated ticket.
+
+=cut
+
+sub getHighwaysTicketUpdates {
+    my ($self, $start, $end) = @_;
+
+    my $resp = $self->get('getHighwaysTicketUpdates', {
+        from => format_datetime($start),
+        to => format_datetime($end),
+        integration_id => $self->config->{integration_ids}->{getHighwaysTicketUpdates},
+    });
+
+    if (my $errors = $resp->{errors}) {
+        $self->logger->error("[Boomi] Error fetching updates:");
+        $self->logger->error($_->{error} . ": " . $_->{details}) for @$errors;
+        die;
+    }
+    if (my $warnings = $resp->{warnings}) {
+        $self->logger->warn("[Boomi] Warnings when fetching updates:");
+        $self->logger->warn($_->{warning} . ": " . $_->{details}) for @$warnings;
+    }
+
+    return $resp->{results} || [];
+}
+
 
 =head2 post
 
@@ -84,7 +116,7 @@ sub post {
 
     my $url = $self->config->{api_url} . $path;
     my $content = encode_json($data);
-    $self->logger->debug("[Boomi] Request URL: $url");
+    $self->logger->debug("[Boomi] Request URL for POST: $url");
     $self->logger->debug("[Boomi] Request content: $content");
 
     my $response = $self->ua->post(
@@ -107,6 +139,41 @@ sub post {
     };
 }
 
+=head2 get
+
+Make a GET request to the Boomi API with the given path and query params.
+
+Returns the decoded JSON from the response, if possible, otherwise logs an
+error and dies.
+
+=cut
+
+sub get {
+    my ($self, $path, $params) = @_;
+
+    my $uri = URI->new( $self->config->{api_url} . $path );
+    $uri->query_form(%$params);
+
+    my $request = HTTP::Request->new("GET", $uri);
+    $self->logger->debug("[Boomi] Request: " . $request->as_string);
+
+    my $response = $self->ua->request($request);
+
+    $self->logger->debug("[Boomi] Response status: " . $response->status_line);
+    $self->logger->debug("[Boomi] Response content: " . $response->decoded_content);
+
+    if (!$response->is_success) {
+        $self->error("[Boomi] GET request failed: " . $response->status_line);
+    }
+
+    try {
+        my $content = $response->decoded_content;
+        return decode_json($content);
+    } catch {
+        $self->error("[Boomi] Error parsing JSON: $_");
+    };
+}
+
 =head2 error
 
 Log an error and die.
@@ -120,4 +187,28 @@ sub error {
     die $message;
 }
 
+
+=head2 format_datetime
+
+Format a DateTime object as a string in the format expected by the Boomi API.
+(Adapted from DateTime::Format::W3CDTF to always include milliseconds.)
+
+=cut
+sub format_datetime {
+    my $dt = shift;
+
+    my $cldr = 'yyyy-MM-ddTHH:mm:ss.SSS';
+
+    my $tz;
+    if ( $dt->time_zone->is_utc ) {
+        $tz = 'Z';
+    }
+    else {
+        $tz = q{};
+        $cldr .= 'ZZZZZ';
+    }
+
+    return $dt->format_cldr($cldr) . $tz;
+}
+
 1;
diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 3d73dfeb6..ca80c4bca 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -9,6 +9,7 @@ with 'Role::Logger';
 use POSIX qw(strftime);
 use MIME::Base64 qw(encode_base64);
 use Open311::Endpoint::Service::UKCouncil::Boomi;
+use Open311::Endpoint::Service::Request::Update::mySociety;
 use Integrations::Surrey::Boomi;
 use JSON::MaybeXS;
 use DateTime::Format::W3CDTF;
@@ -66,7 +67,7 @@ sub post_service_request {
     $self->logger->debug("[Boomi] POST service request args: " . encode_json($args));
 
     my $ticket = {
-        integrationId => $self->endpoint_config->{integrationId},
+        integrationId => $self->endpoint_config->{integration_ids}->{upsertHighwaysTicket},
         subject => $args->{attributes}->{title},
         status => 'open',
         description => $args->{attributes}->{description},
@@ -108,10 +109,32 @@ sub post_service_request {
 sub get_service_request_updates {
     my ($self, $args) = @_;
 
-    # TBC
+    my $start = DateTime::Format::W3CDTF->parse_datetime($args->{start_date});
+    my $end = DateTime::Format::W3CDTF->parse_datetime($args->{end_date});
 
-    return ();
+    my $results = $self->boomi->getHighwaysTicketUpdates($start, $end);
+    my $w3c = DateTime::Format::W3CDTF->new;
+
+    my @updates;
+
+    for my $update (@$results) {
+        my $log = $update->{confirmEnquiryStatusLog};
+        my $fms = $update->{fmsReport};
+
+        my $id = $log->{enquiry}->{externalSystemReference} . "_" . $log->{logNumber};
+        my $status = lc $fms->{status}->{state};
+        $status =~ s/ /_/g;
+
+        push @updates, Open311::Endpoint::Service::Request::Update::mySociety->new(
+            status => $status,
+            update_id => $id,
+            service_request_id => "Zendesk_" . $log->{enquiry}->{externalSystemReference},
+            description => $fms->{status}->{label},
+            updated_datetime => $w3c->parse_datetime( $log->{loggedDate} )->truncate( to => 'second' ),
+        );
+    }
 
+    return @updates;
 }
 
 
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 4ac1267e0..b893817ed 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -11,7 +11,6 @@ use URI;
 extends 'Integrations::Surrey::Boomi';
 
 my $lwp = Test::MockModule->new('LWP::UserAgent');
-my $lwp_counter = 0;
 
 $lwp->mock(request => sub {
     my ($ua, $req) = @_;
@@ -58,6 +57,49 @@ $lwp->mock(request => sub {
         ]
         };
         return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
+    } elsif ($req->uri =~ /getHighwaysTicketUpdates/) {
+        is $req->method, 'GET', "Correct method used";
+
+
+        return HTTP::Response->new(200, 'OK', [], encode_json({
+            "executionId" => "execution-7701f16b-036c-4e6e-8e14-998f81f5b6b8-2024.06.27",
+            "results" => [
+                {
+                    "confirmEnquiryStatusLog" => {
+                        "loggedDate" => "2024-05-01T09:07:47.000Z",
+                        "logNumber" => 11,
+                        "statusCode" => "5800",
+                        "enquiry" => {
+                            "enquiryNumber" => 129293,
+                            "externalSystemReference" => "2929177"
+                        }
+                    },
+                    "fmsReport" => {
+                        "status" => {
+                            "state" => "Closed",
+                            "label" => "Enquiry closed"
+                        }
+                    }
+                },
+                {
+                    "confirmEnquiryStatusLog" => {
+                        "loggedDate" => "2024-05-01T09:10:41.000Z",
+                        "logNumber" => 7,
+                        "statusCode" => "3200",
+                        "enquiry" => {
+                            "enquiryNumber" => 132361,
+                            "externalSystemReference" => "2939061"
+                        }
+                    },
+                    "fmsReport" => {
+                        "status" => {
+                            "state" => "Action scheduled",
+                            "label" => "Assessed - scheduling a repair within 5 Working Days"
+                        }
+                    }
+                },
+            ]
+    }));
     }
 });
 
@@ -143,4 +185,29 @@ subtest "POST report" => sub {
     restore_time();
 };
 
+subtest "GET Service Request Updates" => sub {
+    my $res = $surrey_endpoint->run_test_request(
+        GET => '/servicerequestupdates.json?jurisdiction_id=surrey_boomi&api_key=api-key&start_date=2024-05-01T09:00:00Z&end_date=2024-05-01T10:00:00Z',
+    );
+    is $res->code, 200;
+    is_deeply decode_json($res->content), [
+       {
+          "description" => "Enquiry closed",
+          "media_url" => "",
+          "service_request_id" => "Zendesk_2929177",
+          "status" => "closed",
+          "update_id" => "2929177_11",
+          "updated_datetime" => "2024-05-01T09:07:47Z",
+       },
+       {
+          "description" => "Assessed - scheduling a repair within 5 Working Days",
+          "media_url" => "",
+          "service_request_id" => "Zendesk_2939061",
+          "status" => "action_scheduled",
+          "update_id" => "2939061_7",
+          "updated_datetime" => "2024-05-01T09:10:41Z",
+       }
+    ];
+};
+
 done_testing;
diff --git a/t/open311/endpoint/surrey_boomi.yml b/t/open311/endpoint/surrey_boomi.yml
index 97586af10..d5920df27 100644
--- a/t/open311/endpoint/surrey_boomi.yml
+++ b/t/open311/endpoint/surrey_boomi.yml
@@ -2,4 +2,6 @@ username: "my_surrey_username"
 password: "my_surrey_password"
 api_url: http://localhost/ws/simple/
 
-integrationId: "Integration.1"
+integration_ids:
+  upsertHighwaysTicket: "Integration.1"
+  getHighwaysTicketUpdates: "Integration.2"
diff --git a/t/open311/endpoint/uk.t b/t/open311/endpoint/uk.t
index 1327eae3e..a11bd1365 100644
--- a/t/open311/endpoint/uk.t
+++ b/t/open311/endpoint/uk.t
@@ -19,7 +19,6 @@ test_multi(0, 'Open311::Endpoint::Integration::UK',
     'Open311::Endpoint::Integration::UK::Rutland' => 'rutland',
     'Open311::Endpoint::Integration::UK::Shropshire' => 'shropshire_confirm',
     'Open311::Endpoint::Integration::UK::Southwark' => 'southwark_confirm',
-    'Open311::Endpoint::Integration::UK::Surrey' => 'surrey_boomi',
     'Open311::Endpoint::Integration::UK::Sutton' => 'sutton_echo',
     'Open311::Endpoint::Integration::UK::Hampshire' => 'hampshire_confirm',
 );

From f5952337bf4a51947c8753d5d7510673c069f4cc Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Wed, 3 Jul 2024 14:21:43 +0100
Subject: [PATCH 03/19] [Surrey] Send photos to Boomi

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 32 ++++++-
 t/open311/endpoint/surrey_boomi.t             | 84 +++++++++++--------
 2 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index ca80c4bca..294f11d5a 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -15,6 +15,7 @@ use JSON::MaybeXS;
 use DateTime::Format::W3CDTF;
 use Path::Tiny;
 use Try::Tiny;
+use LWP::UserAgent;
 
 has jurisdiction_id => (
     is => 'ro',
@@ -64,7 +65,7 @@ sub post_service_request {
     die "Args must be a hashref" unless ref $args eq 'HASH';
 
     $self->logger->info("[Boomi] Creating issue");
-    $self->logger->debug("[Boomi] POST service request args: " . encode_json($args));
+    # $self->logger->debug("[Boomi] POST service request args: " . encode_json($args));
 
     my $ticket = {
         integrationId => $self->endpoint_config->{integration_ids}->{upsertHighwaysTicket},
@@ -97,6 +98,35 @@ sub post_service_request {
             },
         ],
     };
+
+    my @attachments;
+
+    my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
+
+    for my $photo (@{ $args->{media_url} }) {
+        my $photo_response = $ua->get($photo);
+        unless ( $photo_response->is_success) {
+            $self->logger->error("Failed to retrieve photo from $photo\n");
+            die "Failed to retrieve photo from $photo";
+        }
+
+        push @attachments, {
+            fileName => $photo_response->filename,
+            url => $photo,
+            base64 => encode_base64($photo_response->content),
+        };
+    }
+
+    if (@{$args->{uploads}}) {
+        foreach (@{$args->{uploads}}) {
+            push @attachments, {
+                fileName => $_->filename,
+                base64 => encode_base64(path($_)->slurp),
+            };
+        }
+    }
+    $ticket->{attachments} = \@attachments if @attachments;
+
     my $service_request_id = $self->boomi->upsertHighwaysTicket($ticket);
 
     return $self->new_request(
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index b893817ed..cda673ccd 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -20,41 +20,48 @@ $lwp->mock(request => sub {
         is $req->uri, 'http://localhost/ws/simple/upsertHighwaysTicket';
         my $content = decode_json($req->content);
         is_deeply $content, {
-        "location" => {
-            "longitude" => "0.1",
-            "northing" => "2",
-            "latitude" => "50",
-            "easting" => "1"
-        },
-        "subject" => "Pot hole on road",
-        "description" => "Big hole in the road",
-        "status" => "open",
-        "integrationId" => "Integration.1",
-        "requester" => {
-            "email" => 'test@example.com',
-            "fullName" => "Bob Mould",
-            "phone" => undef
-        },
-        "customFields" => [
-            {
-                "values" => [
-                    "Roads"
-                ],
-                "id" => "category"
+            "location" => {
+                "longitude" => "0.1",
+                "northing" => "2",
+                "latitude" => "50",
+                "easting" => "1"
             },
-            {
-                "values" => [
-                    "Pothole"
-                ],
-                "id" => "subCategory"
+            "subject" => "Pot hole on road",
+            "description" => "Big hole in the road",
+            "status" => "open",
+            "integrationId" => "Integration.1",
+            "requester" => {
+                "email" => 'test@example.com',
+                "fullName" => "Bob Mould",
+                "phone" => undef
             },
-            {
-                "id" => "fixmystreet_id",
-                "values" => [
-                    "1"
-                ]
-            }
-        ]
+            "customFields" => [
+                {
+                    "values" => [
+                        "Roads"
+                    ],
+                    "id" => "category"
+                },
+                {
+                    "values" => [
+                        "Pothole"
+                    ],
+                    "id" => "subCategory"
+                },
+                {
+                    "id" => "fixmystreet_id",
+                    "values" => [
+                        "1"
+                    ]
+                }
+            ],
+            "attachments" => [
+                {
+                    "url" => "http://localhost/photo/one.jpeg",
+                    "fileName" => "1.jpeg",
+                    "base64" => "/9j/4AAQSkZJRgABAQAAAAAAAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkI\nCQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/wAALCAABAAEBAREA/8QAFAABAAAAAAAA\nAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==\n",
+                },
+            ]
         };
         return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
     } elsif ($req->uri =~ /getHighwaysTicketUpdates/) {
@@ -99,7 +106,14 @@ $lwp->mock(request => sub {
                     }
                 },
             ]
-    }));
+        }));
+    } elsif ($req->uri eq 'http://localhost/photo/one.jpeg') {
+        my $image_data = path(__FILE__)->sibling('files')->child('test_image.jpg')->slurp;
+        my $response = HTTP::Response->new(200, 'OK', []);
+        $response->header('Content-Disposition' => 'attachment; filename="1.jpeg"');
+        $response->header('Content-Type' => 'image/jpeg');
+        $response->content($image_data);
+        return $response;
     }
 });
 
@@ -166,7 +180,7 @@ subtest "POST report" => sub {
         last_name => 'Mould',
         email => 'test@example.com',
         description => 'title: Pothole on road detail: Big hole in the road',
-        media_url => ['https://localhost/one.jpeg?123', 'https://localhost/two.jpeg?123'],
+        media_url => ['http://localhost/photo/one.jpeg'],
         lat => '50',
         long => '0.1',
         'attribute[description]' => 'Big hole in the road',

From 2b1b7313797659445cace4f7570e062c12ee2c89 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Wed, 3 Jul 2024 21:27:40 +0100
Subject: [PATCH 04/19] [Surrey] Forward all Open311 metadata values to Boomi

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 38 ++++++++++++-------
 t/open311/endpoint/surrey_boomi.t             | 17 ++++++++-
 2 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 294f11d5a..7dff6090a 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -67,6 +67,17 @@ sub post_service_request {
     $self->logger->info("[Boomi] Creating issue");
     # $self->logger->debug("[Boomi] POST service request args: " . encode_json($args));
 
+    my @custom_fields = (
+        {
+            id => 'category',
+            values => [ $args->{attributes}->{group} ],
+        },
+        {
+            id => 'subCategory',
+            values => [ $args->{attributes}->{category} ],
+        },
+    );
+
     my $ticket = {
         integrationId => $self->endpoint_config->{integration_ids}->{upsertHighwaysTicket},
         subject => $args->{attributes}->{title},
@@ -83,22 +94,21 @@ sub post_service_request {
             email => $args->{email},
             phone => $args->{attributes}->{phone},
         },
-        customFields => [
-            {
-                id => 'category',
-                values => [ $args->{attributes}->{group} ],
-            },
-            {
-                id => 'subCategory',
-                values => [ $args->{attributes}->{category} ],
-            },
-            {
-                id => 'fixmystreet_id',
-                values => [ $args->{attributes}->{fixmystreet_id} ],
-            },
-        ],
     };
 
+    foreach (qw/group category easting northing title description phone/) {
+        if (defined $args->{attributes}->{$_}) {
+            delete $args->{attributes}->{$_};
+        }
+    }
+    for my $attr (sort keys %{ $args->{attributes} }) {
+        push @custom_fields, {
+            id => $attr,
+            values => [ $args->{attributes}->{$attr} ],
+        };
+    }
+    $ticket->{customFields} = \@custom_fields;
+
     my @attachments;
 
     my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index cda673ccd..4eafa38a3 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -48,12 +48,24 @@ $lwp->mock(request => sub {
                     ],
                     "id" => "subCategory"
                 },
+                {
+                    "id" => "RM1",
+                    "values" => [
+                        "RM1B"
+                    ]
+                },
                 {
                     "id" => "fixmystreet_id",
                     "values" => [
                         "1"
                     ]
-                }
+                },
+                {
+                    "id" => "report_url",
+                    "values" => [
+                        "http://localhost/report/1"
+                    ]
+                },
             ],
             "attachments" => [
                 {
@@ -185,12 +197,13 @@ subtest "POST report" => sub {
         long => '0.1',
         'attribute[description]' => 'Big hole in the road',
         'attribute[title]' => 'Pot hole on road',
-        'attribute[report_url]' => 'http://localhost/1',
+        'attribute[report_url]' => 'http://localhost/report/1',
         'attribute[easting]' => 1,
         'attribute[northing]' => 2,
         'attribute[category]' => 'Pothole',
         'attribute[group]' => 'Roads',
         'attribute[fixmystreet_id]' => 1,
+        'attribute[RM1]' => "RM1B",
         );
     is $res->code, 200;
     is_deeply decode_json($res->content), [{

From 632747b2f0ccf3b435dca2a6fa785f0aa38e56b7 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Thu, 4 Jul 2024 13:36:35 +0100
Subject: [PATCH 05/19] [Surrey] Pass USRN/road name to Boomi

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 4 +++-
 t/open311/endpoint/surrey_boomi.t             | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 7dff6090a..42497b33c 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -88,6 +88,8 @@ sub post_service_request {
             longitude => $args->{long},
             easting => $args->{attributes}->{easting},
             northing => $args->{attributes}->{northing},
+            usrn => $args->{attributes}->{USRN},
+            streetName => $args->{attributes}->{ROADNAME},
         },
         requester => {
             fullName => $args->{first_name} . " " . $args->{last_name},
@@ -96,7 +98,7 @@ sub post_service_request {
         },
     };
 
-    foreach (qw/group category easting northing title description phone/) {
+    foreach (qw/group category easting northing title description phone USRN ROADNAME/) {
         if (defined $args->{attributes}->{$_}) {
             delete $args->{attributes}->{$_};
         }
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 4eafa38a3..69968a18f 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -24,7 +24,9 @@ $lwp->mock(request => sub {
                 "longitude" => "0.1",
                 "northing" => "2",
                 "latitude" => "50",
-                "easting" => "1"
+                "easting" => "1",
+                "usrn" => "31200342",
+                "streetName" => "Cockshot Hill",
             },
             "subject" => "Pot hole on road",
             "description" => "Big hole in the road",
@@ -204,6 +206,8 @@ subtest "POST report" => sub {
         'attribute[group]' => 'Roads',
         'attribute[fixmystreet_id]' => 1,
         'attribute[RM1]' => "RM1B",
+        'attribute[USRN]' => "31200342",
+        'attribute[ROADNAME]' => "Cockshot Hill",
         );
     is $res->code, 200;
     is_deeply decode_json($res->content), [{

From 8d4efbfbeb381fef058fed2da45194f6d5a8eca7 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Thu, 11 Jul 2024 22:43:39 +0100
Subject: [PATCH 06/19] [Surrey] Allow multivaluelist attributes through

---
 perllib/Open311/Endpoint/Integration/Boomi.pm |  4 +-
 perllib/Open311/Endpoint/Service/Attribute.pm | 13 ++++++
 .../Endpoint/Service/UKCouncil/Boomi.pm       | 43 +++++++++++++++++++
 t/open311/endpoint/surrey_boomi.t             |  8 ++++
 4 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 42497b33c..fec274e4e 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -104,9 +104,11 @@ sub post_service_request {
         }
     }
     for my $attr (sort keys %{ $args->{attributes} }) {
+        my $val = $args->{attributes}->{$attr};
         push @custom_fields, {
             id => $attr,
-            values => [ $args->{attributes}->{$attr} ],
+            # multivaluelist fields arrive as an arrayref
+            values => ref $val eq 'ARRAY' ? $val : [ $val ],
         };
     }
     $ticket->{customFields} = \@custom_fields;
diff --git a/perllib/Open311/Endpoint/Service/Attribute.pm b/perllib/Open311/Endpoint/Service/Attribute.pm
index 9745912cb..1a40fac39 100644
--- a/perllib/Open311/Endpoint/Service/Attribute.pm
+++ b/perllib/Open311/Endpoint/Service/Attribute.pm
@@ -70,6 +70,15 @@ has values => (
     }
 );
 
+# for singlevaluelist or multivalue list, allow any value so we don't have to
+# hardcode a list when defining attribute.
+has allow_any_value => (
+    is => 'ro',
+    isa => Bool,
+    default => sub { 0 },
+);
+
+
 has values_sorted => (
     is => 'ro',
     isa => ArrayRef,
@@ -88,6 +97,10 @@ sub schema_definition {
     # FMS will send a blank string for optional singlevaluelist attributes where
     # the user didn't make a selection. Make sure this is allowed by the schema.
     push(@values, { type => '//str', value => '' }) unless $self->required;
+    # Some integrations have extra fields whose options are managed within
+    # the FMS admin rather than being fixed. For these we need to ensure
+    # we can accept any value.
+    push(@values, { type => '//str' }) if $self->allow_any_value;
 
     my %schema_types = (
         string => '//str',
diff --git a/perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm b/perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm
index fef78dc47..c95c94d70 100644
--- a/perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm
+++ b/perllib/Open311/Endpoint/Service/UKCouncil/Boomi.pm
@@ -44,6 +44,49 @@ sub _build_attributes {
             required => 1,
             automated => 'server_set',
         ),
+
+        # XXX these should not be hardcoded here but haven't yet figured out
+        # how to allow any attribute/value to pass schema validation
+        Open311::Endpoint::Service::Attribute->new(
+            code => "Q7",
+            description => "Q7",
+            datatype => "multivaluelist",
+            required => 0,
+            variable => 1,
+            allow_any_value => 1,
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "pothole_severity",
+            description => "pothole_severity",
+            datatype => "multivaluelist",
+            required => 0,
+            variable => 1,
+            allow_any_value => 1,
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "pothole_location",
+            description => "pothole_location",
+            datatype => "multivaluelist",
+            required => 0,
+            variable => 1,
+            allow_any_value => 1,
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "D1_Declaration",
+            description => "D1_Declaration",
+            datatype => "multivaluelist",
+            required => 0,
+            variable => 1,
+            allow_any_value => 1,
+        ),
+        Open311::Endpoint::Service::Attribute->new(
+            code => "1_Location",
+            description => "1_Location",
+            datatype => "multivaluelist",
+            required => 0,
+            variable => 1,
+            allow_any_value => 1,
+        ),
     );
 
     return \@attributes;
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 69968a18f..90c25a548 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -50,6 +50,12 @@ $lwp->mock(request => sub {
                     ],
                     "id" => "subCategory"
                 },
+                {
+                    "id" => "Q7",
+                    "values" => [
+                        "T1", "T3"
+                    ]
+                },
                 {
                     "id" => "RM1",
                     "values" => [
@@ -208,6 +214,8 @@ subtest "POST report" => sub {
         'attribute[RM1]' => "RM1B",
         'attribute[USRN]' => "31200342",
         'attribute[ROADNAME]' => "Cockshot Hill",
+        'attribute[Q7]' => "T1",
+        'attribute[Q7]' => "T3",
         );
     is $res->code, 200;
     is_deeply decode_json($res->content), [{

From 23fd97ac26a755c8293b0972f5a32f8a308c0066 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Fri, 12 Jul 2024 16:34:47 +0100
Subject: [PATCH 07/19] [Surrey] Handle incoming JSON-encoded attribute values

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 14 ++++++++++++++
 t/open311/endpoint/surrey_boomi.t             | 17 +++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index fec274e4e..af5d49544 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -105,10 +105,24 @@ sub post_service_request {
     }
     for my $attr (sort keys %{ $args->{attributes} }) {
         my $val = $args->{attributes}->{$attr};
+
+        my $name;
+        # see if it's a JSON string that encodes a value and a description
+        if ($val =~ /^{/) {
+            try {
+                my $decoded = decode_json($val);
+                $val = $decoded->{value} || $val;
+                $name = $decoded->{description} || "";
+            } catch {
+                $self->logger->debug("[Boomi] Couldn't decode JSON from Open311 attribute value: $val");
+            };
+        }
+
         push @custom_fields, {
             id => $attr,
             # multivaluelist fields arrive as an arrayref
             values => ref $val eq 'ARRAY' ? $val : [ $val ],
+            $name ? ( name => $name) : (),
         };
     }
     $ticket->{customFields} = \@custom_fields;
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 90c25a548..e75b906d0 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -62,6 +62,21 @@ $lwp->mock(request => sub {
                         "RM1B"
                     ]
                 },
+                {
+                    "values" => [
+                        "A value"
+                    ],
+                    "id" => "complexfield",
+                    "name" => "A complex field"
+                },
+                {
+                    "values" => [
+                        "A value",
+                        "Another value"
+                    ],
+                    "id" => "complexlist",
+                    "name" => "A complex array"
+                },
                 {
                     "id" => "fixmystreet_id",
                     "values" => [
@@ -216,6 +231,8 @@ subtest "POST report" => sub {
         'attribute[ROADNAME]' => "Cockshot Hill",
         'attribute[Q7]' => "T1",
         'attribute[Q7]' => "T3",
+        'attribute[complexfield]' => "{\"description\":\"A complex field\", \"value\":\"A value\"}",
+        'attribute[complexlist]' => "{\"description\":\"A complex array\", \"value\":[\"A value\", \"Another value\"]}",
         );
     is $res->code, 200;
     is_deeply decode_json($res->content), [{

From 665963aee9e65b492dd64f9fb90299ade660f49c Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Fri, 12 Jul 2024 22:04:01 +0100
Subject: [PATCH 08/19] [Surrey] Correctly send phone number

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 2 +-
 t/open311/endpoint/surrey_boomi.t             | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index af5d49544..133cdea11 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -94,7 +94,7 @@ sub post_service_request {
         requester => {
             fullName => $args->{first_name} . " " . $args->{last_name},
             email => $args->{email},
-            phone => $args->{attributes}->{phone},
+            phone => $args->{phone},
         },
     };
 
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index e75b906d0..c7a203b8d 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -35,7 +35,7 @@ $lwp->mock(request => sub {
             "requester" => {
                 "email" => 'test@example.com',
                 "fullName" => "Bob Mould",
-                "phone" => undef
+                "phone" => "07123123123"
             },
             "customFields" => [
                 {
@@ -214,6 +214,7 @@ subtest "POST report" => sub {
         first_name => 'Bob',
         last_name => 'Mould',
         email => 'test@example.com',
+        phone => '07123123123',
         description => 'title: Pothole on road detail: Big hole in the road',
         media_url => ['http://localhost/photo/one.jpeg'],
         lat => '50',

From 982ab93187abf17f3194ad5ea145bbb363afb592 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Thu, 18 Jul 2024 10:48:54 +0100
Subject: [PATCH 09/19] [Surrey] Send updates to Boomi

---
 perllib/Open311/Endpoint/Integration/Boomi.pm |  36 ++++
 t/open311/endpoint/surrey_boomi.t             | 195 +++++++++++-------
 2 files changed, 153 insertions(+), 78 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 133cdea11..611c3901c 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -16,6 +16,7 @@ use DateTime::Format::W3CDTF;
 use Path::Tiny;
 use Try::Tiny;
 use LWP::UserAgent;
+use Digest::MD5 qw(md5_hex);
 
 has jurisdiction_id => (
     is => 'ro',
@@ -195,6 +196,41 @@ sub get_service_request_updates {
     return @updates;
 }
 
+sub post_service_request_update {
+    my ($self, $args) = @_;
+
+    die "Args must be a hashref" unless ref $args eq 'HASH';
+
+    if ($args->{media_url}->[0]) {
+        $args->{description} .= "\n\n[ This update contains a photo, see: " . $args->{media_url}->[0] . " ]";
+    }
+
+
+    $self->logger->info("[Boomi] Creating update");
+
+    my ($system, $id) = split('_', $args->{service_request_id});
+
+    my $ticket = {
+        integrationId => $self->endpoint_config->{integration_ids}->{upsertHighwaysTicket},
+        ticketId => $id,
+        comments => [
+            { body => $args->{description} },
+        ],
+    };
+
+    # we don't get back a unique ID from Boomi, so calculate one ourselves
+    # XXX is this going to be mirrored back next time we fetch updates?
+    my $update_id = $args->{service_request_id} . "_" . substr(md5_hex($id . $args->{description}), 0, 8);
+
+    my $service_request_id = $self->boomi->upsertHighwaysTicket($ticket);
+
+    return Open311::Endpoint::Service::Request::Update::mySociety->new(
+        service_request_id => $args->{service_request_id},
+        status => lc $args->{status},
+        update_id => $update_id,
+    );
+}
+
 
 
 1;
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index c7a203b8d..cc8442701 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -19,86 +19,100 @@ $lwp->mock(request => sub {
         is $req->method, 'POST', "Correct method used";
         is $req->uri, 'http://localhost/ws/simple/upsertHighwaysTicket';
         my $content = decode_json($req->content);
-        is_deeply $content, {
-            "location" => {
-                "longitude" => "0.1",
-                "northing" => "2",
-                "latitude" => "50",
-                "easting" => "1",
-                "usrn" => "31200342",
-                "streetName" => "Cockshot Hill",
-            },
-            "subject" => "Pot hole on road",
-            "description" => "Big hole in the road",
-            "status" => "open",
-            "integrationId" => "Integration.1",
-            "requester" => {
-                "email" => 'test@example.com',
-                "fullName" => "Bob Mould",
-                "phone" => "07123123123"
-            },
-            "customFields" => [
-                {
-                    "values" => [
-                        "Roads"
-                    ],
-                    "id" => "category"
-                },
-                {
-                    "values" => [
-                        "Pothole"
-                    ],
-                    "id" => "subCategory"
-                },
-                {
-                    "id" => "Q7",
-                    "values" => [
-                        "T1", "T3"
-                    ]
-                },
-                {
-                    "id" => "RM1",
-                    "values" => [
-                        "RM1B"
-                    ]
-                },
-                {
-                    "values" => [
-                        "A value"
-                    ],
-                    "id" => "complexfield",
-                    "name" => "A complex field"
-                },
-                {
-                    "values" => [
-                        "A value",
-                        "Another value"
-                    ],
-                    "id" => "complexlist",
-                    "name" => "A complex array"
-                },
-                {
-                    "id" => "fixmystreet_id",
-                    "values" => [
-                        "1"
-                    ]
-                },
-                {
-                    "id" => "report_url",
-                    "values" => [
-                        "http://localhost/report/1"
-                    ]
+        if ($content && $content->{ticketId}) {
+            # it's an update to an existing ticket
+            is_deeply $content, {
+                "comments" => [
+                    {
+                        "body" => "This is an update"
+                    }
+                ],
+                "ticketId" => "123456",
+                "integrationId" => "Integration.1"
+            };
+            return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
+        } else {
+            is_deeply $content, {
+                "location" => {
+                    "longitude" => "0.1",
+                    "northing" => "2",
+                    "latitude" => "50",
+                    "easting" => "1",
+                    "usrn" => "31200342",
+                    "streetName" => "Cockshot Hill",
                 },
-            ],
-            "attachments" => [
-                {
-                    "url" => "http://localhost/photo/one.jpeg",
-                    "fileName" => "1.jpeg",
-                    "base64" => "/9j/4AAQSkZJRgABAQAAAAAAAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkI\nCQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/wAALCAABAAEBAREA/8QAFAABAAAAAAAA\nAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==\n",
+                "subject" => "Pot hole on road",
+                "description" => "Big hole in the road",
+                "status" => "open",
+                "integrationId" => "Integration.1",
+                "requester" => {
+                    "email" => 'test@example.com',
+                    "fullName" => "Bob Mould",
+                    "phone" => "07123123123"
                 },
-            ]
-        };
-        return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
+                "customFields" => [
+                    {
+                        "values" => [
+                            "Roads"
+                        ],
+                        "id" => "category"
+                    },
+                    {
+                        "values" => [
+                            "Pothole"
+                        ],
+                        "id" => "subCategory"
+                    },
+                    {
+                        "id" => "Q7",
+                        "values" => [
+                            "T1", "T3"
+                        ]
+                    },
+                    {
+                        "id" => "RM1",
+                        "values" => [
+                            "RM1B"
+                        ]
+                    },
+                    {
+                        "values" => [
+                            "A value"
+                        ],
+                        "id" => "complexfield",
+                        "name" => "A complex field"
+                    },
+                    {
+                        "values" => [
+                            "A value",
+                            "Another value"
+                        ],
+                        "id" => "complexlist",
+                        "name" => "A complex array"
+                    },
+                    {
+                        "id" => "fixmystreet_id",
+                        "values" => [
+                            "1"
+                        ]
+                    },
+                    {
+                        "id" => "report_url",
+                        "values" => [
+                            "http://localhost/report/1"
+                        ]
+                    },
+                ],
+                "attachments" => [
+                    {
+                        "url" => "http://localhost/photo/one.jpeg",
+                        "fileName" => "1.jpeg",
+                        "base64" => "/9j/4AAQSkZJRgABAQAAAAAAAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkI\nCQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/wAALCAABAAEBAREA/8QAFAABAAAAAAAA\nAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==\n",
+                    },
+                ]
+            };
+            return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
+        }
     } elsif ($req->uri =~ /getHighwaysTicketUpdates/) {
         is $req->method, 'GET', "Correct method used";
 
@@ -267,4 +281,29 @@ subtest "GET Service Request Updates" => sub {
     ];
 };
 
+subtest "POST Service Request Update" => sub {
+    set_fixed_time('2023-05-01T12:00:00Z');
+
+    my $res = $surrey_endpoint->run_test_request(
+        POST => '/servicerequestupdates.json',
+        jurisdiction_id => 'surrey_boomi',
+        api_key => 'api-key',
+        updated_datetime => '2023-05-02T12:00:00Z',
+        service_request_id => 'Zendesk_123456',
+        status => 'OPEN',
+        description => 'This is an update',
+        last_name => "Smith",
+        first_name => "John",
+        update_id => '10000000',
+    );
+    is $res->code, 200;
+    is_deeply decode_json($res->content),
+        [ {
+            'update_id' => "Zendesk_123456_6f0922d6",
+        } ], 'correct json returned';
+
+    restore_time();
+};
+
+
 done_testing;

From fb41831cf983e03c923747a1702b7abe16786cc0 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Thu, 18 Jul 2024 11:40:30 +0100
Subject: [PATCH 10/19] [Surrey] Pass town name to Boomi

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 3 ++-
 t/open311/endpoint/surrey_boomi.t             | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 611c3901c..4480e1f32 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -91,6 +91,7 @@ sub post_service_request {
             northing => $args->{attributes}->{northing},
             usrn => $args->{attributes}->{USRN},
             streetName => $args->{attributes}->{ROADNAME},
+            town => $args->{attributes}->{POSTTOWN},
         },
         requester => {
             fullName => $args->{first_name} . " " . $args->{last_name},
@@ -99,7 +100,7 @@ sub post_service_request {
         },
     };
 
-    foreach (qw/group category easting northing title description phone USRN ROADNAME/) {
+    foreach (qw/group category easting northing title description phone USRN ROADNAME POSTTOWN/) {
         if (defined $args->{attributes}->{$_}) {
             delete $args->{attributes}->{$_};
         }
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index cc8442701..860d4b001 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -40,6 +40,7 @@ $lwp->mock(request => sub {
                     "easting" => "1",
                     "usrn" => "31200342",
                     "streetName" => "Cockshot Hill",
+                    "town" => "REIGATE",
                 },
                 "subject" => "Pot hole on road",
                 "description" => "Big hole in the road",
@@ -244,6 +245,7 @@ subtest "POST report" => sub {
         'attribute[RM1]' => "RM1B",
         'attribute[USRN]' => "31200342",
         'attribute[ROADNAME]' => "Cockshot Hill",
+        'attribute[POSTTOWN]' => "REIGATE",
         'attribute[Q7]' => "T1",
         'attribute[Q7]' => "T3",
         'attribute[complexfield]' => "{\"description\":\"A complex field\", \"value\":\"A value\"}",

From b3b0bd847a1b65e8d04f6835407d267ee001d6c6 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Wed, 24 Jul 2024 08:56:13 +0100
Subject: [PATCH 11/19] [Surrey] Upload all update photos to Boomi

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 70 ++++++++++---------
 t/open311/endpoint/surrey_boomi.t             | 67 +++++++++++++++---
 2 files changed, 93 insertions(+), 44 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 4480e1f32..73f0ed4a6 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -129,33 +129,7 @@ sub post_service_request {
     }
     $ticket->{customFields} = \@custom_fields;
 
-    my @attachments;
-
-    my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
-
-    for my $photo (@{ $args->{media_url} }) {
-        my $photo_response = $ua->get($photo);
-        unless ( $photo_response->is_success) {
-            $self->logger->error("Failed to retrieve photo from $photo\n");
-            die "Failed to retrieve photo from $photo";
-        }
-
-        push @attachments, {
-            fileName => $photo_response->filename,
-            url => $photo,
-            base64 => encode_base64($photo_response->content),
-        };
-    }
-
-    if (@{$args->{uploads}}) {
-        foreach (@{$args->{uploads}}) {
-            push @attachments, {
-                fileName => $_->filename,
-                base64 => encode_base64(path($_)->slurp),
-            };
-        }
-    }
-    $ticket->{attachments} = \@attachments if @attachments;
+    $self->_add_attachments($args, $ticket);
 
     my $service_request_id = $self->boomi->upsertHighwaysTicket($ticket);
 
@@ -202,11 +176,6 @@ sub post_service_request_update {
 
     die "Args must be a hashref" unless ref $args eq 'HASH';
 
-    if ($args->{media_url}->[0]) {
-        $args->{description} .= "\n\n[ This update contains a photo, see: " . $args->{media_url}->[0] . " ]";
-    }
-
-
     $self->logger->info("[Boomi] Creating update");
 
     my ($system, $id) = split('_', $args->{service_request_id});
@@ -219,9 +188,13 @@ sub post_service_request_update {
         ],
     };
 
+    $self->_add_attachments($args, $ticket);
+
     # we don't get back a unique ID from Boomi, so calculate one ourselves
     # XXX is this going to be mirrored back next time we fetch updates?
-    my $update_id = $args->{service_request_id} . "_" . substr(md5_hex($id . $args->{description}), 0, 8);
+    my @parts = map { $args->{$_} } qw/service_request_id description update_id updated_datetime/;
+    my $hash = substr(md5_hex(join('', @parts)), 0, 8);
+    my $update_id = $args->{service_request_id} . "_" . $hash;
 
     my $service_request_id = $self->boomi->upsertHighwaysTicket($ticket);
 
@@ -232,6 +205,37 @@ sub post_service_request_update {
     );
 }
 
+sub _add_attachments {
+    my ($self, $args, $ticket) = @_;
+
+    my @attachments;
+
+    my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
+
+    for my $photo (@{ $args->{media_url} }) {
+        my $photo_response = $ua->get($photo);
+        unless ( $photo_response->is_success) {
+            $self->logger->error("Failed to retrieve photo from $photo\n");
+            die "Failed to retrieve photo from $photo";
+        }
+
+        push @attachments, {
+            fileName => $photo_response->filename,
+            url => $photo,
+            base64 => encode_base64($photo_response->content),
+        };
+    }
+
+    if (@{$args->{uploads}}) {
+        foreach (@{$args->{uploads}}) {
+            push @attachments, {
+                fileName => $_->filename,
+                base64 => encode_base64(path($_)->slurp),
+            };
+        }
+    }
 
+    $ticket->{attachments} = \@attachments if @attachments;
+}
 
 1;
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 860d4b001..1acad4353 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -21,16 +21,36 @@ $lwp->mock(request => sub {
         my $content = decode_json($req->content);
         if ($content && $content->{ticketId}) {
             # it's an update to an existing ticket
-            is_deeply $content, {
-                "comments" => [
-                    {
-                        "body" => "This is an update"
-                    }
-                ],
-                "ticketId" => "123456",
-                "integrationId" => "Integration.1"
-            };
-            return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
+            if ($content->{ticketId} eq '123457') {
+                is_deeply $content, {
+                    "comments" => [
+                        {
+                            "body" => "This is an update with a photo"
+                        }
+                    ],
+                    "ticketId" => "123457",
+                    "integrationId" => "Integration.1",
+                    "attachments" => [
+                        {
+                            "url" => "http://localhost/photo/one.jpeg",
+                            "fileName" => "1.jpeg",
+                            "base64" => "/9j/4AAQSkZJRgABAQAAAAAAAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkI\nCQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/wAALCAABAAEBAREA/8QAFAABAAAAAAAA\nAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==\n",
+                        },
+                    ]
+                };
+                return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 123457 }}));
+            } else {
+                is_deeply $content, {
+                    "comments" => [
+                        {
+                            "body" => "This is an update"
+                        }
+                    ],
+                    "ticketId" => "123456",
+                    "integrationId" => "Integration.1"
+                };
+                return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 123456 }}));
+            }
         } else {
             is_deeply $content, {
                 "location" => {
@@ -301,7 +321,32 @@ subtest "POST Service Request Update" => sub {
     is $res->code, 200;
     is_deeply decode_json($res->content),
         [ {
-            'update_id' => "Zendesk_123456_6f0922d6",
+            'update_id' => "Zendesk_123456_ac95b36b",
+        } ], 'correct json returned';
+
+    restore_time();
+};
+
+subtest "POST Service Request Update with photo" => sub {
+    set_fixed_time('2023-05-01T12:00:00Z');
+
+    my $res = $surrey_endpoint->run_test_request(
+        POST => '/servicerequestupdates.json',
+        jurisdiction_id => 'surrey_boomi',
+        api_key => 'api-key',
+        updated_datetime => '2023-05-02T12:42:00Z',
+        service_request_id => 'Zendesk_123457',
+        status => 'OPEN',
+        description => 'This is an update with a photo',
+        last_name => "Smith",
+        first_name => "John",
+        update_id => '10000001',
+        media_url => ['http://localhost/photo/one.jpeg'],
+    );
+    is $res->code, 200;
+    is_deeply decode_json($res->content),
+        [ {
+            'update_id' => "Zendesk_123457_050d1cc8",
         } ], 'correct json returned';
 
     restore_time();

From 1acbaef32e3713ffff083335c50ae88b138202d9 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Thu, 25 Jul 2024 13:35:58 +0100
Subject: [PATCH 12/19] [Surrey] Enable report fetching from Boomi

---
 perllib/Integrations/Surrey/Boomi.pm          |  27 +++++
 perllib/Open311/Endpoint/Integration/Boomi.pm |  73 ++++++++++++
 t/open311/endpoint/surrey_boomi.t             | 106 +++++++++++++++++-
 t/open311/endpoint/surrey_boomi.yml           |   3 +
 4 files changed, 207 insertions(+), 2 deletions(-)

diff --git a/perllib/Integrations/Surrey/Boomi.pm b/perllib/Integrations/Surrey/Boomi.pm
index 33870c7dd..66f168ba6 100644
--- a/perllib/Integrations/Surrey/Boomi.pm
+++ b/perllib/Integrations/Surrey/Boomi.pm
@@ -101,6 +101,33 @@ sub getHighwaysTicketUpdates {
     return $resp->{results} || [];
 }
 
+=head2 getNewHighwaysTickets
+
+Get new tickets from the Surrey Boomi system.
+
+=cut
+
+sub getNewHighwaysTickets {
+    my ($self, $integration_id, $start, $end) = @_;
+
+    my $resp = $self->get('getNewHighwaysTickets', {
+        integration_id => $integration_id,
+        from => format_datetime($start),
+        to => format_datetime($end),
+    });
+
+    if (my $errors = $resp->{errors}) {
+        $self->logger->error("[Boomi] Error fetching new tickets:");
+        $self->logger->error($_->{error} . ": " . $_->{details}) for @$errors;
+        die;
+    }
+    if (my $warnings = $resp->{warnings}) {
+        $self->logger->warn("[Boomi] Warnings when fetching new tickets:");
+        $self->logger->warn($_->{warning} . ": " . $_->{details}) for @$warnings;
+    }
+
+    return $resp->{results} || [];
+}
 
 =head2 post
 
diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 73f0ed4a6..442df76ac 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -10,6 +10,7 @@ use POSIX qw(strftime);
 use MIME::Base64 qw(encode_base64);
 use Open311::Endpoint::Service::UKCouncil::Boomi;
 use Open311::Endpoint::Service::Request::Update::mySociety;
+use Open311::Endpoint::Service::Request::ExtendedStatus;
 use Integrations::Surrey::Boomi;
 use JSON::MaybeXS;
 use DateTime::Format::W3CDTF;
@@ -18,6 +19,15 @@ use Try::Tiny;
 use LWP::UserAgent;
 use Digest::MD5 qw(md5_hex);
 
+has '+request_class' => (
+    is => 'ro',
+    default => 'Open311::Endpoint::Service::Request::ExtendedStatus',
+);
+
+sub service_request_content {
+    '/open311/service_request_extended'
+}
+
 has jurisdiction_id => (
     is => 'ro',
     required => 1,
@@ -138,6 +148,69 @@ sub post_service_request {
     )
 }
 
+sub get_service_requests {
+    my ($self, $args) = @_;
+
+    my $integration_ids = $self->endpoint_config->{integration_ids}->{getNewHighwaysTickets};
+    return () unless $integration_ids;
+    $integration_ids = [ $integration_ids ] unless ref $integration_ids eq 'ARRAY';
+
+    my @requests;
+    foreach (@$integration_ids) {
+        push @requests, $self->_get_service_requests_for_integration_id($_, $args);
+    }
+    return @requests;
+}
+
+
+sub _get_service_requests_for_integration_id {
+    my ($self, $integration_id, $args) = @_;
+
+    my $start = DateTime::Format::W3CDTF->parse_datetime($args->{start_date});
+    my $end = DateTime::Format::W3CDTF->parse_datetime($args->{end_date});
+
+    my $results = $self->boomi->getNewHighwaysTickets($integration_id, $start, $end);
+
+    my @requests;
+    for my $result (@$results) {
+        my ($id, $loggedDate, $e, $n) = do {
+            if (my $enq = $result->{confirmEnquiry}) {
+                ($enq->{enquiryNumber}, $enq->{loggedDate}, $enq->{easting}, $enq->{northing});
+            } else {
+                my $job = $result->{confirmJob};
+                ("JOB_" . $job->{jobNumber}, $job->{entryDate}, $job->{easting}, $job->{northing});
+            }
+        };
+        $loggedDate = DateTime::Format::W3CDTF->parse_datetime($loggedDate);
+
+        # if any of these is missing then ignore this record.
+        next unless $id && $loggedDate && $e && $n;
+
+        my $status = lc $result->{fmsReport}->{status}->{state};
+        $status =~ s/ /_/g;
+
+        my $args = {
+            service_request_id => "Zendesk_$id",
+            status => $status,
+            description => $result->{fmsReport}->{categorisation}->{subCategory} . " problem",
+            title => $result->{fmsReport}->{categorisation}->{subCategory} . " problem",
+            requested_datetime => $loggedDate,
+            updated_datetime => $loggedDate,
+            service => Open311::Endpoint::Service->new(
+                service_name => $result->{fmsReport}->{categorisation}->{subCategory},
+                service_code => "foobar",
+            ),
+            service_notice => $result->{fmsReport}->{categorisation}->{category},
+            latlong => [ $n, $e ],
+        };
+        my $service_request = $self->new_request(%$args);
+
+        push @requests, $service_request;
+    }
+
+    return @requests;
+}
+
 
 
 sub get_service_request_updates {
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 1acad4353..e99c57463 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -136,8 +136,6 @@ $lwp->mock(request => sub {
         }
     } elsif ($req->uri =~ /getHighwaysTicketUpdates/) {
         is $req->method, 'GET', "Correct method used";
-
-
         return HTTP::Response->new(200, 'OK', [], encode_json({
             "executionId" => "execution-7701f16b-036c-4e6e-8e14-998f81f5b6b8-2024.06.27",
             "results" => [
@@ -177,6 +175,70 @@ $lwp->mock(request => sub {
                 },
             ]
         }));
+    } elsif ($req->uri =~ /getNewHighwaysTickets/) {
+        is $req->method, 'GET', "Correct method used";
+        my %query = $req->uri->query_form;
+        if ($query{integration_id} eq 'Integration.3') { # enquiries
+            return HTTP::Response->new(200, 'OK', [], encode_json({
+                "executionId" => "execution-7701f16b-036c-4e6e-8e14-998f81f5b6b8-2024.06.27",
+                "results" => [
+                    {
+                        "confirmEnquiry" => {
+                            "enquiryNumber" => 136416,
+                            "loggedDate" => "2024-07-26T08:42:52.000Z",
+                            "statusCode" => "6000",
+                            "subject" => {
+                                "name" => "Overgrown vegetation",
+                                "code" => "A006",
+                                "serviceCode" => "AR01"
+                            },
+                            "easting" => 507323,
+                            "northing" => 164194
+                        },
+                        "fmsReport" => {
+                            "status" => {
+                                "state" => "Open",
+                                "label" => "Allocated to a Highways Officer"
+                            },
+                            "categorisation" => {
+                                "category" => "Trees and roots",
+                                "subCategory" => "Other tree or roots issue"
+                            }
+                        }
+                    },
+                ]
+            }));
+        } else { # jobs
+            return HTTP::Response->new(200, 'OK', [], encode_json({
+                "executionId" => "execution-7701f16b-036c-4e6e-8e14-998f81f5b6b8-2024.06.27",
+                "results" => [
+                    {
+                        "confirmJob" => {
+                            "entryDate" => "2024-07-26T07:46:33.000Z",
+                            "jobNumber" => 569081,
+                            "statusCode" => "2000",
+                            "priorityCode" => "HW09",
+                            "jobType" => {
+                                "code" => "HW18",
+                                "name" => "HW:Grit Bins-Damaged"
+                            },
+                            "easting" => 522869,
+                            "northing" => 162735
+                        },
+                        "fmsReport" => {
+                            "status" => {
+                                "state" => "Action scheduled",
+                                "label" => "Inspection scheduled"
+                            },
+                            "categorisation" => {
+                                "category" => "Gritting and Grit Bins",
+                                "subCategory" => "Damaged grit bin"
+                            }
+                        }
+                    },
+                ]
+            }));
+        }
     } elsif ($req->uri eq 'http://localhost/photo/one.jpeg') {
         my $image_data = path(__FILE__)->sibling('files')->child('test_image.jpg')->slurp;
         my $response = HTTP::Response->new(200, 'OK', []);
@@ -352,5 +414,45 @@ subtest "POST Service Request Update with photo" => sub {
     restore_time();
 };
 
+subtest "GET Service Requests" => sub {
+    my $res = $surrey_endpoint->run_test_request(
+        GET => '/requests.json?jurisdiction_id=surrey_boomi&api_key=api-key&start_date=2024-05-01T09:00:00Z&end_date=2024-05-01T10:00:00Z',
+    );
+    is $res->code, 200;
+    is_deeply decode_json($res->content), [
+       {
+            'zipcode' => '',
+            'service_name' => 'Other tree or roots issue',
+            'address' => '',
+            'status' => 'open',
+            'long' => 507323,
+            'updated_datetime' => '2024-07-26T08:42:52Z',
+            'service_code' => 'foobar',
+            'lat' => 164194,
+            'requested_datetime' => '2024-07-26T08:42:52Z',
+            'media_url' => '',
+            'service_request_id' => 'Zendesk_136416',
+            'description' => 'Other tree or roots issue problem',
+            'service_notice' => 'Trees and roots',
+            'address_id' => ''
+        },
+        {
+            'requested_datetime' => '2024-07-26T07:46:33Z',
+            'zipcode' => '',
+            'service_name' => 'Damaged grit bin',
+            'lat' => 162735,
+            'address' => '',
+            'service_notice' => 'Gritting and Grit Bins',
+            'description' => 'Damaged grit bin problem',
+            'status' => 'action_scheduled',
+            'service_code' => 'foobar',
+            'media_url' => '',
+            'long' => 522869,
+            'updated_datetime' => '2024-07-26T07:46:33Z',
+            'address_id' => '',
+            'service_request_id' => 'Zendesk_JOB_569081'
+        }
+    ];
+};
 
 done_testing;
diff --git a/t/open311/endpoint/surrey_boomi.yml b/t/open311/endpoint/surrey_boomi.yml
index d5920df27..33d6e7100 100644
--- a/t/open311/endpoint/surrey_boomi.yml
+++ b/t/open311/endpoint/surrey_boomi.yml
@@ -5,3 +5,6 @@ api_url: http://localhost/ws/simple/
 integration_ids:
   upsertHighwaysTicket: "Integration.1"
   getHighwaysTicketUpdates: "Integration.2"
+  getNewHighwaysTickets:
+    - "Integration.3"
+    - "Integration.4"

From bf35b3f1319258857f72d45f5055bf22f62c89fe Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Tue, 30 Jul 2024 09:35:30 +0100
Subject: [PATCH 13/19] [Surrey] Pass Open311 service_code to Boomi

---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 4 ++++
 t/open311/endpoint/surrey_boomi.t             | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 442df76ac..d9fed98ba 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -87,6 +87,10 @@ sub post_service_request {
             id => 'subCategory',
             values => [ $args->{attributes}->{category} ],
         },
+        {
+            id => 'service_code',
+            values => [ $args->{service_code} ],
+        },
     );
 
     my $ticket = {
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index e99c57463..acdd82d59 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -84,6 +84,12 @@ $lwp->mock(request => sub {
                         ],
                         "id" => "subCategory"
                     },
+                    {
+                        "values" => [
+                            "potholes"
+                        ],
+                        "id" => "service_code"
+                    },
                     {
                         "id" => "Q7",
                         "values" => [

From c86cf3360b0ab03133e533cb97b32d759d42bb0d Mon Sep 17 00:00:00 2001
From: Moray Jones <moray@mysociety.org>
Date: Mon, 5 Aug 2024 18:15:31 +0100
Subject: [PATCH 14/19] [Surrey] Expect serviceId from integration

serviceId will be received from new version of
Boomi integration so is not hard-coded

https://3.basecamp.com/4020879/buckets/36546415/todos/7648793252#__recording_7674131654
https://mysociety.slack.com/archives/C06SKSH5KQT/p1722624461300599?thread_ts=1722616474.424329&cid=C06SKSH5KQT
---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 10 +++++-----
 t/open311/endpoint/surrey_boomi.t             | 16 ++++++++--------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index d9fed98ba..9df053d33 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -196,15 +196,15 @@ sub _get_service_requests_for_integration_id {
         my $args = {
             service_request_id => "Zendesk_$id",
             status => $status,
-            description => $result->{fmsReport}->{categorisation}->{subCategory} . " problem",
-            title => $result->{fmsReport}->{categorisation}->{subCategory} . " problem",
+            description => $result->{fmsReport}->{title},
+            title => $result->{fmsReport}->{title},
             requested_datetime => $loggedDate,
             updated_datetime => $loggedDate,
             service => Open311::Endpoint::Service->new(
-                service_name => $result->{fmsReport}->{categorisation}->{subCategory},
-                service_code => "foobar",
+                service_name => $result->{fmsReport}->{title},
+                service_code => $result->{fmsReport}->{categorisation}->{serviceId},
             ),
-            service_notice => $result->{fmsReport}->{categorisation}->{category},
+            service_notice => $result->{fmsReport}->{title},
             latlong => [ $n, $e ],
         };
         my $service_request = $self->new_request(%$args);
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index acdd82d59..2ca378917 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -202,13 +202,13 @@ $lwp->mock(request => sub {
                             "northing" => 164194
                         },
                         "fmsReport" => {
+                            "title" => "Other tree or roots issue problem",
                             "status" => {
                                 "state" => "Open",
                                 "label" => "Allocated to a Highways Officer"
                             },
                             "categorisation" => {
-                                "category" => "Trees and roots",
-                                "subCategory" => "Other tree or roots issue"
+                                "serviceId" => 'foobar',
                             }
                         }
                     },
@@ -232,13 +232,13 @@ $lwp->mock(request => sub {
                             "northing" => 162735
                         },
                         "fmsReport" => {
+                            "title" => "Damaged grit bin",
                             "status" => {
                                 "state" => "Action scheduled",
                                 "label" => "Inspection scheduled"
                             },
                             "categorisation" => {
-                                "category" => "Gritting and Grit Bins",
-                                "subCategory" => "Damaged grit bin"
+                                "serviceId" => 'foobar',
                             }
                         }
                     },
@@ -428,7 +428,7 @@ subtest "GET Service Requests" => sub {
     is_deeply decode_json($res->content), [
        {
             'zipcode' => '',
-            'service_name' => 'Other tree or roots issue',
+            'service_name' => 'Other tree or roots issue problem',
             'address' => '',
             'status' => 'open',
             'long' => 507323,
@@ -439,7 +439,7 @@ subtest "GET Service Requests" => sub {
             'media_url' => '',
             'service_request_id' => 'Zendesk_136416',
             'description' => 'Other tree or roots issue problem',
-            'service_notice' => 'Trees and roots',
+            'service_notice' => 'Other tree or roots issue problem',
             'address_id' => ''
         },
         {
@@ -448,8 +448,8 @@ subtest "GET Service Requests" => sub {
             'service_name' => 'Damaged grit bin',
             'lat' => 162735,
             'address' => '',
-            'service_notice' => 'Gritting and Grit Bins',
-            'description' => 'Damaged grit bin problem',
+            'service_notice' => 'Damaged grit bin',
+            'description' => 'Damaged grit bin',
             'status' => 'action_scheduled',
             'service_code' => 'foobar',
             'media_url' => '',

From 225021faa44b898c4de141c8413f6f704be493a8 Mon Sep 17 00:00:00 2001
From: Moray Jones <moray@mysociety.org>
Date: Tue, 6 Aug 2024 16:43:53 +0100
Subject: [PATCH 15/19] [Surrey] Adds externalId for jobs and reports

Surrey will be sending an externalId specifically to
be used for imported jobs and reports

https://3.basecamp.com/4020879/buckets/36546415/todos/7672617616#__recording_7683887161
https://mysociety.slack.com/archives/C06SKSH5KQT/p1722948392215939
---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 8 +++++---
 t/open311/endpoint/surrey_boomi.t             | 4 ++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index 9df053d33..b547216a7 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -177,12 +177,14 @@ sub _get_service_requests_for_integration_id {
 
     my @requests;
     for my $result (@$results) {
-        my ($id, $loggedDate, $e, $n) = do {
+        my $id = $result->{fmsReport}->{externalId};
+        my ($loggedDate, $e, $n) = do {
             if (my $enq = $result->{confirmEnquiry}) {
-                ($enq->{enquiryNumber}, $enq->{loggedDate}, $enq->{easting}, $enq->{northing});
+                ($enq->{loggedDate}, $enq->{easting}, $enq->{northing});
             } else {
+                $id = "JOB_" . $id;
                 my $job = $result->{confirmJob};
-                ("JOB_" . $job->{jobNumber}, $job->{entryDate}, $job->{easting}, $job->{northing});
+                ($job->{entryDate}, $job->{easting}, $job->{northing});
             }
         };
         $loggedDate = DateTime::Format::W3CDTF->parse_datetime($loggedDate);
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 2ca378917..ff949feb2 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -190,7 +190,6 @@ $lwp->mock(request => sub {
                 "results" => [
                     {
                         "confirmEnquiry" => {
-                            "enquiryNumber" => 136416,
                             "loggedDate" => "2024-07-26T08:42:52.000Z",
                             "statusCode" => "6000",
                             "subject" => {
@@ -203,6 +202,7 @@ $lwp->mock(request => sub {
                         },
                         "fmsReport" => {
                             "title" => "Other tree or roots issue problem",
+                            "externalId" => 136416,
                             "status" => {
                                 "state" => "Open",
                                 "label" => "Allocated to a Highways Officer"
@@ -221,7 +221,6 @@ $lwp->mock(request => sub {
                     {
                         "confirmJob" => {
                             "entryDate" => "2024-07-26T07:46:33.000Z",
-                            "jobNumber" => 569081,
                             "statusCode" => "2000",
                             "priorityCode" => "HW09",
                             "jobType" => {
@@ -233,6 +232,7 @@ $lwp->mock(request => sub {
                         },
                         "fmsReport" => {
                             "title" => "Damaged grit bin",
+                            "externalId" => 569081,
                             "status" => {
                                 "state" => "Action scheduled",
                                 "label" => "Inspection scheduled"

From ec2f017d031d4943fd5cb36e06ba8f3e7c78b323 Mon Sep 17 00:00:00 2001
From: Moray Jones <moray@mysociety.org>
Date: Wed, 14 Aug 2024 15:50:20 +0100
Subject: [PATCH 16/19] [Surrey][Boomi] Fetch job updates

Updates for Jobs are a separate integration id to
updates for Enquiries. Allow array of integration ids
and receive Jobs updates and convert them to an FMS update.

https://3.basecamp.com/4020879/buckets/36546415/todos/7696104321
---
 perllib/Integrations/Surrey/Boomi.pm          |   4 +-
 perllib/Open311/Endpoint/Integration/Boomi.pm |  31 ++++--
 t/open311/endpoint/surrey_boomi.t             | 100 ++++++++++++------
 t/open311/endpoint/surrey_boomi.yml           |   4 +-
 4 files changed, 97 insertions(+), 42 deletions(-)

diff --git a/perllib/Integrations/Surrey/Boomi.pm b/perllib/Integrations/Surrey/Boomi.pm
index 66f168ba6..a9c772cde 100644
--- a/perllib/Integrations/Surrey/Boomi.pm
+++ b/perllib/Integrations/Surrey/Boomi.pm
@@ -80,12 +80,12 @@ Returns the ID of the created or updated ticket.
 =cut
 
 sub getHighwaysTicketUpdates {
-    my ($self, $start, $end) = @_;
+    my ($self, $integration_id, $start, $end) = @_;
 
     my $resp = $self->get('getHighwaysTicketUpdates', {
         from => format_datetime($start),
         to => format_datetime($end),
-        integration_id => $self->config->{integration_ids}->{getHighwaysTicketUpdates},
+        integration_id => $integration_id,
     });
 
     if (my $errors = $resp->{errors}) {
diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index b547216a7..dbfdf4119 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -217,31 +217,50 @@ sub _get_service_requests_for_integration_id {
     return @requests;
 }
 
-
-
 sub get_service_request_updates {
     my ($self, $args) = @_;
 
+    my $integration_ids = $self->endpoint_config->{integration_ids}->{getHighwaysTicketUpdates};
+    return () unless $integration_ids;
+    $integration_ids = [ $integration_ids ] unless ref $integration_ids eq 'ARRAY';
+
+    my @updates;
+    foreach (@$integration_ids) {
+        push @updates, $self->_get_service_request_updates_for_integration_id($_, $args);
+    }
+
+    return @updates;
+}
+
+sub _get_service_request_updates_for_integration_id {
+    my ($self, $integration_id, $args) = @_;
+
     my $start = DateTime::Format::W3CDTF->parse_datetime($args->{start_date});
     my $end = DateTime::Format::W3CDTF->parse_datetime($args->{end_date});
 
-    my $results = $self->boomi->getHighwaysTicketUpdates($start, $end);
+    my $results = $self->boomi->getHighwaysTicketUpdates($integration_id, $start, $end);
     my $w3c = DateTime::Format::W3CDTF->new;
 
     my @updates;
 
     for my $update (@$results) {
-        my $log = $update->{confirmEnquiryStatusLog};
+        my $enquiry_update = $update->{confirmEnquiryStatusLog} ? 1 : 0;
+        my $log = $enquiry_update ? $update->{confirmEnquiryStatusLog} : $update->{confirmJobStatusLog};
         my $fms = $update->{fmsReport};
 
-        my $id = $log->{enquiry}->{externalSystemReference} . "_" . $log->{logNumber};
+        my $id = $enquiry_update
+            ? $log->{enquiry}->{externalSystemReference} . "_" . $log->{logNumber}
+            : $log->{job}->{jobNumber} . "_" . $log->{logNumber};
+        my $service_request_id = $enquiry_update
+            ? "Zendesk_" . $log->{enquiry}->{externalSystemReference}
+            : "Zendesk_JOB_" . $log->{job}->{jobNumber};
         my $status = lc $fms->{status}->{state};
         $status =~ s/ /_/g;
 
         push @updates, Open311::Endpoint::Service::Request::Update::mySociety->new(
             status => $status,
             update_id => $id,
-            service_request_id => "Zendesk_" . $log->{enquiry}->{externalSystemReference},
+            service_request_id => $service_request_id,
             description => $fms->{status}->{label},
             updated_datetime => $w3c->parse_datetime( $log->{loggedDate} )->truncate( to => 'second' ),
         );
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index ff949feb2..073838fd0 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -141,46 +141,72 @@ $lwp->mock(request => sub {
             return HTTP::Response->new(200, 'OK', [], encode_json({"ticket" => { system => "Zendesk", id => 1234 }}));
         }
     } elsif ($req->uri =~ /getHighwaysTicketUpdates/) {
+        my %query = $req->uri->query_form;
         is $req->method, 'GET', "Correct method used";
-        return HTTP::Response->new(200, 'OK', [], encode_json({
-            "executionId" => "execution-7701f16b-036c-4e6e-8e14-998f81f5b6b8-2024.06.27",
-            "results" => [
-                {
-                    "confirmEnquiryStatusLog" => {
-                        "loggedDate" => "2024-05-01T09:07:47.000Z",
-                        "logNumber" => 11,
-                        "statusCode" => "5800",
-                        "enquiry" => {
-                            "enquiryNumber" => 129293,
-                            "externalSystemReference" => "2929177"
+        if ($query{integration_id} eq 'Integration.2') {
+            return HTTP::Response->new(200, 'OK', [], encode_json({
+                "executionId" => "execution-7701f16b-036c-4e6e-8e14-998f81f5b6b8-2024.06.27",
+                "results" => [
+                    {
+                        "confirmEnquiryStatusLog" => {
+                            "loggedDate" => "2024-05-01T09:07:47.000Z",
+                            "logNumber" => 11,
+                            "statusCode" => "5800",
+                            "enquiry" => {
+                                "enquiryNumber" => 129293,
+                                "externalSystemReference" => "2929177"
+                            }
+                        },
+                        "fmsReport" => {
+                            "status" => {
+                                "state" => "Closed",
+                                "label" => "Enquiry closed"
+                            }
                         }
                     },
-                    "fmsReport" => {
-                        "status" => {
-                            "state" => "Closed",
-                            "label" => "Enquiry closed"
-                        }
-                    }
-                },
-                {
-                    "confirmEnquiryStatusLog" => {
-                        "loggedDate" => "2024-05-01T09:10:41.000Z",
-                        "logNumber" => 7,
-                        "statusCode" => "3200",
-                        "enquiry" => {
-                            "enquiryNumber" => 132361,
-                            "externalSystemReference" => "2939061"
+                    {
+                        "confirmEnquiryStatusLog" => {
+                            "loggedDate" => "2024-05-01T09:10:41.000Z",
+                            "logNumber" => 7,
+                            "statusCode" => "3200",
+                            "enquiry" => {
+                                "enquiryNumber" => 132361,
+                                "externalSystemReference" => "2939061"
+                            }
+                        },
+                        "fmsReport" => {
+                            "status" => {
+                                "state" => "Action scheduled",
+                                "label" => "Assessed - scheduling a repair within 5 Working Days"
+                            }
                         }
                     },
-                    "fmsReport" => {
-                        "status" => {
-                            "state" => "Action scheduled",
-                            "label" => "Assessed - scheduling a repair within 5 Working Days"
+                ]
+            }));
+        } elsif ($query{integration_id} eq 'Integration.5') {
+            return HTTP::Response->new(200, 'OK', [], encode_json({
+                "executionId" => "execution-7701f16b-036c-4e6e-8e14-998f81f5b6b8-2024.06.27",
+                "results" => [
+                    {
+                        "confirmJobStatusLog" => {
+                            "loggedDate" => "2024-08-09T11:23:10.000Z",
+                            "logNumber" => 2,
+                            "statusCode" => "2000",
+                            "job" => {
+                            "jobNumber" => 569276
+                            }
+                        },
+                        "fmsReport" => {
+                            "externalId" => "569276",
+                            "status" => {
+                                "state" => "Action scheduled",
+                                "label" => "Assessed - scheduling a repair within 5 Working Days"
+                            }
                         }
                     }
-                },
-            ]
-        }));
+                ]
+            }));
+        }
     } elsif ($req->uri =~ /getNewHighwaysTickets/) {
         is $req->method, 'GET', "Correct method used";
         my %query = $req->uri->query_form;
@@ -367,6 +393,14 @@ subtest "GET Service Request Updates" => sub {
           "status" => "action_scheduled",
           "update_id" => "2939061_7",
           "updated_datetime" => "2024-05-01T09:10:41Z",
+       },
+       {
+          "update_id" => "569276_2",
+          "service_request_id" => "Zendesk_JOB_569276",
+          "description" => "Assessed - scheduling a repair within 5 Working Days",
+          "media_url" => "",
+          "updated_datetime" => "2024-08-09T11:23:10Z",
+          "status" => "action_scheduled"
        }
     ];
 };
diff --git a/t/open311/endpoint/surrey_boomi.yml b/t/open311/endpoint/surrey_boomi.yml
index 33d6e7100..60f00899b 100644
--- a/t/open311/endpoint/surrey_boomi.yml
+++ b/t/open311/endpoint/surrey_boomi.yml
@@ -4,7 +4,9 @@ api_url: http://localhost/ws/simple/
 
 integration_ids:
   upsertHighwaysTicket: "Integration.1"
-  getHighwaysTicketUpdates: "Integration.2"
+  getHighwaysTicketUpdates:
+    - "Integration.2"
+    - "Integration.5"
   getNewHighwaysTickets:
     - "Integration.3"
     - "Integration.4"

From a42c6f150814cd0e7d2ccd56d83d37507f2d13b7 Mon Sep 17 00:00:00 2001
From: Moray Jones <moray@mysociety.org>
Date: Wed, 28 Aug 2024 10:29:16 +0100
Subject: [PATCH 17/19] [Surrey] Add description field

https://github.com/mysociety/societyworks/issues/4506
---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 2 +-
 t/open311/endpoint/surrey_boomi.t             | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index dbfdf4119..ddf21809f 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -198,7 +198,7 @@ sub _get_service_requests_for_integration_id {
         my $args = {
             service_request_id => "Zendesk_$id",
             status => $status,
-            description => $result->{fmsReport}->{title},
+            description => $result->{fmsReport}->{description},
             title => $result->{fmsReport}->{title},
             requested_datetime => $loggedDate,
             updated_datetime => $loggedDate,
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 073838fd0..3783f6faa 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -228,6 +228,7 @@ $lwp->mock(request => sub {
                         },
                         "fmsReport" => {
                             "title" => "Other tree or roots issue problem",
+                            "description" => "Roots sticking through pavement",
                             "externalId" => 136416,
                             "status" => {
                                 "state" => "Open",
@@ -258,6 +259,7 @@ $lwp->mock(request => sub {
                         },
                         "fmsReport" => {
                             "title" => "Damaged grit bin",
+                            "description" => "Lid come off grit bin",
                             "externalId" => 569081,
                             "status" => {
                                 "state" => "Action scheduled",
@@ -472,7 +474,7 @@ subtest "GET Service Requests" => sub {
             'requested_datetime' => '2024-07-26T08:42:52Z',
             'media_url' => '',
             'service_request_id' => 'Zendesk_136416',
-            'description' => 'Other tree or roots issue problem',
+            'description' => 'Roots sticking through pavement',
             'service_notice' => 'Other tree or roots issue problem',
             'address_id' => ''
         },
@@ -483,7 +485,7 @@ subtest "GET Service Requests" => sub {
             'lat' => 162735,
             'address' => '',
             'service_notice' => 'Damaged grit bin',
-            'description' => 'Damaged grit bin',
+            'description' => 'Lid come off grit bin',
             'status' => 'action_scheduled',
             'service_code' => 'foobar',
             'media_url' => '',

From 39cd0e185c6aa9f8ff059651a7bb5e3f7e936e34 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Tue, 17 Sep 2024 11:54:51 +0100
Subject: [PATCH 18/19] [Surrey] Increase Boomi timeout to 3 minutes

---
 perllib/Integrations/Surrey/Boomi.pm | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/perllib/Integrations/Surrey/Boomi.pm b/perllib/Integrations/Surrey/Boomi.pm
index a9c772cde..a25307c55 100644
--- a/perllib/Integrations/Surrey/Boomi.pm
+++ b/perllib/Integrations/Surrey/Boomi.pm
@@ -33,7 +33,10 @@ has ua => (
     is  => 'rw',
     default => sub {
         my $self = shift;
-        my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
+        my $ua = LWP::UserAgent->new(
+            agent => "FixMyStreet/open311-adapter",
+            timeout => 3 * 60, # Boomi can take longer than the default 60s when handling large/multiple photos.
+        );
         $ua->ssl_opts(SSL_cipher_list => 'DEFAULT:!DH'); # Disable DH ciphers, server's key is too small apparently
         my $hash = encode_base64($self->config->{username} . ':' . $self->config->{password}, "");
         $ua->default_header('Authorization' => "Basic $hash");

From 805c9d5896a58ae1747daff631f17efb118099e1 Mon Sep 17 00:00:00 2001
From: Dave Arter <davea@mysociety.org>
Date: Tue, 17 Sep 2024 15:27:17 +0100
Subject: [PATCH 19/19] =?UTF-8?q?[Surrey]=20Don=E2=80=99t=20fetch=20&=20up?=
 =?UTF-8?q?load=20photos=20from=20media=5Furl?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Photos should only be uploaded if the photo content was
POSTed from FMS.
---
 perllib/Open311/Endpoint/Integration/Boomi.pm | 8 --------
 t/open311/endpoint/surrey_boomi.t             | 4 ----
 2 files changed, 12 deletions(-)

diff --git a/perllib/Open311/Endpoint/Integration/Boomi.pm b/perllib/Open311/Endpoint/Integration/Boomi.pm
index ddf21809f..a4e91442c 100644
--- a/perllib/Open311/Endpoint/Integration/Boomi.pm
+++ b/perllib/Open311/Endpoint/Integration/Boomi.pm
@@ -311,16 +311,8 @@ sub _add_attachments {
     my $ua = LWP::UserAgent->new(agent => "FixMyStreet/open311-adapter");
 
     for my $photo (@{ $args->{media_url} }) {
-        my $photo_response = $ua->get($photo);
-        unless ( $photo_response->is_success) {
-            $self->logger->error("Failed to retrieve photo from $photo\n");
-            die "Failed to retrieve photo from $photo";
-        }
-
         push @attachments, {
-            fileName => $photo_response->filename,
             url => $photo,
-            base64 => encode_base64($photo_response->content),
         };
     }
 
diff --git a/t/open311/endpoint/surrey_boomi.t b/t/open311/endpoint/surrey_boomi.t
index 3783f6faa..6e5090db7 100644
--- a/t/open311/endpoint/surrey_boomi.t
+++ b/t/open311/endpoint/surrey_boomi.t
@@ -33,8 +33,6 @@ $lwp->mock(request => sub {
                     "attachments" => [
                         {
                             "url" => "http://localhost/photo/one.jpeg",
-                            "fileName" => "1.jpeg",
-                            "base64" => "/9j/4AAQSkZJRgABAQAAAAAAAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkI\nCQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/wAALCAABAAEBAREA/8QAFAABAAAAAAAA\nAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==\n",
                         },
                     ]
                 };
@@ -133,8 +131,6 @@ $lwp->mock(request => sub {
                 "attachments" => [
                     {
                         "url" => "http://localhost/photo/one.jpeg",
-                        "fileName" => "1.jpeg",
-                        "base64" => "/9j/4AAQSkZJRgABAQAAAAAAAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkI\nCQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/wAALCAABAAEBAREA/8QAFAABAAAAAAAA\nAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==\n",
                     },
                 ]
             };