From f08f9899137d553b545f430ca90e966898e1852f Mon Sep 17 00:00:00 2001 From: Oliver Wickham Date: Thu, 29 Dec 2016 15:27:18 +0000 Subject: [PATCH 1/3] Add address lookup demo --- app/config/frontendAppConfig.scala | 3 + app/connectors/AddressLookup.scala | 214 ++++++++++++++++++ app/controllers/AddressLookupController.scala | 34 +++ conf/application.conf | 4 + conf/testOnlyDoNotUseInAppConf.routes | 2 + sbtTestRoutes.sh | 3 + 6 files changed, 260 insertions(+) create mode 100644 app/connectors/AddressLookup.scala create mode 100644 app/controllers/AddressLookupController.scala create mode 100755 sbtTestRoutes.sh diff --git a/app/config/frontendAppConfig.scala b/app/config/frontendAppConfig.scala index 3a60b00..3fab469 100644 --- a/app/config/frontendAppConfig.scala +++ b/app/config/frontendAppConfig.scala @@ -37,6 +37,7 @@ case class FasttrackConfig(url: FasttrackUrl) case class FasttrackUrl(host: String, base: String) case class FasttrackFrontendConfig(blockNewAccountsDate: Option[LocalDateTime], blockApplicationsDate: Option[LocalDateTime]) +case class AddressLookupConfig(url: String) object FasttrackFrontendConfig { def read(blockNewAccountsDate: Option[String], blockApplicationsDate: Option[String]): FasttrackFrontendConfig = { @@ -57,6 +58,7 @@ trait AppConfig { val userManagementConfig: UserManagementConfig val fasttrackConfig: FasttrackConfig val fasttrackFrontendConfig: FasttrackFrontendConfig + val addressLookupConfig: AddressLookupConfig } object FrontendAppConfig extends AppConfig with ServicesConfig { @@ -81,4 +83,5 @@ object FrontendAppConfig extends AppConfig with ServicesConfig { blockNewAccountsDate = configuration.getString("application.blockNewAccountsDate"), blockApplicationsDate = configuration.getString("application.blockApplicationsDate") ) + override val addressLookupConfig = configuration.underlying.as[AddressLookupConfig]("microservice.services.address-lookup") } diff --git a/app/connectors/AddressLookup.scala b/app/connectors/AddressLookup.scala new file mode 100644 index 0000000..152fc08 --- /dev/null +++ b/app/connectors/AddressLookup.scala @@ -0,0 +1,214 @@ +/* + * Copyright 2016 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package connectors.addresslookup + +import java.net.URLEncoder + +import config.CSRHttp +import play.api.libs.json.Json +import uk.gov.hmrc.play.http.HeaderCarrier +import uk.gov.hmrc.play.http._ + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +import java.util.regex.Pattern + +/** + * The following DTOs are taken from https://github.com/hmrc/address-reputation-store. The project has + * not been added as a dependency, as it brings in many transitive dependencies that are not needed, + * as well as data cleansing/ingestion and backward compatibility logic that is not needed for this project. + * If the version 2 api gets deprecated, then these DTOs will have to change. + * There have been some minor changes made to the code to ensure that it compiles and passes scalastyle, + * but there is some copied code that is not idiomatic Scala and should be changed at some point in the future + */ + +case class LocalCustodian(code: Int, name: String) + +object LocalCustodian { + implicit val reads = Json.reads[LocalCustodian] +} + +/** Represents a country as per ISO3166. */ +case class Country( + // ISO3166-1 or ISO3166-2 code, e.g. "GB" or "GB-ENG" (note that "GB" is the official + // code for UK although "UK" is a reserved synonym and may be used instead) + // See https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 + // and https://en.wikipedia.org/wiki/ISO_3166-2:GB + code: String, + // The printable name for the country, e.g. "United Kingdom" + name: String) + +object Country { + implicit val countryReads = Json.reads[Country] +} + +case class Outcode(area: String, district: String) { + override lazy val toString: String = area + district +} + +object Outcode { + implicit val outcodeReads = Json.reads[Outcode] +} + +object Countries { + // note that "GB" is the official ISO code for UK, although "UK" is a reserved synonym and is less confusing + val UK = Country("UK", "United Kingdom") + val GB = Country("GB", "United Kingdom") // special case provided for in ISO-3166 + val GG = Country("GG", "Guernsey") + val IM = Country("IM", "Isle of Man") + val JE = Country("JE", "Jersey") + + val England = Country("GB-ENG", "England") + val Scotland = Country("GB-SCT", "Scotland") + val Wales = Country("GB-WLS", "Wales") + val Cymru = Country("GB-CYM", "Cymru") + val NorthernIreland = Country("GB-NIR", "Northern Ireland") + + private val all = List(UK, GB, GG, IM, JE, England, Scotland, Wales, Cymru, NorthernIreland) + + def find(code: String): Option[Country] = all.find(_.code == code) + + def findByName(name: String): Option[Country] = all.find(_.name == name) + + // TODO this is possibly not good enough - should consult a reference HMG-approved list of countries +} + +/** + * Address typically represents a postal address. + * For UK addresses, 'town' will always be present. + * For non-UK addresses, 'town' may be absent and there may be an extra line instead. + */ +case class Address(lines: List[String], + town: Option[String], + county: Option[String], + postcode: String, + subdivision: Option[Country], + country: Country) { + + import Address._ + + def nonEmptyFields: List[String] = lines ::: town.toList ::: county.toList ::: List(postcode) + + /** Gets a conjoined representation, excluding the country. */ + def printable(separator: String): String = nonEmptyFields.mkString(separator) + + /** Gets a single-line representation, excluding the country. */ + def printable: String = printable(", ") + + def line1: String = if (lines.nonEmpty) lines.head else "" + + def line2: String = if (lines.size > 1) lines(1) else "" + + def line3: String = if (lines.size > 2) lines(2) else "" + + def line4: String = if (lines.size > 3) lines(3) else "" + + def longestLineLength: Int = nonEmptyFields.map(_.length).max + + def truncatedAddress(maxLen: Int = maxLineLength): Address = + Address(lines.map(limit(_, maxLen)), town.map(limit(_, maxLen)), county.map(limit(_, maxLen)), postcode, subdivision, country) + +} + +object Address { + val maxLineLength = 35 + val danglingLetter: Pattern = Pattern.compile(".* [A-Z0-9]$") + implicit val addressFormat = Json.reads[Address] + + private[addresslookup] def limit(str: String, max: Int): String = { + var s = str + while (s.length > max && s.indexOf(", ") > 0) { + s = s.replaceFirst(", ", ",") + } + if (s.length > max) { + s = s.substring(0, max).trim + if (Address.danglingLetter.matcher(s).matches()) { + s = s.substring(0, s.length - 2) + } + s + } + else { s } + } +} + +case class LatLong(lat: Double, long: Double) { + def toLocation: String = lat.toString + "," + long.toString +} + +case class AddressRecord( + id: String, + uprn: Option[Long], + address: Address, + // ISO639-1 code, e.g. 'en' for English + // see https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes + language: String, + localCustodian: Option[LocalCustodian], + blpuState: Option[String], + logicalState: Option[String], + streetClassification: Option[String]) { + + import Address._ + def truncatedAddress(maxLen: Int = Address.maxLineLength): AddressRecord = + if (address.longestLineLength <= maxLen) { this } + else { copy(address = address.truncatedAddress(maxLen)) } + + def withoutMetadata: AddressRecord = copy(blpuState = None, logicalState = None, streetClassification = None) + + def geoLocation: LatLong = ??? +} + +object AddressRecord { + implicit val addressRecordFormat = Json.reads[AddressRecord] +} + +trait AddressLookupClient{ + + def addressLookupEndpoint: String + val http: CSRHttp + + private def url = s"$addressLookupEndpoint/v2/uk/addresses" + + def findById(id: String)(implicit hc: HeaderCarrier): Future[Option[AddressRecord]] = { + assert(id.length <= 100, "Postcodes cannot be larger than 100 characters") + val uq = "/" + enc(id) + http.GET[Option[AddressRecord]](url + uq).recover { + case _: NotFoundException => None + } + } + + def findByUprn(uprn: Long)(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { + val uq = "?uprn=" + uprn.toString + http.GET[List[AddressRecord]](url + uq) + } + + def findByPostcode(postcode: String, filter: Option[String])(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { + val safePostcode = postcode.replaceAll("[^A-Za-z0-9]", "") + assert(safePostcode.length <= 10, "Postcodes cannot be larger than 10 characters") + val pq = "?postcode=" + enc(safePostcode) + val fq = filter.map(fi => "&filter=" + enc(fi)).getOrElse("") + http.GET[List[AddressRecord]](url + pq + fq) + } + + def findByOutcode(outcode: Outcode, filter: String)(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { + val pq = "?outcode=" + outcode.toString + val fq = "&filter=" + enc(filter) + http.GET[List[AddressRecord]](url + pq + fq) + } + + private def enc(s: String) = URLEncoder.encode(s, "UTF-8") +} diff --git a/app/controllers/AddressLookupController.scala b/app/controllers/AddressLookupController.scala new file mode 100644 index 0000000..5de8fdf --- /dev/null +++ b/app/controllers/AddressLookupController.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2016 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import config.CSRHttp +import connectors.addresslookup.AddressLookupClient +import play.api.mvc.Action + +object AddressLookupController extends AddressLookupController { + val http = CSRHttp + val addressLookupEndpoint = config.FrontendAppConfig.addressLookupConfig.url +} + +trait AddressLookupController extends BaseController with AddressLookupClient { + // TODO Add permissions into this once the feature is ready to be moved from test routes + def addressLookup(postcode: String) = Action.async { implicit request => + val decoded = java.net.URLDecoder.decode(postcode, "UTF8") + findByPostcode(decoded, None).map(r => Ok(r.toString)) + } +} diff --git a/conf/application.conf b/conf/application.conf index 4536636..e22707c 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -107,5 +107,9 @@ microservice { host = "http://localhost:8094" } } + + address-lookup { + url = "http://localhost:9022" + } } } \ No newline at end of file diff --git a/conf/testOnlyDoNotUseInAppConf.routes b/conf/testOnlyDoNotUseInAppConf.routes index b98c783..64dfdbb 100644 --- a/conf/testOnlyDoNotUseInAppConf.routes +++ b/conf/testOnlyDoNotUseInAppConf.routes @@ -11,3 +11,5 @@ # Add all the application routes to the prod.routes file -> / prod.Routes + +GET /address/search/:postcode controllers.AddressLookupController.addressLookup(postcode: String) diff --git a/sbtTestRoutes.sh b/sbtTestRoutes.sh new file mode 100755 index 0000000..5e85ed1 --- /dev/null +++ b/sbtTestRoutes.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +sbt -J-Dapplication.router=testOnlyDoNotUseInAppConf.Routes -Dhttp.port=9283 -Dplay.filters.headers.contentSecurityPolicy='www.google-analytics.com' From e2bdd42f16025cb9f826063f15f4a3a2f87522e6 Mon Sep 17 00:00:00 2001 From: Oliver Wickham Date: Thu, 29 Dec 2016 16:28:54 +0000 Subject: [PATCH 2/3] Move the address lookup to the test package for the time being --- app/config/frontendAppConfig.scala | 2 +- .../{ => test}/AddressLookupController.scala | 13 +++++++------ conf/prod.routes | 2 +- conf/testOnlyDoNotUseInAppConf.routes | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) rename app/controllers/{ => test}/AddressLookupController.scala (95%) diff --git a/app/config/frontendAppConfig.scala b/app/config/frontendAppConfig.scala index 3fab469..51f1545 100644 --- a/app/config/frontendAppConfig.scala +++ b/app/config/frontendAppConfig.scala @@ -83,5 +83,5 @@ object FrontendAppConfig extends AppConfig with ServicesConfig { blockNewAccountsDate = configuration.getString("application.blockNewAccountsDate"), blockApplicationsDate = configuration.getString("application.blockApplicationsDate") ) - override val addressLookupConfig = configuration.underlying.as[AddressLookupConfig]("microservice.services.address-lookup") + override lazy val addressLookupConfig = configuration.underlying.as[AddressLookupConfig]("microservice.services.address-lookup") } diff --git a/app/controllers/AddressLookupController.scala b/app/controllers/test/AddressLookupController.scala similarity index 95% rename from app/controllers/AddressLookupController.scala rename to app/controllers/test/AddressLookupController.scala index 5de8fdf..4607aa0 100644 --- a/app/controllers/AddressLookupController.scala +++ b/app/controllers/test/AddressLookupController.scala @@ -14,17 +14,13 @@ * limitations under the License. */ -package controllers +package controllers.test import config.CSRHttp import connectors.addresslookup.AddressLookupClient +import controllers.BaseController import play.api.mvc.Action -object AddressLookupController extends AddressLookupController { - val http = CSRHttp - val addressLookupEndpoint = config.FrontendAppConfig.addressLookupConfig.url -} - trait AddressLookupController extends BaseController with AddressLookupClient { // TODO Add permissions into this once the feature is ready to be moved from test routes def addressLookup(postcode: String) = Action.async { implicit request => @@ -32,3 +28,8 @@ trait AddressLookupController extends BaseController with AddressLookupClient { findByPostcode(decoded, None).map(r => Ok(r.toString)) } } + +object AddressLookupController extends AddressLookupController { + val http = CSRHttp + val addressLookupEndpoint = config.FrontendAppConfig.addressLookupConfig.url +} diff --git a/conf/prod.routes b/conf/prod.routes index 110a66e..07a4364 100644 --- a/conf/prod.routes +++ b/conf/prod.routes @@ -1,6 +1,6 @@ # Add all the application routes to the app.routes file --> /fset-fast-track app.Routes +-> /fset-fast-track app.Routes -> / health.Routes GET /admin/metrics com.kenshoo.play.metrics.MetricsController.metrics diff --git a/conf/testOnlyDoNotUseInAppConf.routes b/conf/testOnlyDoNotUseInAppConf.routes index 64dfdbb..533a233 100644 --- a/conf/testOnlyDoNotUseInAppConf.routes +++ b/conf/testOnlyDoNotUseInAppConf.routes @@ -12,4 +12,4 @@ # Add all the application routes to the prod.routes file -> / prod.Routes -GET /address/search/:postcode controllers.AddressLookupController.addressLookup(postcode: String) +GET /fset-fast-track/address-search/:postcode controllers.test.AddressLookupController.addressLookup(postcode: String) From f08b6fe326acbc6fb46b71a5c5f2b535c39c33f8 Mon Sep 17 00:00:00 2001 From: Oliver Wickham Date: Fri, 30 Dec 2016 11:08:10 +0000 Subject: [PATCH 3/3] Split the DTOs and the address client --- .../addresslookup/AddressLookupClient.scala | 71 +++++++++++++++ .../AddressRecord.scala} | 88 ++----------------- 2 files changed, 78 insertions(+), 81 deletions(-) create mode 100644 app/connectors/addresslookup/AddressLookupClient.scala rename app/connectors/{AddressLookup.scala => addresslookup/AddressRecord.scala} (63%) diff --git a/app/connectors/addresslookup/AddressLookupClient.scala b/app/connectors/addresslookup/AddressLookupClient.scala new file mode 100644 index 0000000..83e29de --- /dev/null +++ b/app/connectors/addresslookup/AddressLookupClient.scala @@ -0,0 +1,71 @@ +/* + * Copyright 2016 HM Revenue & Customs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package connectors.addresslookup + +import java.net.URLEncoder + +import config.CSRHttp +import uk.gov.hmrc.play.http.{ HeaderCarrier, _ } + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +/** + * The following client has been take from taken from https://github.com/hmrc/address-reputation-store. The project has + * not been added as a dependency, as it brings in many transitive dependencies that are not needed, + * as well as data cleansing/ingestion and backward compatibility logic that is not needed for this project. + * If the version 2 api gets deprecated, then these DTOs will have to change. + * There have been some minor changes made to the code to ensure that it compiles and passes scalastyle, + * but there is some copied code that is not idiomatic Scala and should be changed at some point in the future + */ + +trait AddressLookupClient { + + def addressLookupEndpoint: String + val http: CSRHttp + + private def url = s"$addressLookupEndpoint/v2/uk/addresses" + + def findById(id: String)(implicit hc: HeaderCarrier): Future[Option[AddressRecord]] = { + assert(id.length <= 100, "Postcodes cannot be larger than 100 characters") + val uq = "/" + enc(id) + http.GET[Option[AddressRecord]](url + uq).recover { + case _: NotFoundException => None + } + } + + def findByUprn(uprn: Long)(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { + val uq = "?uprn=" + uprn.toString + http.GET[List[AddressRecord]](url + uq) + } + + def findByPostcode(postcode: String, filter: Option[String])(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { + val safePostcode = postcode.replaceAll("[^A-Za-z0-9]", "") + assert(safePostcode.length <= 10, "Postcodes cannot be larger than 10 characters") + val pq = "?postcode=" + enc(safePostcode) + val fq = filter.map(fi => "&filter=" + enc(fi)).getOrElse("") + http.GET[List[AddressRecord]](url + pq + fq) + } + + def findByOutcode(outcode: Outcode, filter: String)(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { + val pq = "?outcode=" + outcode.toString + val fq = "&filter=" + enc(filter) + http.GET[List[AddressRecord]](url + pq + fq) + } + + private def enc(s: String) = URLEncoder.encode(s, "UTF-8") +} diff --git a/app/connectors/AddressLookup.scala b/app/connectors/addresslookup/AddressRecord.scala similarity index 63% rename from app/connectors/AddressLookup.scala rename to app/connectors/addresslookup/AddressRecord.scala index 152fc08..755ef3c 100644 --- a/app/connectors/AddressLookup.scala +++ b/app/connectors/addresslookup/AddressRecord.scala @@ -16,17 +16,8 @@ package connectors.addresslookup -import java.net.URLEncoder - -import config.CSRHttp import play.api.libs.json.Json -import uk.gov.hmrc.play.http.HeaderCarrier -import uk.gov.hmrc.play.http._ - -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future -import java.util.regex.Pattern /** * The following DTOs are taken from https://github.com/hmrc/address-reputation-store. The project has @@ -100,8 +91,6 @@ case class Address(lines: List[String], subdivision: Option[Country], country: Country) { - import Address._ - def nonEmptyFields: List[String] = lines ::: town.toList ::: county.toList ::: List(postcode) /** Gets a conjoined representation, excluding the country. */ @@ -110,40 +99,17 @@ case class Address(lines: List[String], /** Gets a single-line representation, excluding the country. */ def printable: String = printable(", ") - def line1: String = if (lines.nonEmpty) lines.head else "" - - def line2: String = if (lines.size > 1) lines(1) else "" - - def line3: String = if (lines.size > 2) lines(2) else "" + def line1: String = lines.lift(0).getOrElse("") - def line4: String = if (lines.size > 3) lines(3) else "" + def line2: String = lines.lift(1).getOrElse("") - def longestLineLength: Int = nonEmptyFields.map(_.length).max - - def truncatedAddress(maxLen: Int = maxLineLength): Address = - Address(lines.map(limit(_, maxLen)), town.map(limit(_, maxLen)), county.map(limit(_, maxLen)), postcode, subdivision, country) + def line3: String = lines.lift(2).getOrElse("") + def line4: String = lines.lift(3).getOrElse("") } object Address { - val maxLineLength = 35 - val danglingLetter: Pattern = Pattern.compile(".* [A-Z0-9]$") implicit val addressFormat = Json.reads[Address] - - private[addresslookup] def limit(str: String, max: Int): String = { - var s = str - while (s.length > max && s.indexOf(", ") > 0) { - s = s.replaceFirst(", ", ",") - } - if (s.length > max) { - s = s.substring(0, max).trim - if (Address.danglingLetter.matcher(s).matches()) { - s = s.substring(0, s.length - 2) - } - s - } - else { s } - } } case class LatLong(lat: Double, long: Double) { @@ -162,53 +128,13 @@ case class AddressRecord( logicalState: Option[String], streetClassification: Option[String]) { - import Address._ - def truncatedAddress(maxLen: Int = Address.maxLineLength): AddressRecord = - if (address.longestLineLength <= maxLen) { this } - else { copy(address = address.truncatedAddress(maxLen)) } - def withoutMetadata: AddressRecord = copy(blpuState = None, logicalState = None, streetClassification = None) - def geoLocation: LatLong = ??? + // TODO put in the real lat/lng once the address lookup service supports it + // Hardcoded to Coventry for the time being + val geoLocation: LatLong = LatLong(52.40656, -1.51217) } object AddressRecord { implicit val addressRecordFormat = Json.reads[AddressRecord] } - -trait AddressLookupClient{ - - def addressLookupEndpoint: String - val http: CSRHttp - - private def url = s"$addressLookupEndpoint/v2/uk/addresses" - - def findById(id: String)(implicit hc: HeaderCarrier): Future[Option[AddressRecord]] = { - assert(id.length <= 100, "Postcodes cannot be larger than 100 characters") - val uq = "/" + enc(id) - http.GET[Option[AddressRecord]](url + uq).recover { - case _: NotFoundException => None - } - } - - def findByUprn(uprn: Long)(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { - val uq = "?uprn=" + uprn.toString - http.GET[List[AddressRecord]](url + uq) - } - - def findByPostcode(postcode: String, filter: Option[String])(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { - val safePostcode = postcode.replaceAll("[^A-Za-z0-9]", "") - assert(safePostcode.length <= 10, "Postcodes cannot be larger than 10 characters") - val pq = "?postcode=" + enc(safePostcode) - val fq = filter.map(fi => "&filter=" + enc(fi)).getOrElse("") - http.GET[List[AddressRecord]](url + pq + fq) - } - - def findByOutcode(outcode: Outcode, filter: String)(implicit hc: HeaderCarrier): Future[List[AddressRecord]] = { - val pq = "?outcode=" + outcode.toString - val fq = "&filter=" + enc(filter) - http.GET[List[AddressRecord]](url + pq + fq) - } - - private def enc(s: String) = URLEncoder.encode(s, "UTF-8") -}