Merge pull request #66 from hmrc/API-3951

API-3951: Additional metrics for rate limits and missing fields in Mongo

app/uk/gov/hmrc/thirdpartyapplication/metrics/MetricsModule.scala

@@ -86,8 +86,10 @@ class MetricsOrchestratorProvider @Inject()(configuration: Configuration,
 }
 
 @Singleton
-class MetricsSourcesProvider @Inject()(applicationCount: ApplicationCount) extends Provider[MetricsSources] {
-  override def get(): MetricsSources = MetricsSources(applicationCount)
+class MetricsSourcesProvider @Inject()(applicationCount: ApplicationCount,
+                                       rateLimitMetrics: RateLimitMetrics,
+                                       missingMongoFields: MissingMongoFields) extends Provider[MetricsSources] {
+  override def get(): MetricsSources = MetricsSources(applicationCount, rateLimitMetrics, missingMongoFields)
 }
 
 case class MetricsSources(metricSources: MetricSource*) {

app/uk/gov/hmrc/thirdpartyapplication/metrics/MissingMongoFields.scala

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 HM Revenue & Customs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package uk.gov.hmrc.thirdpartyapplication.metrics
+
+import javax.inject.Inject
+import play.api.Logger
+import uk.gov.hmrc.metrix.domain.MetricSource
+import uk.gov.hmrc.thirdpartyapplication.repository.ApplicationRepository
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class MissingMongoFields @Inject()(val applicationRepository: ApplicationRepository) extends MetricSource {
+  override def metrics(implicit ec: ExecutionContext): Future[Map[String, Int]] = {
+    val counts: Future[(Int, Int)] = for {
+      missingRateLimit <- applicationRepository.documentsWithFieldMissing("rateLimitTier")
+      missingLastAccessDate <- applicationRepository.documentsWithFieldMissing("lastAccess")
+    } yield (missingRateLimit, missingLastAccessDate)
+
+    counts.map(a => {
+      Logger.info(s"[METRIC]: Applications Missing Rate Limit Field: ${a._1}")
+      Logger.info(s"[METRIC]: Applications Missing Last Access Date Field: ${a._2}")
+
+      Map("applicationsMissingRateLimitField" -> a._1, "applicationsMissingLastAccessDateField" -> a._2)
+    })
+  }
+}

app/uk/gov/hmrc/thirdpartyapplication/metrics/RateLimitMetrics.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2019 HM Revenue & Customs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package uk.gov.hmrc.thirdpartyapplication.metrics
+
+import javax.inject.Inject
+import play.api.Logger
+import uk.gov.hmrc.metrix.domain.MetricSource
+import uk.gov.hmrc.thirdpartyapplication.models.RateLimitTier.RateLimitTier
+import uk.gov.hmrc.thirdpartyapplication.repository.ApplicationRepository
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class RateLimitMetrics @Inject()(val applicationRepository: ApplicationRepository) extends MetricSource {
+  override def metrics(implicit ec: ExecutionContext): Future[Map[String, Int]] =
+    numberOfApplicationsByRateLimit.map(
+      applicationCounts =>
+        applicationCounts.map(rateLimit => {
+          Logger.info(s"[METRIC] Number of Applications for Rate Limit ${rateLimit._1}: ${rateLimit._2}")
+          applicationsByRateLimitKey(rateLimit._1) -> rateLimit._2
+        }))
+
+  def numberOfApplicationsByRateLimit(implicit ec: ExecutionContext): Future[Map[Option[RateLimitTier], Int]] =
+    applicationRepository.fetchAll().map(applications => applications.groupBy(_.rateLimitTier).mapValues(_.size))
+
+  private def applicationsByRateLimitKey(rateLimit: Option[RateLimitTier]): String = {
+    val rateLimitString = if(rateLimit.isDefined) rateLimit.get.toString else "UNKNOWN"
+    s"applicationsByRateLimit.$rateLimitString"
+  }
+}

app/uk/gov/hmrc/thirdpartyapplication/repository/ApplicationRepository.scala

@@ -24,8 +24,9 @@ import play.api.libs.iteratee._
 import play.api.libs.json.Json._
 import play.api.libs.json.{JsObject, _}
 import play.modules.reactivemongo.ReactiveMongoComponent
+import reactivemongo.api.ReadConcern.Available
 import reactivemongo.api.commands.Command.CommandWithPackRunner
-import reactivemongo.api.{FailoverStrategy, ReadPreference}
+import reactivemongo.api.{FailoverStrategy, ReadConcern, ReadPreference}
 import reactivemongo.bson.{BSONDateTime, BSONObjectID}
 import reactivemongo.play.iteratees.cursorProducer
 import reactivemongo.play.json._
@@ -40,7 +41,7 @@ import uk.gov.hmrc.thirdpartyapplication.models.db.ApplicationData
 import uk.gov.hmrc.thirdpartyapplication.util.mongo.IndexHelper._
 
 import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Future
+import scala.concurrent.{ExecutionContext, Future}
 
 @Singleton
 class ApplicationRepository @Inject()(mongo: ReactiveMongoComponent)
@@ -303,6 +304,11 @@ class ApplicationRepository @Inject()(mongo: ReactiveMongoComponent)
   }
 
   def delete(id: UUID): Future[HasSucceeded] = remove("id" -> id).map(_ => HasSucceeded)
+
+  def documentsWithFieldMissing(fieldName: String): Future[Int] = {
+    collection.count(Some(Json.obj(fieldName -> Json.obj(f"$$exists" -> false))), None, 0, None, Available).map(_.toInt)
+  }
+
 }
 
 sealed trait ApplicationModificationResult

test/unit/uk/gov/hmrc/thirdpartyapplication/metrics/MissingMongoFieldsSpec.scala

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 HM Revenue & Customs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package unit.uk.gov.hmrc.thirdpartyapplication.metrics
+
+import org.mockito.Mockito.when
+import org.scalatest.mockito.MockitoSugar
+import uk.gov.hmrc.play.test.UnitSpec
+import uk.gov.hmrc.thirdpartyapplication.metrics.MissingMongoFields
+import uk.gov.hmrc.thirdpartyapplication.repository.ApplicationRepository
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+
+class MissingMongoFieldsSpec extends UnitSpec with MockitoSugar {
+
+  trait Setup {
+    val mockApplicationRepository: ApplicationRepository = mock[ApplicationRepository]
+    val metricUnderTest: MissingMongoFields = new MissingMongoFields(mockApplicationRepository)
+  }
+
+  "refresh metrics" should {
+    "update values for number of applications with missing fields" in new Setup {
+      private val numberOfMissingRateLimits = 10
+      private val numberOfMissingLastAccessedDate = 3
+
+      when(mockApplicationRepository.documentsWithFieldMissing("rateLimitTier")).thenReturn(Future.successful(numberOfMissingRateLimits))
+      when(mockApplicationRepository.documentsWithFieldMissing("lastAccess")).thenReturn(Future.successful(numberOfMissingLastAccessedDate))
+
+      private val result = await(metricUnderTest.metrics)
+
+      result("applicationsMissingRateLimitField") shouldBe numberOfMissingRateLimits
+      result("applicationsMissingLastAccessDateField") shouldBe numberOfMissingLastAccessedDate
+    }
+  }
+}

test/unit/uk/gov/hmrc/thirdpartyapplication/metrics/RateLimitMetricsSpec.scala

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 HM Revenue & Customs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package unit.uk.gov.hmrc.thirdpartyapplication.metrics
+
+import org.mockito.Mockito.when
+import org.scalatest.mockito.MockitoSugar
+import uk.gov.hmrc.play.test.UnitSpec
+import uk.gov.hmrc.thirdpartyapplication.metrics.RateLimitMetrics
+import uk.gov.hmrc.thirdpartyapplication.models.RateLimitTier
+import uk.gov.hmrc.thirdpartyapplication.models.RateLimitTier.RateLimitTier
+import uk.gov.hmrc.thirdpartyapplication.models.db.ApplicationData
+import uk.gov.hmrc.thirdpartyapplication.repository.ApplicationRepository
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+
+class RateLimitMetricsSpec extends UnitSpec with MockitoSugar {
+
+  trait Setup {
+    def applicationsWithRateLimit(rateLimit: Option[RateLimitTier], numberOfApplications: Int): Seq[ApplicationData] = {
+      def mockedApplication: ApplicationData = {
+        val application: ApplicationData = mock[ApplicationData]
+        when(application.rateLimitTier).thenReturn(rateLimit)
+
+        application
+      }
+
+      Seq.fill(numberOfApplications)(mockedApplication)
+    }
+
+    val mockApplicationRepository: ApplicationRepository = mock[ApplicationRepository]
+
+    val metricUnderTest: RateLimitMetrics = new RateLimitMetrics(mockApplicationRepository)
+  }
+
+  "metrics refresh" should {
+    "update application by rate limit counts" in new Setup {
+      private val numberOfBronze = 10
+      private val numberOfSilver = 5
+      private val numberOfGold = 2
+      private val numberOfUnknown = 1
+
+      private val applicationsToReturn: Seq[ApplicationData] =
+        applicationsWithRateLimit(Some(RateLimitTier.BRONZE), numberOfBronze) ++
+          applicationsWithRateLimit(Some(RateLimitTier.SILVER), numberOfSilver) ++
+          applicationsWithRateLimit(Some(RateLimitTier.GOLD), numberOfGold) ++
+          applicationsWithRateLimit(None, numberOfUnknown)
+
+      when(mockApplicationRepository.fetchAll()).thenReturn(Future.successful(applicationsToReturn))
+
+      private val result = await(metricUnderTest.metrics)
+
+      result("applicationsByRateLimit.BRONZE") shouldBe numberOfBronze
+      result("applicationsByRateLimit.SILVER") shouldBe numberOfSilver
+      result("applicationsByRateLimit.GOLD") shouldBe numberOfGold
+      result("applicationsByRateLimit.UNKNOWN") shouldBe numberOfUnknown
+    }
+  }
+}