From fdcc26e2b3098205a0a9f3a6f32956b0d108cc4b Mon Sep 17 00:00:00 2001 From: Ty Date: Sat, 28 Jan 2023 18:56:51 -0500 Subject: [PATCH 1/2] update to handle multiple filters --- .../hatdex/hat/api/controllers/RichData.scala | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/hat/app/org/hatdex/hat/api/controllers/RichData.scala b/hat/app/org/hatdex/hat/api/controllers/RichData.scala index 935c38fc..fd3a9c5d 100644 --- a/hat/app/org/hatdex/hat/api/controllers/RichData.scala +++ b/hat/app/org/hatdex/hat/api/controllers/RichData.scala @@ -86,8 +86,6 @@ class RichData @Inject() ( endpoint: String, orderBy: Option[String], ordering: Option[String], - // sort: Option[String], - // order: Option[String], skip: Option[Int], take: Option[Int]): Action[AnyContent] = SecuredAction( @@ -101,12 +99,26 @@ class RichData @Inject() ( val k: List[String] = request.queryString.keys.toList filterNot knownKeys.contains val dataFilter = - if (k.length == 0) - None - else - Some(RichDataFilter(k.head, request.queryString.get(k.head).get.toList)) + if (k.length == 0) { + List.empty + } + else { + println(s"request.queryString: ${request.queryString}") + println(s"request.queryString.get(k.head).get: ${request.queryString.get(k.head).get.head.split(',')}") + println(s"request.queryString.get(k.head).get.toList: ${request.queryString.get(k.head).get.toList}") + + val qsMap = request.queryString.toMap + println(s"qsMap : ${qsMap}") + println(s"qsMap.keySet: ${qsMap.keySet}") + println(s"qsMap.type : ${qsMap.get("type")}") + + val searchTerms = qsMap.get("type").getOrElse(List.empty[String]) + println(s"searchTerms : ${searchTerms}") + + qsMap.map(item => RichDataFilter(item._1, item._2.toList)) + } - makeData(namespace, endpoint, orderBy, ordering, skip, take, dataFilter) + makeData(namespace, endpoint, orderBy, ordering, skip, take, dataFilter.toSeq) } def saveEndpointData( @@ -625,7 +637,7 @@ class RichData @Inject() ( ordering: Option[String], skip: Option[Int], take: Option[Int], - filter: Option[RichDataFilter] = None + filters: Seq[RichDataFilter] )(implicit db: HATPostgresProfile.api.Database): Future[Result] = { val dataEndpoint = s"$namespace/$endpoint" val query = @@ -638,14 +650,18 @@ class RichData @Inject() ( take.orElse(Some(defaultRecordLimit)) ) - val processedData = filter match { - case Some(dataFilter) => - filterJson(data, dataFilter) - case (None) => - data + val requestedData = if (filters.isEmpty) { + data } - - processedData.map(d => Ok(Json.toJson(d))) + else { + val processedData: Seq[Future[Seq[EndpointData]]] = filters.map(filter => filterJson(data, filter)) + val sequencedData = Future.sequence(processedData) + for { + x <- sequencedData + } yield x.flatten + } + + requestedData.map(d => Ok(Json.toJson(d))) } // Ty: I will convert to Option[String] @@ -663,6 +679,13 @@ class RichData @Inject() ( d.filter { a => val newValue = (a.data \\ filter.attribute).toList val intermediate: List[String] = mashList(newValue) + println("---------------------") + println(s"intermediate: $intermediate") + println(s"filter: ${filter.value}") + + filter.value.map(x => x.trim().split(',').map(y => println(y.trim()))) + println(s"intersect: ${filter.value.intersect(intermediate).length}") + filter.value.intersect(intermediate).length > 0 } } From 9014ba8e308e25accf890c5ea36f745b5d43b4a8 Mon Sep 17 00:00:00 2001 From: Ty Date: Wed, 1 Feb 2023 14:46:41 -0500 Subject: [PATCH 2/2] Extending rich data api --- .../hatdex/hat/api/controllers/RichData.scala | 109 +++++++++++++----- hat/conf/v20.routes | 2 +- hat/conf/v26.routes | 2 +- .../hat/api/controllers/RichDataSpec.scala | 22 ++-- 4 files changed, 96 insertions(+), 39 deletions(-) diff --git a/hat/app/org/hatdex/hat/api/controllers/RichData.scala b/hat/app/org/hatdex/hat/api/controllers/RichData.scala index fd3a9c5d..b60a5720 100644 --- a/hat/app/org/hatdex/hat/api/controllers/RichData.scala +++ b/hat/app/org/hatdex/hat/api/controllers/RichData.scala @@ -47,6 +47,11 @@ import play.api.libs.json.JsResultException import play.api.libs.json.JsUndefined import play.api.libs.json.JsDefined import play.api.libs.json.JsString +import play.api.libs.json.JsBoolean +import play.api.libs.json.JsObject +import play.api.libs.json.JsNumber +import play.api.libs.json.JsNull +import play.api.libs.json.JsLookupResult class RichData @Inject() ( components: ControllerComponents, @@ -87,7 +92,10 @@ class RichData @Inject() ( orderBy: Option[String], ordering: Option[String], skip: Option[Int], - take: Option[Int]): Action[AnyContent] = + take: Option[Int], + major: Option[String], + minor: Option[String], + sort: Option[String]): Action[AnyContent] = SecuredAction( WithRole(Owner(), NamespaceRead(namespace)) || ContainsApplicationRole( Owner(), @@ -95,30 +103,18 @@ class RichData @Inject() ( ) ).async { implicit request => // this could be better - val knownKeys = List("ordering", "orderBy", "skip", "take") + val knownKeys = List("ordering", "orderBy", "skip", "take", "minor", "major", "sort") val k: List[String] = request.queryString.keys.toList filterNot knownKeys.contains val dataFilter = - if (k.length == 0) { + if (k.length == 0) List.empty - } else { - println(s"request.queryString: ${request.queryString}") - println(s"request.queryString.get(k.head).get: ${request.queryString.get(k.head).get.head.split(',')}") - println(s"request.queryString.get(k.head).get.toList: ${request.queryString.get(k.head).get.toList}") - val qsMap = request.queryString.toMap - println(s"qsMap : ${qsMap}") - println(s"qsMap.keySet: ${qsMap.keySet}") - println(s"qsMap.type : ${qsMap.get("type")}") - - val searchTerms = qsMap.get("type").getOrElse(List.empty[String]) - println(s"searchTerms : ${searchTerms}") - qsMap.map(item => RichDataFilter(item._1, item._2.toList)) } - makeData(namespace, endpoint, orderBy, ordering, skip, take, dataFilter.toSeq) + makeData(namespace, endpoint, orderBy, ordering, skip, take, dataFilter.toSeq, major, minor, sort) } def saveEndpointData( @@ -637,9 +633,13 @@ class RichData @Inject() ( ordering: Option[String], skip: Option[Int], take: Option[Int], - filters: Seq[RichDataFilter] + filters: Seq[RichDataFilter], + major: Option[String], + minor: Option[String], + sort: Option[String] )(implicit db: HATPostgresProfile.api.Database): Future[Result] = { val dataEndpoint = s"$namespace/$endpoint" + val query = Seq(EndpointQuery(dataEndpoint, None, None, None)) val data: Future[Seq[EndpointData]] = dataService.propertyData( @@ -650,20 +650,71 @@ class RichData @Inject() ( take.orElse(Some(defaultRecordLimit)) ) - val requestedData = if (filters.isEmpty) { - data - } - else { - val processedData: Seq[Future[Seq[EndpointData]]] = filters.map(filter => filterJson(data, filter)) - val sequencedData = Future.sequence(processedData) - for { - x <- sequencedData - } yield x.flatten - } - + val requestedData = + if (filters.isEmpty) + data + else { + val processedData: Seq[Future[Seq[EndpointData]]] = filters.map(filter => filterJson(data, filter)) + val sequencedData = Future.sequence(processedData) + for { + x <- sequencedData + } yield x.flatten.sortWith((a, b) => resultSorter(a, b, major.getOrElse(""), minor, sort)) + // x.flatten.sortWith((a, b) => resultSorter(a, b, "type", None)) + } + requestedData.map(d => Ok(Json.toJson(d))) } + private def resultSorter( + a: EndpointData, + b: EndpointData, + major: String, + minor: Option[String], + sort: Option[String] = Some("descending")) = { + val sortAscending_? = sort match { + case Some("ascending") => true + case _ => false + } + + println("comparing %s and %s and sorting ASC: %s".format(a, b, sortAscending_?)) + + // Can this sub search? Not yet + val majorSortA = a.data \ major + val majorSortB = b.data \ major + + val minorSortA = a.data \ minor.getOrElse("") + val minorSortB = b.data \ minor.getOrElse("") + + val majorSort = extractTerms(majorSortA, majorSortB) + + val minorSort = extractTerms(minorSortA, minorSortB) + + // println(s"(${majorSort._1} < ${majorSort._2} || (${majorSort._1} == ${majorSort._2} && ${minorSort._1} < ${minorSort._2})") + // println((majorSort._1 < majorSort._2 || (majorSort._1 == majorSort._2 && minorSort._1 < minorSort._2))) + + if (sortAscending_?) + if (minor.isDefined) + (majorSort._1 < majorSort._2 || (majorSort._1 == majorSort._2 && minorSort._1 < minorSort._2)) + else + (majorSort._1 < majorSort._2) + else if (minor.isDefined) + (majorSort._1 > majorSort._2 || (majorSort._1 == majorSort._2 && minorSort._1 > minorSort._2)) + else + (majorSort._1 > majorSort._2) + } + + private def extractTerms( + a: JsLookupResult, + b: JsLookupResult): (String, String) = + (a.get, b.get) match { + case (s1: JsString, s2: JsString) => + (s1.value.replaceAll("\"", ""), s2.value.replaceAll("\"", "")) + case (a1: JsArray, a2: JsArray) => + (a1.value.head.toString.replaceAll("\"", ""), a2.value.head.toString.replaceAll("\"", "")) + case (_, _) => + ("", "") + } + // Ty: I will convert to Option[String] private def mashList(l: List[JsValue]): List[String] = l.flatten { @@ -690,7 +741,7 @@ class RichData @Inject() ( } } - def getSub( + private def getSub( keys: List[String], body: JsValue): Option[JsValue] = if (keys.length == 0) diff --git a/hat/conf/v20.routes b/hat/conf/v20.routes index 183c31ac..be0dff09 100644 --- a/hat/conf/v20.routes +++ b/hat/conf/v20.routes @@ -23,7 +23,7 @@ GET /files/restrictAccessPublic/:fileId POST /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.registerCombinator(combinator) GET /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getCombinatorData(combinator, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int]) -GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int]) +GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int], major: Option[String], minor: Option[String], sort: Option[String]) POST /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.saveEndpointData(namespace, endpoint, skipErrors: Option[Boolean]) DELETE /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.deleteEndpointData(namespace, endpoint) PUT /data org.hatdex.hat.api.controllers.RichData.updateRecords() diff --git a/hat/conf/v26.routes b/hat/conf/v26.routes index ce83bfa5..33cced90 100644 --- a/hat/conf/v26.routes +++ b/hat/conf/v26.routes @@ -27,7 +27,7 @@ GET /files/restrictAccessPublic/:fileId # RICH DATA routes POST /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.registerCombinator(combinator) GET /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getCombinatorData(combinator, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int]) -GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int]) +GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int], major: Option[String], minor: Option[String], sort: Option[String]) POST /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.saveEndpointData(namespace, endpoint, skipErrors: Option[Boolean]) DELETE /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.deleteEndpointData(namespace, endpoint) PUT /data org.hatdex.hat.api.controllers.RichData.updateRecords() diff --git a/hat/test/org/hatdex/hat/api/controllers/RichDataSpec.scala b/hat/test/org/hatdex/hat/api/controllers/RichDataSpec.scala index 17ce379e..34bedee7 100644 --- a/hat/test/org/hatdex/hat/api/controllers/RichDataSpec.scala +++ b/hat/test/org/hatdex/hat/api/controllers/RichDataSpec.scala @@ -68,7 +68,8 @@ class RichDataSpec extends RichDataContext { val controller = application.injector.instanceOf[RichData] - val response = Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request) + val response = Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None), + request) val responseData = contentAsJson(response).as[Seq[EndpointData]] responseData.length must equal(0) } @@ -88,7 +89,8 @@ class RichDataSpec extends RichDataContext { val response = for { _ <- service.saveData(owner.userId, data) - r <- Helpers.call(controller.getEndpointData("test", "endpoint", Some("field"), None, None, Some(2)), request) + r <- Helpers.call(controller.getEndpointData("test", "endpoint", Some("field"), None, None, Some(2), None, None, None), + request) } yield r val responseData = contentAsJson(response).as[Seq[EndpointData]] responseData.length must equal(2) @@ -111,7 +113,7 @@ class RichDataSpec extends RichDataContext { val response = for { _ <- service.saveData(owner.userId, data) r <- Helpers.call( - controller.getEndpointData("test", "endpoint", Some("field"), Some("descending"), Some(1), Some(2)), + controller.getEndpointData("test", "endpoint", Some("field"), Some("descending"), Some(1), Some(2), None, None, None), request ) } yield r @@ -147,7 +149,8 @@ class RichDataSpec extends RichDataContext { val response = for { _ <- Helpers.call(controller.saveEndpointData("test", "endpoint", None), request) - r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request) + r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None), + request) } yield r val responseData = contentAsJson(response).as[Seq[EndpointData]] responseData.length must equal(1) @@ -163,7 +166,8 @@ class RichDataSpec extends RichDataContext { val response = for { _ <- Helpers.call(controller.saveEndpointData("test", "endpoint", None), request) - r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request) + r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None), + request) } yield r val responseData = contentAsJson(response).as[Seq[EndpointData]] responseData.length must equal(2) @@ -195,7 +199,8 @@ class RichDataSpec extends RichDataContext { val response = for { _ <- Helpers.call(controller.saveEndpointData("test", "endpoint", Some(true)), request) - r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request) + r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None), + request) } yield r val responseData = contentAsJson(response).as[Seq[EndpointData]] @@ -248,7 +253,8 @@ class RichDataSpec extends RichDataContext { val result = for { _ <- dataService.saveData(owner.userId, data) _ <- Helpers.call(controller.deleteEndpointData("test", "test"), request) - response <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None), request) + response <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None, None, None, None), + request) } yield response val responseData = contentAsJson(result).as[Seq[EndpointData]] @@ -270,7 +276,7 @@ class RichDataSpec extends RichDataContext { val response = for { _ <- Helpers.call(controller.saveBatchData(), request) - r <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None), request) + r <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None, None, None, None), request) } yield r status(response) must equal(OK)