Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extending the RichData API to include major and minor sorting and direction. #650

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Extending the RichData API to include major and minor sorting and direction. #650

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fdcc26e
update to handle multiple filters
dataswifty Jan 28, 2023
9014ba8
Extending rich data api
dataswifty Feb 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 89 additions & 15 deletions hat/app/org/hatdex/hat/api/controllers/RichData.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ import play.api.libs.json.JsResultException
import play.api.libs.json.JsUndefined
import play.api.libs.json.JsDefined
import play.api.libs.json.JsString
import play.api.libs.json.JsBoolean
import play.api.libs.json.JsObject
import play.api.libs.json.JsNumber
import play.api.libs.json.JsNull
import play.api.libs.json.JsLookupResult

class RichData @Inject() (
components: ControllerComponents,
Expand Down Expand Up @@ -86,27 +91,30 @@ class RichData @Inject() (
endpoint: String,
orderBy: Option[String],
ordering: Option[String],
// sort: Option[String],
// order: Option[String],
skip: Option[Int],
take: Option[Int]): Action[AnyContent] =
take: Option[Int],
major: Option[String],
minor: Option[String],
sort: Option[String]): Action[AnyContent] =
SecuredAction(
WithRole(Owner(), NamespaceRead(namespace)) || ContainsApplicationRole(
Owner(),
NamespaceRead(namespace)
)
).async { implicit request =>
// this could be better
val knownKeys = List("ordering", "orderBy", "skip", "take")
val knownKeys = List("ordering", "orderBy", "skip", "take", "minor", "major", "sort")
val k: List[String] = request.queryString.keys.toList filterNot knownKeys.contains

val dataFilter =
if (k.length == 0)
None
else
Some(RichDataFilter(k.head, request.queryString.get(k.head).get.toList))
List.empty
else {
val qsMap = request.queryString.toMap
qsMap.map(item => RichDataFilter(item._1, item._2.toList))
}

makeData(namespace, endpoint, orderBy, ordering, skip, take, dataFilter)
makeData(namespace, endpoint, orderBy, ordering, skip, take, dataFilter.toSeq, major, minor, sort)
}

def saveEndpointData(
Expand Down Expand Up @@ -625,9 +633,13 @@ class RichData @Inject() (
ordering: Option[String],
skip: Option[Int],
take: Option[Int],
filter: Option[RichDataFilter] = None
filters: Seq[RichDataFilter],
major: Option[String],
minor: Option[String],
sort: Option[String]
)(implicit db: HATPostgresProfile.api.Database): Future[Result] = {
val dataEndpoint = s"$namespace/$endpoint"

val query =
Seq(EndpointQuery(dataEndpoint, None, None, None))
val data: Future[Seq[EndpointData]] = dataService.propertyData(
Expand All @@ -638,16 +650,71 @@ class RichData @Inject() (
take.orElse(Some(defaultRecordLimit))
)

val processedData = filter match {
case Some(dataFilter) =>
filterJson(data, dataFilter)
case (None) =>
val requestedData =
if (filters.isEmpty)
data
else {
val processedData: Seq[Future[Seq[EndpointData]]] = filters.map(filter => filterJson(data, filter))
val sequencedData = Future.sequence(processedData)
for {
x <- sequencedData
} yield x.flatten.sortWith((a, b) => resultSorter(a, b, major.getOrElse(""), minor, sort))
// x.flatten.sortWith((a, b) => resultSorter(a, b, "type", None))
}

requestedData.map(d => Ok(Json.toJson(d)))
}

private def resultSorter(
a: EndpointData,
b: EndpointData,
major: String,
minor: Option[String],
sort: Option[String] = Some("descending")) = {
val sortAscending_? = sort match {
case Some("ascending") => true
case _ => false
}

processedData.map(d => Ok(Json.toJson(d)))
println("comparing %s and %s and sorting ASC: %s".format(a, b, sortAscending_?))

// Can this sub search? Not yet
val majorSortA = a.data \ major
val majorSortB = b.data \ major

val minorSortA = a.data \ minor.getOrElse("")
val minorSortB = b.data \ minor.getOrElse("")

val majorSort = extractTerms(majorSortA, majorSortB)

val minorSort = extractTerms(minorSortA, minorSortB)

// println(s"(${majorSort._1} < ${majorSort._2} || (${majorSort._1} == ${majorSort._2} && ${minorSort._1} < ${minorSort._2})")
// println((majorSort._1 < majorSort._2 || (majorSort._1 == majorSort._2 && minorSort._1 < minorSort._2)))

if (sortAscending_?)
if (minor.isDefined)
(majorSort._1 < majorSort._2 || (majorSort._1 == majorSort._2 && minorSort._1 < minorSort._2))
else
(majorSort._1 < majorSort._2)
else if (minor.isDefined)
(majorSort._1 > majorSort._2 || (majorSort._1 == majorSort._2 && minorSort._1 > minorSort._2))
else
(majorSort._1 > majorSort._2)
}

private def extractTerms(
a: JsLookupResult,
b: JsLookupResult): (String, String) =
(a.get, b.get) match {
case (s1: JsString, s2: JsString) =>
(s1.value.replaceAll("\"", ""), s2.value.replaceAll("\"", ""))
case (a1: JsArray, a2: JsArray) =>
(a1.value.head.toString.replaceAll("\"", ""), a2.value.head.toString.replaceAll("\"", ""))
case (_, _) =>
("", "")
}

// Ty: I will convert to Option[String]
private def mashList(l: List[JsValue]): List[String] =
l.flatten {
Expand All @@ -663,11 +730,18 @@ class RichData @Inject() (
d.filter { a =>
val newValue = (a.data \\ filter.attribute).toList
val intermediate: List[String] = mashList(newValue)
println("---------------------")
println(s"intermediate: $intermediate")
println(s"filter: ${filter.value}")

filter.value.map(x => x.trim().split(',').map(y => println(y.trim())))
println(s"intersect: ${filter.value.intersect(intermediate).length}")

filter.value.intersect(intermediate).length > 0
}
}

def getSub(
private def getSub(
keys: List[String],
body: JsValue): Option[JsValue] =
if (keys.length == 0)
Expand Down
2 changes: 1 addition & 1 deletion hat/conf/v20.routes
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ GET /files/restrictAccessPublic/:fileId

POST /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.registerCombinator(combinator)
GET /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getCombinatorData(combinator, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int])
GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int])
GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int], major: Option[String], minor: Option[String], sort: Option[String])
POST /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.saveEndpointData(namespace, endpoint, skipErrors: Option[Boolean])
DELETE /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.deleteEndpointData(namespace, endpoint)
PUT /data org.hatdex.hat.api.controllers.RichData.updateRecords()
Expand Down
2 changes: 1 addition & 1 deletion hat/conf/v26.routes
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ GET /files/restrictAccessPublic/:fileId
# RICH DATA routes
POST /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.registerCombinator(combinator)
GET /combinator/$combinator<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getCombinatorData(combinator, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int])
GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int])
GET /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.getEndpointData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int], major: Option[String], minor: Option[String], sort: Option[String])
POST /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.saveEndpointData(namespace, endpoint, skipErrors: Option[Boolean])
DELETE /data/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.RichData.deleteEndpointData(namespace, endpoint)
PUT /data org.hatdex.hat.api.controllers.RichData.updateRecords()
Expand Down
22 changes: 14 additions & 8 deletions hat/test/org/hatdex/hat/api/controllers/RichDataSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ class RichDataSpec extends RichDataContext {

val controller = application.injector.instanceOf[RichData]

val response = Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request)
val response = Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None),
request)
val responseData = contentAsJson(response).as[Seq[EndpointData]]
responseData.length must equal(0)
}
Expand All @@ -88,7 +89,8 @@ class RichDataSpec extends RichDataContext {

val response = for {
_ <- service.saveData(owner.userId, data)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", Some("field"), None, None, Some(2)), request)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", Some("field"), None, None, Some(2), None, None, None),
request)
} yield r
val responseData = contentAsJson(response).as[Seq[EndpointData]]
responseData.length must equal(2)
Expand All @@ -111,7 +113,7 @@ class RichDataSpec extends RichDataContext {
val response = for {
_ <- service.saveData(owner.userId, data)
r <- Helpers.call(
controller.getEndpointData("test", "endpoint", Some("field"), Some("descending"), Some(1), Some(2)),
controller.getEndpointData("test", "endpoint", Some("field"), Some("descending"), Some(1), Some(2), None, None, None),
request
)
} yield r
Expand Down Expand Up @@ -147,7 +149,8 @@ class RichDataSpec extends RichDataContext {

val response = for {
_ <- Helpers.call(controller.saveEndpointData("test", "endpoint", None), request)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None),
request)
} yield r
val responseData = contentAsJson(response).as[Seq[EndpointData]]
responseData.length must equal(1)
Expand All @@ -163,7 +166,8 @@ class RichDataSpec extends RichDataContext {

val response = for {
_ <- Helpers.call(controller.saveEndpointData("test", "endpoint", None), request)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None),
request)
} yield r
val responseData = contentAsJson(response).as[Seq[EndpointData]]
responseData.length must equal(2)
Expand Down Expand Up @@ -195,7 +199,8 @@ class RichDataSpec extends RichDataContext {

val response = for {
_ <- Helpers.call(controller.saveEndpointData("test", "endpoint", Some(true)), request)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None), request)
r <- Helpers.call(controller.getEndpointData("test", "endpoint", None, None, None, None, None, None, None),
request)
} yield r

val responseData = contentAsJson(response).as[Seq[EndpointData]]
Expand Down Expand Up @@ -248,7 +253,8 @@ class RichDataSpec extends RichDataContext {
val result = for {
_ <- dataService.saveData(owner.userId, data)
_ <- Helpers.call(controller.deleteEndpointData("test", "test"), request)
response <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None), request)
response <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None, None, None, None),
request)
} yield response

val responseData = contentAsJson(result).as[Seq[EndpointData]]
Expand All @@ -270,7 +276,7 @@ class RichDataSpec extends RichDataContext {

val response = for {
_ <- Helpers.call(controller.saveBatchData(), request)
r <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None), request)
r <- Helpers.call(controller.getEndpointData("test", "test", None, None, None, None, None, None, None), request)
} yield r

status(response) must equal(OK)
Expand Down