Merge pull request #3713 from ProjectSidewalk/develop

v8.0.0

Gruntfile.js

@@ -49,6 +49,7 @@ module.exports = function(grunt) {
                 src: [
                     'public/javascripts/Admin/src/*.js',
                     'public/javascripts/common/Utilities.js',
+                    'public/javascripts/common/UtilitiesMath.js',
                     'public/javascripts/common/UtilitiesSidewalk.js',
                     'public/javascripts/common/Panomarker.js',
                     'public/javascripts/common/UtilitiesPanomarker.js'

Makefile

@@ -39,6 +39,9 @@ docker-run:
 ssh:
 	@docker exec -it projectsidewalk-$${target} /bin/bash
 
+import-users:
+	@docker exec -it projectsidewalk-db sh -c "/opt/import-users.sh"
+
 import-dump:
 	@docker exec -it projectsidewalk-db sh -c "/opt/import-dump.sh $(db)"
 

README.md

@@ -83,6 +83,7 @@ On Windows, we recommend [Windows Powershell](https://docs.microsoft.com/en-us/p
 1. Modify the `SIDEWALK_CITY_ID` line in the `docker-compose.yml` to use the ID of the appropriate city, listed [here](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting#first-heres-a-table-that-youll-reference-when-setting-up-your-dev-env) (it's the city that matches your database dump, so check the name of the db dump file).
 1. Modify the `DATABASE_USER` line in the `docker-compose.yml`, replacing "sidewalk" with the username from the table [linked above](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting#first-heres-a-table-that-youll-reference-when-setting-up-your-dev-env).
 1. Rename the database dump file that you got from Mikey to "\<database_user\>-dump" (using the name from the prev step) and put it in the `db/` directory (other files in this dir include `init.sh` and `import-dump.sh`).
+1. Rename the users dump file that you got from Mikey to "sidewalk_users-dump" and put it in the `db/` directory as well.
 1. From the root SidewalkWebpage dir, run `make dev`. This will take time (20-30 mins or more depending on your Internet connection) as the command downloads the docker images, spins up the containers, and opens a Docker shell into the webpage container in that same terminal. The containers (running Ubuntu Stretch) will have all the necessary packages and tools so no installation is necessary. This command also initializes the database, though we still need to import the data. Successful output of this command will look like:
 
     ```
@@ -93,7 +94,8 @@ On Windows, we recommend [Windows Powershell](https://docs.microsoft.com/en-us/p
     root@[container-id]:/home#
     ```
 
-1. Run `make import-dump db=<database_user>` (needs to be the same thing you set for `DATABASE_USER`) from the root project directory outside the Docker shell (from a new Ubuntu terminal). This may take a while depending on the size of the dump. Don't panic if this step fails :) and consult the [Docker Troubleshooting wiki](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting). Check the output carefully. If it looks like there are errors, do not skip to the next step, check the wiki and ask Mikey if you don't find solutions in there.
+1. Run `make import-users` from the root project directory outside the Docker shell (from a new Ubuntu terminal). This may take 1-2 minutes.
+1. Run `make import-dump db=<database_user>` (needs to be the same value you set for `DATABASE_USER`). This may take a while depending on the size of the dump. Don't panic if this step fails :) and consult the [Docker Troubleshooting wiki](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting). Check the output carefully. If it looks like there are errors, do not skip to the next step, check the wiki and ask Mikey if you don't find solutions in there.
 1. Run `npm start` from inside the Docker shell (the terminal where you ran `make dev`). If this is your first time running the command, *everything* will need to be compiled. So, it may take 5+ minutes initially, but will be orders of magnitude faster in the future (~10 secs).
 
     The behavior of `npm start` is dictated by what `start` is supposed to do as defined in `package.json` file. As per the current code, running this command will run `grunt watch` & `sbt compile "~ run"` (the `~` here is triggered execution that allows for the server to run in watch mode). This should start the web server. Successful output of this command will look like:
@@ -121,7 +123,8 @@ On Windows, we recommend [Windows Powershell](https://docs.microsoft.com/en-us/p
 
 ### Setting up another database or city
 1. Acquire another database dump, put it in the `db/` directory, and rename it to "\<database_user\>-dump", using the appropriate database user from [this table](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting#first-heres-a-table-that-youll-reference-when-setting-up-your-dev-env).
-1. Run `make import-dump db=<db_user>` (using the name from the prev step) from the root project directory outside the Docker shell.
+1. If your new database dump was created at a later date than the ones you've already imported, you'll need to rerun `make import-users` with a new `sidewalk_users-dump` file. Ask Mikey if you're not sure! You can see the dates the dumps were created in their original filenames.
+1. Run `make import-dump db=<db_user>` (using the name from the step 1) from the root project directory outside the Docker shell.
 1. Update the `DATABASE_USER` variable in the `docker-compose.yml` to the same name.
 1. Modify the `SIDEWALK_CITY_ID` line in `docker-compose.yml` to use the appropriate ID from [this table](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting#first-heres-a-table-that-youll-reference-when-setting-up-your-dev-env).
 1. Rerun `make dev`.

app/controllers/AdminController.scala

@@ -108,7 +108,7 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
    */
   def getAllLabels = UserAwareAction.async { implicit request =>
     if (isAdmin(request.identity)) {
-      val labels = LabelTable.selectLocationsAndSeveritiesOfLabels(List())
+      val labels = LabelTable.selectLocationsAndSeveritiesOfLabels(List(), List())
       val features: List[JsObject] = labels.par.map { label =>
         val point = geojson.Point(geojson.LatLng(label.lat.toDouble, label.lng.toDouble))
         val properties = Json.obj(
@@ -131,9 +131,10 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
   /**
    * Get a list of all labels with metadata needed for /labelMap.
    */
-  def getAllLabelsForLabelMap(regions: Option[String]) = UserAwareAction.async { implicit request =>
+  def getAllLabelsForLabelMap(regions: Option[String], routes: Option[String]) = UserAwareAction.async { implicit request =>
     val regionIds: List[Int] = regions.map(parseIntegerList).getOrElse(List())
-    val labels: List[LabelLocationWithSeverity] = LabelTable.selectLocationsAndSeveritiesOfLabels(regionIds)
+    val routeIds: List[Int] = routes.map(parseIntegerList).getOrElse(List())
+    val labels: List[LabelLocationWithSeverity] = LabelTable.selectLocationsAndSeveritiesOfLabels(regionIds, routeIds)
     val features: List[JsObject] = labels.par.map { label =>
       val point: Point[LatLng] = geojson.Point(geojson.LatLng(label.lat.toDouble, label.lng.toDouble))
       val properties: JsObject = Json.obj(
@@ -574,9 +575,9 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
           if (currentOrg.nonEmpty) {
             UserOrgTable.remove(userId, currentOrg.get)
           }
-          val confirmedOrgId: Int = UserOrgTable.save(userId, newOrgId)
+          val rowsUpdated: Int = UserOrgTable.save(userId, newOrgId)
 
-          if (confirmedOrgId == newOrgId) {
+          if (rowsUpdated > 0) {
             Future.successful(Ok(Json.obj("user_id" -> userId, "org_id" -> newOrgId)))
           } else {
             Future.successful(BadRequest("Error saving org"))
@@ -669,4 +670,74 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
       }
     )
   }
+
+  /**
+   * Gets street edge data for the coverage section of the admin page.
+   */
+  def getCoverageData = UserAwareAction.async { implicit request =>
+    val streetCountsData = Json.obj(
+      "total" -> StreetEdgeTable.countTotalStreets(),
+      "audited" -> Json.obj(
+        "all_users" -> Json.obj(
+          "all" -> StreetEdgeTable.countAuditedStreets(),
+          "high_quality" -> StreetEdgeTable.countAuditedStreets(1, "All", true)
+        ),
+        "registered" -> Json.obj(
+          "all" -> StreetEdgeTable.countAuditedStreets(1, "Registered"),
+          "high_quality" -> StreetEdgeTable.countAuditedStreets(1, "Registered", true)
+        ),
+        "anonymous" -> Json.obj(
+          "all" -> StreetEdgeTable.countAuditedStreets(1, "Anonymous"),
+          "high_quality" -> StreetEdgeTable.countAuditedStreets(1, "Anonymous", true)
+        ),
+        "turker" -> Json.obj(
+          "all" -> StreetEdgeTable.countAuditedStreets(1, "Turker"),
+          "high_quality" -> StreetEdgeTable.countAuditedStreets(1, "Turker", true)
+        ),
+        "researcher" -> Json.obj(
+          "all" -> StreetEdgeTable.countAuditedStreets(1, "Researcher"),
+          "high_quality" -> StreetEdgeTable.countAuditedStreets(1, "Researcher", true)
+        )
+      )
+    )
+
+    val streetDistanceData = Json.obj(
+      "total" -> StreetEdgeTable.totalStreetDistance(),
+      "audited" -> Json.obj(
+        "all_users" -> Json.obj(
+          "all" -> StreetEdgeTable.auditedStreetDistance(1),
+          "high_quality" -> StreetEdgeTable.auditedStreetDistance(1, "All", true)
+        ),
+        "registered" -> Json.obj(
+          "all" -> StreetEdgeTable.auditedStreetDistance(1, "Registered"),
+          "high_quality" -> StreetEdgeTable.auditedStreetDistance(1, "Registered", true)
+        ),
+        "anonymous" -> Json.obj(
+          "all" -> StreetEdgeTable.auditedStreetDistance(1, "Anonymous"),
+          "high_quality" -> StreetEdgeTable.auditedStreetDistance(1, "Anonymous", true)
+        ),
+        "turker" -> Json.obj(
+          "all" -> StreetEdgeTable.auditedStreetDistance(1, "Turker"),
+          "high_quality" -> StreetEdgeTable.auditedStreetDistance(1, "Turker", true)
+        ),
+        "researcher" -> Json.obj(
+          "all" -> StreetEdgeTable.auditedStreetDistance(1, "Researcher"),
+          "high_quality" -> StreetEdgeTable.auditedStreetDistance(1, "Researcher", true)
+        ),
+
+        // Audited distance over time is related, but included in a separate table on the Admin page.
+        "with_overlap" -> Json.obj(
+          "all_time" -> StreetEdgeTable.auditedStreetDistanceOverTime("all time"),
+          "today" -> StreetEdgeTable.auditedStreetDistanceOverTime("today"),
+          "week" -> StreetEdgeTable.auditedStreetDistanceOverTime("week")
+        )
+      )
+    )
+
+    val data = Json.obj(
+      "street_counts" -> streetCountsData,
+      "street_distance" -> streetDistanceData
+    )
+    Future.successful(Ok(data))
+  }
 }

app/controllers/ApplicationController.scala

@@ -348,21 +348,27 @@ class ApplicationController @Inject() (implicit val env: Environment[User, Sessi
   }
 
   /**
-   * Returns the labelmap page that contains a cool visualization.
+   * Returns the LabelMap page that contains a cool visualization.
    */
-  def labelMap(regions: Option[String]) = UserAwareAction.async { implicit request =>
+  def labelMap(regions: Option[String], routes: Option[String]) = UserAwareAction.async { implicit request =>
     val regionIds: List[Int] = regions.map(parseIntegerList).getOrElse(List())
+    val routeIds: List[Int] = routes.map(parseIntegerList).getOrElse(List())
     request.identity match {
       case Some(user) =>
         val timestamp: Timestamp = new Timestamp(Instant.now.toEpochMilli)
         val ipAddress: String = request.remoteAddress
 
         val activityStr: String = if (regions.isEmpty) "Visit_LabelMap" else s"Visit_LabelMap_Regions=$regions"
         WebpageActivityTable.save(WebpageActivity(0, user.userId.toString, ipAddress, activityStr, timestamp))
-        Future.successful(Ok(views.html.labelMap("Sidewalk - LabelMap", Some(user), regionIds)))
+        Future.successful(Ok(views.html.labelMap("Sidewalk - LabelMap", Some(user), regionIds, routeIds)))
       case None =>
         // UTF-8 codes needed to pass a URL that contains parameters: ? is %3F, & is %26
-        val queryParams: String = regions.map(r => s"%3Fregions=$r").getOrElse("")
+        val queryParams: String = (regionIds, routeIds) match {
+          case (Nil, Nil) => ""
+          case (r, Nil) => s"%3Fregions=${r.mkString(",")}"
+          case (Nil, r) => s"%3Froutes=${r.mkString(",")}"
+          case (r, s) => s"%3Fregions=${r.mkString(",")}%26routes=${s.mkString(",")}"
+        }
         Future.successful(Redirect("/anonSignUp?url=/labelmap" + queryParams))
     }
   }

app/controllers/UserProfileController.scala

@@ -78,9 +78,10 @@ class UserProfileController @Inject() (implicit val env: Environment[User, Sessi
   /**
    * Get the list of all streets and whether they have been audited or not, regardless of user.
    */
-  def getAllStreets(filterLowQuality: Boolean, regions: Option[String]) = UserAwareAction.async { implicit request =>
+  def getAllStreets(filterLowQuality: Boolean, regions: Option[String], routes: Option[String]) = UserAwareAction.async { implicit request =>
     val regionIds: List[Int] = regions.map(parseIntegerList).getOrElse(List())
-    val streets: List[StreetEdgeWithAuditStatus] = AuditTaskTable.selectStreetsWithAuditStatus(filterLowQuality, regionIds)
+    val routeIds: List[Int] = routes.map(parseIntegerList).getOrElse(List())
+    val streets: List[StreetEdgeWithAuditStatus] = AuditTaskTable.selectStreetsWithAuditStatus(filterLowQuality, regionIds, routeIds)
     val features: List[JsObject] = streets.map { edge =>
       val coordinates: Array[Coordinate] = edge.geom.getCoordinates
       val latlngs: List[geojson.LatLng] = coordinates.map(coord => geojson.LatLng(coord.y, coord.x)).toList  // Map it to an immutable list

app/models/audit/AuditTaskTable.scala

@@ -12,6 +12,7 @@ import models.label.{LabelTable, LabelTypeTable}
 import models.street.StreetEdgePriorityTable
 import models.user.{UserRoleTable, UserStatTable}
 import models.mission.{Mission, MissionTable}
+import models.route.RouteStreetTable
 import play.api.libs.json._
 import play.api.Play.current
 import play.extras.geojson
@@ -313,7 +314,7 @@ object AuditTaskTable {
   /**
     * Return all street edges and whether they have been audited or not. If provided, filter for only given regions.
     */
-  def selectStreetsWithAuditStatus(filterLowQuality: Boolean, regionIds: List[Int]): List[StreetEdgeWithAuditStatus] = db.withSession { implicit session =>
+  def selectStreetsWithAuditStatus(filterLowQuality: Boolean, regionIds: List[Int], routeIds: List[Int]): List[StreetEdgeWithAuditStatus] = db.withSession { implicit session =>
     // Optionally filter out data marked as low quality.
     val _filteredTasks = if (filterLowQuality) {
       for {
@@ -335,7 +336,16 @@ object AuditTaskTable {
       .leftJoin(_distinctCompleted).on(_._1.streetEdgeId === _)
       .map(s => (s._1._1.streetEdgeId, s._1._1.geom, s._1._2.regionId, s._1._1.wayType, !s._2.?.isEmpty))
 
-    streetsWithAuditedStatus.list.map(StreetEdgeWithAuditStatus.tupled)
+    // If routeIds are provided, filter out streets that are not part of the route.
+    val streetsWithAuditedStatusFiltered = if (routeIds.nonEmpty) {
+      RouteStreetTable.routeStreets.filter(_.routeId inSet routeIds)
+        .innerJoin(streetsWithAuditedStatus).on(_.streetEdgeId === _._1)
+        .map(_._2)
+    } else {
+      streetsWithAuditedStatus
+    }
+
+    streetsWithAuditedStatusFiltered.list.map(StreetEdgeWithAuditStatus.tupled)
   }
 
   /**

app/models/daos/slick/DBTableDefinitions.scala

@@ -8,7 +8,7 @@ object DBTableDefinitions {
 
   case class DBUser (userId: String, username: String, email: String )
 
-  class UserTable(tag: Tag) extends Table[DBUser](tag, "sidewalk_user") {
+  class UserTable(tag: Tag) extends Table[DBUser](tag, Some("sidewalk_login"), "sidewalk_user") {
     def userId = column[String]("user_id", O.PrimaryKey)
     def username = column[String]("username")
     def email = column[String]("email")
@@ -17,7 +17,7 @@ object DBTableDefinitions {
 
   case class DBLoginInfo (id: Option[Long], providerID: String, providerKey: String )
 
-  class LoginInfos(tag: Tag) extends Table[DBLoginInfo](tag, "login_info") {
+  class LoginInfos(tag: Tag) extends Table[DBLoginInfo](tag, Some("sidewalk_login"), "login_info") {
     def loginInfoId = column[Long]("login_info_id", O.PrimaryKey, O.AutoInc)
     def providerID = column[String]("provider_id")
     def providerKey = column[String]("provider_key")
@@ -26,15 +26,15 @@ object DBTableDefinitions {
 
   case class DBUserLoginInfo (userID: String, loginInfoId: Long)
 
-  class UserLoginInfoTable(tag: Tag) extends Table[DBUserLoginInfo](tag, "user_login_info") {
+  class UserLoginInfoTable(tag: Tag) extends Table[DBUserLoginInfo](tag, Some("sidewalk_login"), "user_login_info") {
     def userID = column[String]("user_id", O.NotNull)
     def loginInfoId = column[Long]("login_info_id", O.NotNull)
     def * = (userID, loginInfoId) <> (DBUserLoginInfo.tupled, DBUserLoginInfo.unapply)
   }
 
   case class DBPasswordInfo (hasher: String, password: String, salt: Option[String], loginInfoId: Long)
 
-  class PasswordInfoTable(tag: Tag) extends Table[DBPasswordInfo](tag, "user_password_info") {
+  class PasswordInfoTable(tag: Tag) extends Table[DBPasswordInfo](tag, Some("sidewalk_login"), "user_password_info") {
     def hasher = column[String]("hasher")
     def password = column[String]("password")
     def salt = column[Option[String]]("salt")
@@ -44,7 +44,7 @@ object DBTableDefinitions {
 
   case class DBAuthToken (id: Array[Byte], userID: String, expirationTimestamp: Timestamp)
 
-  class AuthTokenTable(tag: Tag) extends Table[DBAuthToken](tag, "auth_tokens") {
+  class AuthTokenTable(tag: Tag) extends Table[DBAuthToken](tag, Some("sidewalk_login"), "auth_tokens") {
     def id = column[Array[Byte]]("id")
     def userID = column[String]("user_id", O.PrimaryKey)
     def expirationTimestamp = column[Timestamp]("expiration_timestamp")