Skip to content

Commit

Permalink
Update Hiveless dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
pomadchin committed Apr 9, 2022
1 parent 11bbf9d commit 76f77bf
Show file tree
Hide file tree
Showing 5 changed files with 15 additions and 44 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -59,5 +59,6 @@ java/*.so
java/*dll

# Spark files

metastore_db
derby.log
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ val shapelessVersion = "2.3.3" // to be compatible with Spark 3.1.x
val scalaTestVersion = "3.2.11"
val jtsVersion = "1.18.1"
val geomesaVersion = "3.3.0"
val hivelessVersion = "0.0.4"
val hivelessVersion = "0.0.6"
val geotrellisVersion = "3.6.2"

// GeoTrellis depends on Shapeless 2.3.7
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,19 @@
* limitations under the License.
*/

package org.apache.spark.sql.hive.carto.spatial.rules
package com.carto.analyticstoolbox.spark.rules

import com.carto.analyticstoolbox.core._
import com.carto.analyticstoolbox.index.ST_IntersectsExtent
import com.azavea.hiveless.serializers.syntax._
import org.locationtech.jts.geom.Geometry
import com.azavea.hiveless.spark.rules.syntax._
import geotrellis.vector._
import cats.syntax.option._
import org.apache.spark.sql.hive.HivelessInternals.HiveGenericUDF
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.hive.HiveGenericUDF

object SpatialFilterPushdownRules extends Rule[LogicalPlan] {

Expand All @@ -40,13 +40,15 @@ object SpatialFilterPushdownRules extends Rule[LogicalPlan] {
val extent = geometryExpr.eval(null).convert[Geometry].extent

// transform expression
val expr = List(
IsNotNull(bboxExpr),
GreaterThanOrEqual(GetStructField(bboxExpr, 0, "xmin".some), Literal(extent.xmin)),
GreaterThanOrEqual(GetStructField(bboxExpr, 1, "ymin".some), Literal(extent.ymin)),
LessThanOrEqual(GetStructField(bboxExpr, 2, "xmax".some), Literal(extent.xmax)),
LessThanOrEqual(GetStructField(bboxExpr, 3, "ymax".some), Literal(extent.ymax))
).and
val expr = AndList(
List(
IsNotNull(bboxExpr),
GreaterThanOrEqual(GetStructField(bboxExpr, 0, "xmin".some), Literal(extent.xmin)),
GreaterThanOrEqual(GetStructField(bboxExpr, 1, "ymin".some), Literal(extent.ymin)),
LessThanOrEqual(GetStructField(bboxExpr, 2, "xmax".some), Literal(extent.xmax)),
LessThanOrEqual(GetStructField(bboxExpr, 3, "ymax".some), Literal(extent.ymax))
)
)

Filter(expr, plan)
}
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@

package com.carto.analyticstoolbox

import com.carto.analyticstoolbox.spark.rules.SpatialFilterPushdownRules
import geotrellis.spark.testkit.TestEnvironment
import org.apache.spark.SparkConf
import org.apache.spark.serializer.KryoSerializer
import org.apache.spark.sql.hive.carto.spatial.rules.SpatialFilterPushdownRules
import org.apache.spark.sql.{SQLContext, SparkSession}
import org.scalatest.{BeforeAndAfterAll, Suite}

Expand Down

0 comments on commit 76f77bf

Please sign in to comment.