diff --git a/build.sbt b/build.sbt index 5dc6821..cc3ec4d 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ val shapelessVersion = "2.3.3" // to be compatible with Spark 3.1.x val scalaTestVersion = "3.2.11" val jtsVersion = "1.18.1" val geomesaVersion = "3.3.0" -val hivelessVersion = "0.0.7" +val hivelessVersion = "0.0.8" val geotrellisVersion = "3.6.2" // GeoTrellis depends on Shapeless 2.3.7 diff --git a/core/src/main/scala/com/carto/analyticstoolbox/spark/rules/SpatialFilterPushdownRules.scala b/core/src/main/scala/com/carto/analyticstoolbox/spark/rules/SpatialFilterPushdownRules.scala index 593cf83..9c6693e 100644 --- a/core/src/main/scala/com/carto/analyticstoolbox/spark/rules/SpatialFilterPushdownRules.scala +++ b/core/src/main/scala/com/carto/analyticstoolbox/spark/rules/SpatialFilterPushdownRules.scala @@ -19,6 +19,7 @@ package com.carto.analyticstoolbox.spark.rules import com.carto.analyticstoolbox.core._ import com.carto.analyticstoolbox.index._ import com.azavea.hiveless.spark.rules.syntax._ +import com.azavea.hiveless.serializers.UnaryDeserializer.Errors.ProductDeserializationError import com.azavea.hiveless.serializers.syntax._ import geotrellis.vector._ import cats.syntax.option._ @@ -37,7 +38,6 @@ object SpatialFilterPushdownRules extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan.transformDown { - // HiveGenericUDF is a private[hive] case class case f @ Filter(condition: HiveGenericUDF, plan) if condition.of[ST_Intersects] => try { val Seq(extentExpr, geometryExpr) = condition.children @@ -50,15 +50,16 @@ object SpatialFilterPushdownRules extends Rule[LogicalPlan] { ) // transform expression - val expr = Try { - // ST_Intersects is polymorphic by the second argument - // Extract Extent literal from the right - // The second argument can be Geometry or Extent - val g = geometryExpr.eval(null) - Try(g.convert[Geometry].extent).getOrElse(g.convert[Extent]) - } match { + val expr = Try(geometryExpr.eval(null)) match { // Literals push-down support only - case Success(extent) => + case Success(g) => + // ST_Intersects is polymorphic by the second argument + // Extract Extent literal from the right + // The second argument can be Geometry or Extent + val extent = Try(g.convert[Geometry].extent) + .orElse(Try(g.convert[Extent])) + .getOrElse(throw ProductDeserializationError[ST_Intersects.Arg](classOf[ST_Intersects], "second")) + // transform expression AndList( List(