diff --git a/requirements.txt b/requirements.txt index e7adf0af..77a38209 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -py4j==0.10.7.0 +py4j mlflow==1.8.0 pyyaml -mleap==0.15.0 +mleap==0.16.0 graphviz requests gorilla==0.3.0 @@ -10,7 +10,7 @@ pyspark-dist-explore==0.1.8 numpy pandas scipy -pyspark>=2.4.0,<=2.4.5 -h2o-pysparkling-2.4==3.28.1.2-1 +pyspark>=3.0.1 +h2o-pysparkling-3.0==3.32.0.4-1 IPython cloudpickle==1.6.0 diff --git a/requirements3.txt b/requirements2.txt similarity index 61% rename from requirements3.txt rename to requirements2.txt index 77a38209..e7adf0af 100644 --- a/requirements3.txt +++ b/requirements2.txt @@ -1,7 +1,7 @@ -py4j +py4j==0.10.7.0 mlflow==1.8.0 pyyaml -mleap==0.16.0 +mleap==0.15.0 graphviz requests gorilla==0.3.0 @@ -10,7 +10,7 @@ pyspark-dist-explore==0.1.8 numpy pandas scipy -pyspark>=3.0.1 -h2o-pysparkling-3.0==3.32.0.4-1 +pyspark>=2.4.0,<=2.4.5 +h2o-pysparkling-2.4==3.28.1.2-1 IPython cloudpickle==1.6.0 diff --git a/splicemachine/mlflow_support/mlflow_support.py b/splicemachine/mlflow_support/mlflow_support.py index 964160ff..2b997f2c 100644 --- a/splicemachine/mlflow_support/mlflow_support.py +++ b/splicemachine/mlflow_support/mlflow_support.py @@ -63,8 +63,6 @@ import requests import sklearn import yaml -# from h2o.model.model_base import ModelBase as H2OModel TODO: For Spark3 -from h2o.estimators.estimator_base import ModelBase as H2OModel from pandas.core.frame import DataFrame as PandasDF from pyspark.ml.base import Model as SparkModel from pyspark.sql import DataFrame as SparkDF @@ -78,6 +76,11 @@ from splicemachine import SpliceMachineException from splicemachine.spark.context import PySpliceContext +try: # PySpark/H2O 3.X + from h2o.model.model_base import ModelBase as H2OModel +except: # PySpark/H2O 2.X + from h2o.estimators.estimator_base import ModelBase as H2OModel + # For recording notebook history try: from IPython import get_ipython