Update Tune

.devcontainer/devcontainer.json

@@ -1,29 +1,46 @@
 {
 	"name": "Fugue Development Environment",
-	"image": "fugueproject/devenv:latest",
-	"settings": {
-		"terminal.integrated.shell.linux": "/bin/bash",
-		"python.pythonPath": "/usr/local/bin/python",
-		"python.linting.enabled": true,
-		"python.linting.pylintEnabled": true,
-		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+	"image": "mcr.microsoft.com/vscode/devcontainers/python:3.10",
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.shell.linux": "/bin/bash",
+				"python.pythonPath": "/usr/local/bin/python",
+				"python.defaultInterpreterPath": "/usr/local/bin/python",
+				"editor.defaultFormatter": "ms-python.black-formatter",
+				"isort.interpreter": [
+					"/usr/local/bin/python"
+				],
+				"flake8.interpreter": [
+					"/usr/local/bin/python"
+				],
+				"pylint.interpreter": [
+					"/usr/local/bin/python"
+				],
+				"black-formatter.interpreter": [
+					"/usr/local/bin/python"
+				]
+			},
+			"extensions": [
+				"ms-python.python",
+				"ms-python.isort",
+				"ms-python.flake8",
+				"ms-python.pylint",
+				"ms-python.mypy",
+				"ms-python.black-formatter",
+				"GitHub.copilot",
+				"njpwerner.autodocstring"
+			]
+		}
 	},
-	"extensions": [
-		"ms-python.python"
-	],
 	"forwardPorts": [
 		8888
 	],
 	"postCreateCommand": "make devenv",
-	"mounts": [
-		"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
-	]
+	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2.11.0": {},
+		"ghcr.io/devcontainers/features/java:1": {
+			"version": "11"
+		}
+	}
 }

.pre-commit-config.yaml

@@ -34,9 +34,13 @@ repos:
             - flake8-tidy-imports
             - pycodestyle
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v0.782
+      rev: v0.971
       hooks:
         - id: mypy
+    - repo: https://github.com/pre-commit/mirrors-pylint
+      rev: v2.6.0
+      hooks:
+        - id: pylint
     - repo: https://github.com/ambv/black
       rev: 22.3.0
       hooks:

requirements.txt

@@ -19,8 +19,7 @@ sphinx-autodoc-typehints
 nbsphinx
 flask
 
-pyspark
-dask[dataframe,distributed]
+fugue[spark,dask]
 
 # publish to pypi
 wheel

setup.py

@@ -30,7 +30,7 @@ def get_version() -> str:
     author_email="[email protected]",
     keywords="hyper parameter hyperparameter tuning tune tuner optimzation",
     url="http://github.com/fugue-project/tune",
-    install_requires=["fugue", "cloudpickle", "triad>=0.8.4"],
+    install_requires=["fugue", "cloudpickle", "triad>=0.8.4", "fs"],
     extras_require={
         "hyperopt": ["hyperopt"],
         "optuna": ["optuna"],
@@ -54,14 +54,14 @@ def get_version() -> str:
         "Intended Audience :: Developers",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3 :: Only",
     ],
-    python_requires=">=3.6",
+    python_requires=">=3.8",
     entry_points={
         "tune.plugins": [
             "mlflow = tune_mlflow[mlflow]",

tests/tune/concepts/test_flow.py

@@ -56,7 +56,7 @@ def test_trial_report():
     report = copy.copy(
         TrialReport(
             trial,
-            metric=np.float(0.1),
+            metric=np.float64(0.1),
             params={"c": Rand(1, 2)},
             metadata={"d": 4},
             cost=2.0,
@@ -73,7 +73,7 @@ def test_trial_report():
     assert 0.1 == report.sort_metric
 
     report = copy.deepcopy(
-        TrialReport(trial, metric=np.float(0.111), cost=2.0, rung=4, sort_metric=1.23)
+        TrialReport(trial, metric=np.float64(0.111), cost=2.0, rung=4, sort_metric=1.23)
     )
     assert trial is report.trial
     report = cloudpickle.loads(cloudpickle.dumps(report))
@@ -109,7 +109,7 @@ def test_trial_report():
     assert 5 == report.with_rung(5).rung
 
     td = trial.with_dfs({"a": pd.DataFrame})
-    report = TrialReport(td, metric=np.float(0.1))
+    report = TrialReport(td, metric=np.float64(0.1))
     assert 0 == len(report.trial.dfs)
 
 
@@ -177,7 +177,7 @@ def test_trial_report_heap():
 def test_trial_decision():
     trial = Trial("abc", {"a": 1}, {"b": Rand(0, 2)})
     report = TrialReport(
-        trial, metric=np.float(0.1), params={"c": Rand(0, 3)}, metadata={"d": 4}
+        trial, metric=np.float64(0.1), params={"c": Rand(0, 3)}, metadata={"d": 4}
     )
     decision = TrialDecision(
         report, budget=0.0, should_checkpoint=True, metadata={"x": 1}, reason="p"

tests/tune_sklearn/test_suggest.py

@@ -1,13 +1,15 @@
-from fugue_dask import DaskExecutionEngine
+import fugue.test as ft
 from sklearn.datasets import load_diabetes
 from sklearn.linear_model import Lasso, LinearRegression
+
 from tune import TUNE_OBJECT_FACTORY, Grid, Rand
 from tune_hyperopt.optimizer import HyperoptLocalOptimizer
 from tune_sklearn import sk_space, suggest_sk_models, suggest_sk_models_by_cv
 
 # from fugue_spark import SparkExecutionEngine
 
 
+@ft.with_backend("dask")
 def test_suggest(tmpdir):
     TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
 
@@ -31,6 +33,7 @@ def test_suggest(tmpdir):
         top_n=0,
         distributed=False,
         local_optimizer=HyperoptLocalOptimizer(max_iter=10, seed=0),
+        execution_engine="native",
     )
     assert 4 == len(result)
     assert 50 > result[0].sort_metric
@@ -44,6 +47,7 @@ def test_suggest(tmpdir):
         partition_keys=["sex"],
         temp_path=str(tmpdir),
         save_model=True,
+        execution_engine="native",
     )
     assert 16 == len(result)
     assert 50 > result[0].sort_metric
@@ -55,12 +59,13 @@ def test_suggest(tmpdir):
         "neg_mean_absolute_error",
         top_n=1,
         partition_keys=["sex"],
-        execution_engine=DaskExecutionEngine,
+        execution_engine="dask",
     )
     assert 2 == len(result)
     assert 50 > result[0].sort_metric
 
 
+@ft.with_backend("dask")
 def test_suggest_cv(tmpdir):
     TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
 
@@ -80,6 +85,7 @@ def test_suggest_cv(tmpdir):
         top_n=0,
         distributed=False,
         local_optimizer=HyperoptLocalOptimizer(max_iter=10, seed=0),
+        execution_engine="native",
     )
     assert 4 == len(result)
     assert 50 > result[0].sort_metric
@@ -92,6 +98,7 @@ def test_suggest_cv(tmpdir):
         partition_keys=["sex"],
         temp_path=str(tmpdir),
         save_model=True,
+        execution_engine="native",
     )
     assert 16 == len(result)
     assert 50 > result[0].sort_metric
@@ -102,7 +109,7 @@ def test_suggest_cv(tmpdir):
         "neg_mean_absolute_error",
         top_n=1,
         partition_keys=["sex"],
-        execution_engine=DaskExecutionEngine,
+        execution_engine="dask",
     )
     assert 2 == len(result)
     assert 50 > result[0].sort_metric

tune_tensorflow/spec.py

@@ -39,13 +39,13 @@ def get_model(self) -> keras.models.Model:
         raise NotImplementedError  # pragma: no cover
 
     def save_checkpoint(self, fs: FSBase, model: keras.models.Model) -> None:
-        with tempfile.NamedTemporaryFile(suffix=".h5") as tf:
+        with tempfile.NamedTemporaryFile(suffix=".weights.h5") as tf:
             model.save_weights(tf.name)
             with open(tf.name, "rb") as fin:
                 fs.writefile("model.h5", fin)
 
     def load_checkpoint(self, fs: FSBase, model: keras.models.Model) -> None:
-        with tempfile.NamedTemporaryFile(suffix=".h5") as tf:
+        with tempfile.NamedTemporaryFile(suffix=".weights.h5") as tf:
             local_fs = FileSystem()
             with fs.open("model.h5", "rb") as fin:
                 local_fs.writefile(tf.name, fin)

tune_version/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.5"
+__version__ = "0.1.6"