From bd7a76f779cd98af4da9af49e09f65847321e6aa Mon Sep 17 00:00:00 2001 From: Han Wang Date: Fri, 16 Aug 2024 06:24:09 +0000 Subject: [PATCH] Update Tune --- .devcontainer/devcontainer.json | 59 +++++++++++++++++++----------- .pre-commit-config.yaml | 6 ++- requirements.txt | 3 +- setup.py | 8 ++-- tests/tune/concepts/test_flow.py | 8 ++-- tests/tune_sklearn/test_suggest.py | 13 +++++-- tune_tensorflow/spec.py | 4 +- tune_version/__init__.py | 2 +- 8 files changed, 65 insertions(+), 38 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 06f078c..afd5392 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,29 +1,46 @@ { "name": "Fugue Development Environment", - "image": "fugueproject/devenv:latest", - "settings": { - "terminal.integrated.shell.linux": "/bin/bash", - "python.pythonPath": "/usr/local/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" + "image": "mcr.microsoft.com/vscode/devcontainers/python:3.10", + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash", + "python.pythonPath": "/usr/local/bin/python", + "python.defaultInterpreterPath": "/usr/local/bin/python", + "editor.defaultFormatter": "ms-python.black-formatter", + "isort.interpreter": [ + "/usr/local/bin/python" + ], + "flake8.interpreter": [ + "/usr/local/bin/python" + ], + "pylint.interpreter": [ + "/usr/local/bin/python" + ], + "black-formatter.interpreter": [ + "/usr/local/bin/python" + ] + }, + "extensions": [ + "ms-python.python", + "ms-python.isort", + "ms-python.flake8", + "ms-python.pylint", + "ms-python.mypy", + "ms-python.black-formatter", + "GitHub.copilot", + "njpwerner.autodocstring" + ] + } }, - "extensions": [ - "ms-python.python" - ], "forwardPorts": [ 8888 ], "postCreateCommand": "make devenv", - "mounts": [ - "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" - ] + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2.11.0": {}, + "ghcr.io/devcontainers/features/java:1": { + "version": "11" + } + } } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 499de78..0a15b5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,9 +34,13 @@ repos: - flake8-tidy-imports - pycodestyle - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.971 hooks: - id: mypy + - repo: https://github.com/pre-commit/mirrors-pylint + rev: v2.6.0 + hooks: + - id: pylint - repo: https://github.com/ambv/black rev: 22.3.0 hooks: diff --git a/requirements.txt b/requirements.txt index 15d338d..4263206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,8 +19,7 @@ sphinx-autodoc-typehints nbsphinx flask -pyspark -dask[dataframe,distributed] +fugue[spark,dask] # publish to pypi wheel diff --git a/setup.py b/setup.py index 87bbaac..88f1d57 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def get_version() -> str: author_email="goodwanghan@gmail.com", keywords="hyper parameter hyperparameter tuning tune tuner optimzation", url="http://github.com/fugue-project/tune", - install_requires=["fugue", "cloudpickle", "triad>=0.8.4"], + install_requires=["fugue", "cloudpickle", "triad>=0.8.4", "fs"], extras_require={ "hyperopt": ["hyperopt"], "optuna": ["optuna"], @@ -54,14 +54,14 @@ def get_version() -> str: "Intended Audience :: Developers", "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", ], - python_requires=">=3.6", + python_requires=">=3.8", entry_points={ "tune.plugins": [ "mlflow = tune_mlflow[mlflow]", diff --git a/tests/tune/concepts/test_flow.py b/tests/tune/concepts/test_flow.py index 427f6c8..47c70d6 100644 --- a/tests/tune/concepts/test_flow.py +++ b/tests/tune/concepts/test_flow.py @@ -56,7 +56,7 @@ def test_trial_report(): report = copy.copy( TrialReport( trial, - metric=np.float(0.1), + metric=np.float64(0.1), params={"c": Rand(1, 2)}, metadata={"d": 4}, cost=2.0, @@ -73,7 +73,7 @@ def test_trial_report(): assert 0.1 == report.sort_metric report = copy.deepcopy( - TrialReport(trial, metric=np.float(0.111), cost=2.0, rung=4, sort_metric=1.23) + TrialReport(trial, metric=np.float64(0.111), cost=2.0, rung=4, sort_metric=1.23) ) assert trial is report.trial report = cloudpickle.loads(cloudpickle.dumps(report)) @@ -109,7 +109,7 @@ def test_trial_report(): assert 5 == report.with_rung(5).rung td = trial.with_dfs({"a": pd.DataFrame}) - report = TrialReport(td, metric=np.float(0.1)) + report = TrialReport(td, metric=np.float64(0.1)) assert 0 == len(report.trial.dfs) @@ -177,7 +177,7 @@ def test_trial_report_heap(): def test_trial_decision(): trial = Trial("abc", {"a": 1}, {"b": Rand(0, 2)}) report = TrialReport( - trial, metric=np.float(0.1), params={"c": Rand(0, 3)}, metadata={"d": 4} + trial, metric=np.float64(0.1), params={"c": Rand(0, 3)}, metadata={"d": 4} ) decision = TrialDecision( report, budget=0.0, should_checkpoint=True, metadata={"x": 1}, reason="p" diff --git a/tests/tune_sklearn/test_suggest.py b/tests/tune_sklearn/test_suggest.py index f6e2c91..b512233 100644 --- a/tests/tune_sklearn/test_suggest.py +++ b/tests/tune_sklearn/test_suggest.py @@ -1,6 +1,7 @@ -from fugue_dask import DaskExecutionEngine +import fugue.test as ft from sklearn.datasets import load_diabetes from sklearn.linear_model import Lasso, LinearRegression + from tune import TUNE_OBJECT_FACTORY, Grid, Rand from tune_hyperopt.optimizer import HyperoptLocalOptimizer from tune_sklearn import sk_space, suggest_sk_models, suggest_sk_models_by_cv @@ -8,6 +9,7 @@ # from fugue_spark import SparkExecutionEngine +@ft.with_backend("dask") def test_suggest(tmpdir): TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir)) @@ -31,6 +33,7 @@ def test_suggest(tmpdir): top_n=0, distributed=False, local_optimizer=HyperoptLocalOptimizer(max_iter=10, seed=0), + execution_engine="native", ) assert 4 == len(result) assert 50 > result[0].sort_metric @@ -44,6 +47,7 @@ def test_suggest(tmpdir): partition_keys=["sex"], temp_path=str(tmpdir), save_model=True, + execution_engine="native", ) assert 16 == len(result) assert 50 > result[0].sort_metric @@ -55,12 +59,13 @@ def test_suggest(tmpdir): "neg_mean_absolute_error", top_n=1, partition_keys=["sex"], - execution_engine=DaskExecutionEngine, + execution_engine="dask", ) assert 2 == len(result) assert 50 > result[0].sort_metric +@ft.with_backend("dask") def test_suggest_cv(tmpdir): TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir)) @@ -80,6 +85,7 @@ def test_suggest_cv(tmpdir): top_n=0, distributed=False, local_optimizer=HyperoptLocalOptimizer(max_iter=10, seed=0), + execution_engine="native", ) assert 4 == len(result) assert 50 > result[0].sort_metric @@ -92,6 +98,7 @@ def test_suggest_cv(tmpdir): partition_keys=["sex"], temp_path=str(tmpdir), save_model=True, + execution_engine="native", ) assert 16 == len(result) assert 50 > result[0].sort_metric @@ -102,7 +109,7 @@ def test_suggest_cv(tmpdir): "neg_mean_absolute_error", top_n=1, partition_keys=["sex"], - execution_engine=DaskExecutionEngine, + execution_engine="dask", ) assert 2 == len(result) assert 50 > result[0].sort_metric diff --git a/tune_tensorflow/spec.py b/tune_tensorflow/spec.py index 8c70c97..989716b 100644 --- a/tune_tensorflow/spec.py +++ b/tune_tensorflow/spec.py @@ -39,13 +39,13 @@ def get_model(self) -> keras.models.Model: raise NotImplementedError # pragma: no cover def save_checkpoint(self, fs: FSBase, model: keras.models.Model) -> None: - with tempfile.NamedTemporaryFile(suffix=".h5") as tf: + with tempfile.NamedTemporaryFile(suffix=".weights.h5") as tf: model.save_weights(tf.name) with open(tf.name, "rb") as fin: fs.writefile("model.h5", fin) def load_checkpoint(self, fs: FSBase, model: keras.models.Model) -> None: - with tempfile.NamedTemporaryFile(suffix=".h5") as tf: + with tempfile.NamedTemporaryFile(suffix=".weights.h5") as tf: local_fs = FileSystem() with fs.open("model.h5", "rb") as fin: local_fs.writefile(tf.name, fin) diff --git a/tune_version/__init__.py b/tune_version/__init__.py index 1276d02..0a8da88 100644 --- a/tune_version/__init__.py +++ b/tune_version/__init__.py @@ -1 +1 @@ -__version__ = "0.1.5" +__version__ = "0.1.6"