diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index 7c4e586d..c99e5c27 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -112,6 +112,7 @@ jobs: - {script: "integrations/workflow-orchestration/metaflow/metaflow-hello-world/helloworld.py", arg: "run"} - {script: "integrations/workflow-orchestration/metaflow/metaflow-model-evaluation/metaflow-model-evaluation.py", arg: "run --max-workers 1 --n_samples 100"} - {script: "integrations/workflow-orchestration/metaflow/metaflow-regression/metaflow-regression-example.py", arg: "run"} + - {script: "integrations/model-optimization/optuna/optuna-hello-world/optuna-hello-world.py", arg: ""} env: SCRIPT_TO_TEST: ${{ matrix.example.script }} steps: diff --git a/integrations/model-optimization/optuna/optuna-hello-world/README.md b/integrations/model-optimization/optuna/optuna-hello-world/README.md new file mode 100644 index 00000000..e1baa8bc --- /dev/null +++ b/integrations/model-optimization/optuna/optuna-hello-world/README.md @@ -0,0 +1,27 @@ +# Optuna integration with Comet.ml + +[Optuna](https://optuna.org/) is an automatic hyperparameter optimization software framework, particularly designed for machine learning. + +Log each Optuna trial to Comet to monitor in real-time the progress of your study and analyse the hyper-parameters importance, giving you full debuggability and reproducibility. + + +## See it + +Take a look at this [public Comet Project](https://www.comet.com/examples/comet-example-optuna-hello-world/view/45MrjyCtPcJPpKG2gGbkbAHZo/panels). + +## Setup + +Install dependencies + +```bash +python -m pip install -r requirements.txt +``` + +## Run the example + +This example is based on the [official quickstart example](https://colab.research.google.com/github/optuna/optuna-examples/blob/main/quickstart.ipynb). + + +```bash +python optuna-hello-world.py +``` diff --git a/integrations/model-optimization/optuna/optuna-hello-world/optuna-hello-world.py b/integrations/model-optimization/optuna/optuna-hello-world/optuna-hello-world.py new file mode 100644 index 00000000..a62ff827 --- /dev/null +++ b/integrations/model-optimization/optuna/optuna-hello-world/optuna-hello-world.py @@ -0,0 +1,32 @@ +# coding: utf-8 +from comet_ml import Experiment, init + +import optuna + +# Login to Comet if needed +init() + + +def objective(trial): + x = trial.suggest_float("x", -10, 10) + objective = (x - 2) ** 2 + + experiment = Experiment(project_name="comet-example-optuna-hello-world") + + experiment.log_optimization( + optimization_id=trial.study.study_name, + metric_name="objective", + metric_value=objective, + parameters={"x": x}, + objective="minimize", + ) + + return objective + + +study = optuna.create_study() +study.optimize(objective, n_trials=20) + +best_params = study.best_params +found_x = best_params["x"] +print("Found x: {}, (x - 2)^2: {}".format(found_x, (found_x - 2) ** 2)) diff --git a/integrations/model-optimization/optuna/optuna-hello-world/requirements.txt b/integrations/model-optimization/optuna/optuna-hello-world/requirements.txt new file mode 100644 index 00000000..2612fcbe --- /dev/null +++ b/integrations/model-optimization/optuna/optuna-hello-world/requirements.txt @@ -0,0 +1,2 @@ +comet_ml>=3.33.10 +optuna diff --git a/integrations/model-training/pytorch/pytorch-rich-logging/README.md b/integrations/model-training/pytorch/pytorch-rich-logging/README.md index f0235930..f7a26e48 100644 --- a/integrations/model-training/pytorch/pytorch-rich-logging/README.md +++ b/integrations/model-training/pytorch/pytorch-rich-logging/README.md @@ -28,5 +28,5 @@ This example is based on the tutorial from [Yunjey](https://github.com/yunjey/py ```bash -python pytorch-rich-logging.py run +python pytorch-rich-logging.py ``` diff --git a/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_keras.ipynb b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_keras.ipynb index 0843dc42..d3c44d8c 100644 --- a/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_keras.ipynb +++ b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_keras.ipynb @@ -92,7 +92,6 @@ "import numpy as np\n", "import ray\n", "from ray.air.config import RunConfig, ScalingConfig\n", - "from ray.air.integrations.keras import Callback as TrainCheckpointReportCallback\n", "from ray.air.result import Result\n", "from ray.train.tensorflow import TensorflowTrainer\n", "\n", @@ -269,7 +268,7 @@ "if num_workers < 1:\n", " num_workers = 1\n", "\n", - "train_tensorflow_mnist(num_workers, use_gpu=False, epochs=30)" + "train_tensorflow_mnist(num_workers, use_gpu=False, epochs=10)" ] } ], @@ -292,9 +291,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.15" + "version": "3.10.12" } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/integrations/reinforcement-learning/gymnasium/notebooks/comet_gymnasium_example.ipynb b/integrations/reinforcement-learning/gymnasium/notebooks/comet_gymnasium_example.ipynb index 5e4410aa..1a61b1d7 100644 --- a/integrations/reinforcement-learning/gymnasium/notebooks/comet_gymnasium_example.ipynb +++ b/integrations/reinforcement-learning/gymnasium/notebooks/comet_gymnasium_example.ipynb @@ -12,7 +12,8 @@ "\n", "**This notebook shows you how to log your Gymnasium metrics with Comet.** For more information about Comet's integration with Gymnasium, visit our [Docs](https://www.comet.com/docs/v2/integrations/ml-frameworks/gymnasium/?utm_source=gymnasium&utm_medium=partner&utm_campaign=partner_gymnasium_2023&utm_content=comet_colab) page.\n", "\n", - "If you prefer to preview what's to come, check out a completed experiment created from this notebook [here](https://www.comet.com/examples/comet-examples-gymnasium-notebook/58a1e400d18342fdabb4ddbbb07c9802?utm_source=gymnasium&utm_medium=partner&utm_campaign=partner_gymnasium_2023&utm_content=comet_colab)." + "If you prefer to preview what's to come, check out completed experiments created from this notebook [here](https://www.comet.com/examples/comet-examples-gymnasium-notebook/?utm_source=gymnasium&utm_medium=partner&utm_campaign=partner_gymnasium_2023&utm_content=comet_colab).\n", + "\n" ] }, { @@ -32,7 +33,7 @@ }, "outputs": [], "source": [ - "%pip install gymnasium[classic_control] comet_ml" + "%pip install 'gymnasium[classic-control]' comet_ml stable-baselines3" ] }, { @@ -65,7 +66,7 @@ "id": "031ezY2Dr2n4" }, "source": [ - "# Import Gymnasium and Initialize Your Enviornment" + "# Train an Agent using StableBaselines3 A2C Algorithm" ] }, { @@ -76,100 +77,72 @@ }, "outputs": [], "source": [ + "from comet_ml.integration.gymnasium import CometLogger\n", + "from stable_baselines3 import A2C\n", "import gymnasium as gym\n", "\n", "env = gym.make(\"Acrobot-v1\", render_mode=\"rgb_array\")\n", "\n", - "# Uncomment if you want to Upload Videos of your enviornment with Comet\n", - "# env = gym.wrappers.RecordVideo(env, 'test')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "g4c6nL7ysczO" - }, - "source": [ - "# Initialize your Comet Experiment and Wrap your Environment with the Comet Logger" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bxUWMLHJtCxw" - }, - "outputs": [], - "source": [ - "from comet_ml.integration.gymnasium import CometLogger\n", + "# Uncomment if you want to Upload Videos of your environment to Comet\n", + "# env = gym.wrappers.RecordVideo(env, 'test')\n", "\n", "experiment = comet_ml.Experiment()\n", "\n", - "env = CometLogger(env, experiment)" + "env = CometLogger(env, experiment)\n", + "\n", + "model = A2C(\"MlpPolicy\", env, verbose=0)\n", + "model.learn(total_timesteps=10000)\n", + "\n", + "env.close()\n", + "experiment.end()" ] }, { "cell_type": "markdown", - "metadata": { - "id": "RkHkaVn5t8O5" - }, + "metadata": {}, "source": [ - "# Step Through The Environment Randomly For 20 Episodes \n" + "# Train an Agent using StableBaselines3 PPO Algorithm" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "id": "Go-xDU-7uLl0" - }, + "metadata": {}, "outputs": [], "source": [ - "for x in range(20):\n", - "\n", - " observation, info = env.reset()\n", - " truncated = False\n", - " terminated = False\n", - " while not (truncated or terminated):\n", - " observation, reward, terminated, truncated, info = env.step(\n", - " env.action_space.sample()\n", - " )\n", - " env.render()\n", - "\n", - "env.close() # Will Upload videos to Comet if RecordVideo was used" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EzpGq4xJuWcg" - }, - "source": [ - "# View Metrics like Cumulative Episode Reward and Episode Length in Comet\n", + "from stable_baselines3 import PPO\n", + "\n", + "\n", + "env = gym.make(\"Acrobot-v1\", render_mode=\"rgb_array\")\n", + "\n", + "# Uncomment if you want to Upload Videos of your environment to Comet\n", + "# env = gym.wrappers.RecordVideo(env, 'test')\n", + "\n", + "experiment = comet_ml.Experiment()\n", + "\n", + "env = CometLogger(env, experiment)\n", "\n", - "After running an experiment, run this cell to view the Comet UI in this notebook. " + "model = PPO(\"MlpPolicy\", env, verbose=0)\n", + "model.learn(total_timesteps=10000)\n", + "\n", + "env.close()\n", + "experiment.end()" ] }, { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "wEowdeOxuqnH" - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "experiment.display(tab=\"charts\")" + "# Use Comet's UI to Benchmark Different RL Algorithims " ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "id": "yy6oDoYiKuSQ" - }, + "metadata": {}, "outputs": [], "source": [ - "experiment.end()" + "experiment.display()" ] } ], @@ -192,9 +165,14 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.10.12" + }, + "vscode": { + "interpreter": { + "hash": "8c9587381b2341d562742e36a89690be32a732b11830813473890249dd40a07d" + } } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/integrations/workflow-orchestration/vertex/vertex-hello-world/README.md b/integrations/workflow-orchestration/vertex/vertex-hello-world/README.md index ab058dc3..f6cc1197 100644 --- a/integrations/workflow-orchestration/vertex/vertex-hello-world/README.md +++ b/integrations/workflow-orchestration/vertex/vertex-hello-world/README.md @@ -4,6 +4,9 @@ Comet integrates with Google Vertex AI. [Google Vertex AI](https://cloud.google.com/vertex-ai/) lets you build, deploy, and scale ML models faster, with pre-trained and custom tooling within a unified artificial intelligence platform. +> [!NOTE] +> This example uses the first version of the KFP package + ## Documentation For more information on using and configuring the Vertex integration, see: [https://www.comet.com/docs/v2/integrations/third-party-tools/vertex-ai/](https://www.comet.com/docs/v2/integrations/third-party-tools/vertex-ai/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=vertex) diff --git a/integrations/workflow-orchestration/vertex/vertex-hello-world/requirements.txt b/integrations/workflow-orchestration/vertex/vertex-hello-world/requirements.txt index ba66183a..ac2df04a 100644 --- a/integrations/workflow-orchestration/vertex/vertex-hello-world/requirements.txt +++ b/integrations/workflow-orchestration/vertex/vertex-hello-world/requirements.txt @@ -1,3 +1,3 @@ -comet_ml>=3.33.7 +comet_ml>=3.33.10 google-cloud-aiplatform kfp<2 diff --git a/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/README.md b/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/README.md new file mode 100644 index 00000000..7b3cda4f --- /dev/null +++ b/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/README.md @@ -0,0 +1,29 @@ +# Vertex AI integration with Comet.ml + +Comet integrates with Google Vertex AI. + +[Google Vertex AI](https://cloud.google.com/vertex-ai/) lets you build, deploy, and scale ML models faster, with pre-trained and custom tooling within a unified artificial intelligence platform. + +## Documentation + +For more information on using and configuring the Vertex integration, see: [https://www.comet.com/docs/v2/integrations/third-party-tools/vertex-ai/](https://www.comet.com/docs/v2/integrations/third-party-tools/vertex-ai/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=vertex) + +## See it + +Take a look at this [public Comet Project](https://www.comet.com/examples/comet-example-vertex-v2-hello-world/view/mz2vYWFTYZ3vNzgWIK0r4ZRUR/panels?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=vertex). + +## Setup + +Install dependencies + +```bash +python -m pip install -r requirements.txt +``` + +## Run the example + +The following example demonstrates how to use the Comet pipelines integration to track the state of pipelines run on Vertex. Before running, make sure that you are correctly authenticated against your Google Cloud Platform account and project, the easiest way to do so is by using the [Google Cloud CLI](https://cloud.google.com/sdk/docs/). + +```bash +python demo_pipeline.py +``` diff --git a/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/demo_pipeline.py b/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/demo_pipeline.py new file mode 100644 index 00000000..9ad754a1 --- /dev/null +++ b/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/demo_pipeline.py @@ -0,0 +1,104 @@ +# coding: utf-8 +import os + +from comet_ml import init + +import google.cloud.aiplatform as aip +from kfp import compiler, dsl + +# Login to Comet if needed +init() + + +COMET_PROJECT_NAME = "comet-example-vertex-v2-hello-world" + + +@dsl.component(packages_to_install=["comet_ml"]) +def data_preprocessing(a: str = None) -> str: + import math + import random + import time + + import comet_ml + + experiment = comet_ml.Experiment() + + for i in range(60): + experiment.log_metric("accuracy", math.log(i + random.random())) + time.sleep(0.1) + experiment.end() + + return a + + +@dsl.component(packages_to_install=["comet_ml"]) +def model_training(a: str = None) -> str: + import math + import random + import time + + import comet_ml + + experiment = comet_ml.Experiment() + + for i in range(60): + experiment.log_metric("accuracy", math.log(i + random.random())) + time.sleep(0.1) + experiment.end() + + return a + + +@dsl.component(packages_to_install=["comet_ml"]) +def model_evaluation(a: str = None, b: str = None) -> str: + import math + import random + import time + + import comet_ml + + experiment = comet_ml.Experiment() + + for i in range(60): + experiment.log_metric("accuracy", math.log(i + random.random())) + time.sleep(0.1) + experiment.end() + + return a + + +@dsl.pipeline(name="comet-integration-example") +def pipeline(): + import comet_ml.integration.vertex + + logger = comet_ml.integration.vertex.CometVertexPipelineLogger( + # api_key=XXX, + project_name=COMET_PROJECT_NAME, + # workspace=XXX + share_api_key_to_workers=True, + ) + + task_1 = logger.track_task(data_preprocessing(a="test")) + + task_2 = logger.track_task(model_training(a=task_1.output)) + + task_3 = logger.track_task(model_training(a=task_1.output)) + + _ = logger.track_task(model_evaluation(a=task_2.output, b=task_3.output)) + + +if __name__ == "__main__": + print("Running pipeline") + compiler.Compiler().compile( + pipeline_func=pipeline, package_path="demo_pipeline.json" + ) + + job = aip.PipelineJob( + display_name="comet-integration-example", + template_path="demo_pipeline.json", + pipeline_root=os.getenv("PIPELINE_ROOT"), + project=os.getenv("GCP_PROJECT"), + enable_caching=False, + ) + + job.submit() diff --git a/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/requirements.txt b/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/requirements.txt new file mode 100644 index 00000000..8964b029 --- /dev/null +++ b/integrations/workflow-orchestration/vertex/vertex-v2-hello-world/requirements.txt @@ -0,0 +1,4 @@ +comet_ml>=3.33.10 +google-cloud-aiplatform +# Ignore versions impacted by https://github.com/kubeflow/pipelines/issues/9974 +kfp>=2,!=2.1.3