-
Notifications
You must be signed in to change notification settings - Fork 436
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add Skypilot Kubernetes integration for remote orchestration This commit adds the Skypilot Kubernetes integration to enable remote orchestration of ZenML pipelines on VMs. The integration includes the necessary initialization, configuration, and flavor classes. It also adds the Skypilot Kubernetes orchestrator and its settings. This integration provides an alternative to the local orchestrator for running pipelines on Kubernetes. * Add Skypilot Kubernetes integration for remote orchestration * Refactor SkypilotKubernetesOrchestrator prepare_environment_variable method * Refactor SkypilotKubernetesOrchestrator prepare_environment_variable method * Refactor SkypilotKubernetesOrchestrator prepare_environment_variable method * Refactor SkypilotBaseOrchestrator setup_credentials method * Refactor SkypilotBaseOrchestrator setup_credentials method and SkypilotKubernetesOrchestrator prepare_environment_variable method * Refactor SkypilotBaseOrchestrator setup_credentials method and SkypilotKubernetesOrchestrator prepare_environment_variable method * Update src/zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py Co-authored-by: Stefan Nica <[email protected]> * Refactor SkypilotKubernetesOrchestrator to handle missing service connector * Refactor SkypilotBaseOrchestrator to use the correct virtual environment path for Kubernetes * specll check * fix run command * Refactor SkypilotKubernetesVMOrchestrator to remove unnecessary code * Refactor SkypilotKubernetesVMOrchestrator to remove unnecessary code --------- Co-authored-by: Stefan Nica <[email protected]>
- Loading branch information
1 parent
3eb3d0d
commit b711963
Showing
9 changed files
with
419 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
# Copyright (c) ZenML GmbH 2024. All Rights Reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at: | ||
# | ||
# https://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express | ||
# or implied. See the License for the specific language governing | ||
# permissions and limitations under the License. | ||
"""Initialization of the Skypilot Kubernetes integration for ZenML. | ||
The Skypilot integration sub-module powers an alternative to the local | ||
orchestrator for a remote orchestration of ZenML pipelines on VMs. | ||
""" | ||
from typing import List, Type | ||
|
||
from zenml.integrations.constants import ( | ||
SKYPILOT_KUBERNETES, | ||
) | ||
from zenml.integrations.integration import Integration | ||
from zenml.stack import Flavor | ||
|
||
SKYPILOT_KUBERNETES_ORCHESTRATOR_FLAVOR = "vm_kubernetes" | ||
|
||
|
||
class SkypilotKubernetesIntegration(Integration): | ||
"""Definition of Skypilot Kubernetes Integration for ZenML.""" | ||
|
||
NAME = SKYPILOT_KUBERNETES | ||
# all 0.6.x versions of skypilot[kubernetes] are compatible | ||
REQUIREMENTS = ["skypilot[kubernetes]~=0.6.1"] | ||
APT_PACKAGES = ["openssh-client", "rsync"] | ||
|
||
@classmethod | ||
def flavors(cls) -> List[Type[Flavor]]: | ||
"""Declare the stack component flavors for the Skypilot Kubernetes integration. | ||
Returns: | ||
List of stack component flavors for this integration. | ||
""" | ||
from zenml.integrations.skypilot_kubernetes.flavors import ( | ||
SkypilotKubernetesOrchestratorFlavor, | ||
) | ||
|
||
return [SkypilotKubernetesOrchestratorFlavor] | ||
|
||
|
||
SkypilotKubernetesIntegration.check_installation() |
26 changes: 26 additions & 0 deletions
26
src/zenml/integrations/skypilot_kubernetes/flavors/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
# Copyright (c) ZenML GmbH 2024. All Rights Reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at: | ||
# | ||
# https://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express | ||
# or implied. See the License for the specific language governing | ||
# permissions and limitations under the License. | ||
"""Skypilot integration flavor for Skypilot Kubernetes orchestrator.""" | ||
|
||
from zenml.integrations.skypilot_kubernetes.flavors.skypilot_orchestrator_kubernetes_vm_flavor import ( | ||
SkypilotKubernetesOrchestratorConfig, | ||
SkypilotKubernetesOrchestratorFlavor, | ||
SkypilotKubernetesOrchestratorSettings, | ||
) | ||
|
||
__all__ = [ | ||
"SkypilotKubernetesOrchestratorConfig", | ||
"SkypilotKubernetesOrchestratorFlavor", | ||
"SkypilotKubernetesOrchestratorSettings", | ||
] |
Oops, something went wrong.