Merge branch 'develop' into feature/better-metadata

zenml-io · Oct 21, 2024 · adab934 · adab934
2 parents 1e1991a + e5c8b22
commit adab934
Show file tree

Hide file tree

Showing 154 changed files with 1,511 additions and 6,793 deletions.
diff --git a/.gitbook.yaml b/.gitbook.yaml
@@ -15,3 +15,5 @@ redirects:
   getting-started/deploying-zenml/manage-the-deployed-services/upgrade-the-version-of-the-zenml-server.md: how-to/manage-the-zenml-server/upgrade-zenml-server.md
   getting-started/deploying-zenml/manage-the-deployed-services/troubleshoot-your-deployed-server.md: how-to/manage-the-zenml-server/troubleshoot-your-deployed-server.md
   how-to/stack-deployment/implement-a-custom-integration.md: how-to/contribute-to-zenml/implement-a-custom-integration.md
+
+  getting-started/zenml-pro/system-architectures: getting-started/system-architectures.md
diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
@@ -160,11 +160,6 @@ jobs:
       - name: Create virtual environment
         run: |
           uv venv
-      - name: Check mlstacks compatibility
-        run: |
-          source .venv/bin/activate
-          uv pip install -e .
-          uv pip install mlstacks
       - name: Check for broken dependencies
         run: |
           source .venv/bin/activate

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -10,35 +10,6 @@ jobs:
       os: arc-runner-set
       python-version: '3.9'
     secrets: inherit
-  mlstacks-compatibility-check:
-    needs: setup-and-test
-    runs-on: arc-runner-set
-    steps:
-      - name: Checkout code
-        uses: actions/[email protected]
-      - name: Set up Python
-        uses: actions/[email protected]
-        with:
-          python-version: '3.9'
-      - name: Install uv
-        run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          source $HOME/.cargo/env
-      - name: Create virtual environment
-        run: |
-          source $HOME/.cargo/env
-          uv venv
-      - name: Check mlstacks compatibility
-        run: |
-          source .venv/bin/activate
-          source $HOME/.cargo/env
-          uv pip install -e .
-          uv pip install mlstacks
-      - name: Check for broken dependencies
-        run: |
-          source .venv/bin/activate
-          source $HOME/.cargo/env
-          uv pip check
   mysql-db-migration-testing:
     runs-on: arc-runner-set
     env:
@@ -101,7 +72,6 @@ jobs:
     if: github.repository == 'zenml-io/zenml'
     needs:
       - setup-and-test
-      - mlstacks-compatibility-check
       - sqlite-db-migration-testing
       - mysql-db-migration-testing
       - mariadb-db-migration-testing

diff --git a/.github/workflows/scarf-check.yml b/.github/workflows/scarf-check.yml
diff --git a/.gitignore b/.gitignore
@@ -193,8 +193,5 @@ examples/feast_feature_store/feast_feature_repo/data/registry.db
 #generated folder by zenml
 zenml_tutorial/
 
-# script for testing
-mlstacks_reset.sh
-
 .local/
 # PLEASE KEEP THIS LINE AT THE EOF: never include here src/zenml/zen_server/dashboard, since it is affecting release flow
diff --git a/.test_durations b/.test_durations
@@ -1226,18 +1226,6 @@
     "tests/unit/utils/test_io_utils.py::test_resolve_relative_path": 0.0019732369999019284,
     "tests/unit/utils/test_io_utils.py::test_write_file_contents_as_string_fails_with_non_string_types": 0.001999538000063694,
     "tests/unit/utils/test_io_utils.py::test_write_file_contents_as_string_works": 0.0021266399999149144,
-    "tests/unit/utils/test_mlstacks_utils.py::test_click_params_to_mlstacks_conversion": 0.11761479299991606,
-    "tests/unit/utils/test_mlstacks_utils.py::test_component_construction_works_for_component_deploy": 0.05780457900016245,
-    "tests/unit/utils/test_mlstacks_utils.py::test_component_construction_works_for_stack_deploy": 0.061128635000045506,
-    "tests/unit/utils/test_mlstacks_utils.py::test_component_flavor_parsing_works": 0.0018454319999818836,
-    "tests/unit/utils/test_mlstacks_utils.py::test_config_addition_works": 0.03295685900002354,
-    "tests/unit/utils/test_mlstacks_utils.py::test_extra_config_validation": 0.0014914250000401807,
-    "tests/unit/utils/test_mlstacks_utils.py::test_get_stack_spec_file_path_fails_when_no_stack": 0.009623762999922292,
-    "tests/unit/utils/test_mlstacks_utils.py::test_get_stack_spec_file_path_only_works_with_full_name": 0.01112438799998472,
-    "tests/unit/utils/test_mlstacks_utils.py::test_get_stack_spec_file_path_works": 0.008664245999966624,
-    "tests/unit/utils/test_mlstacks_utils.py::test_spec_file_exists_works_when_no_file": 0.06172314599996298,
-    "tests/unit/utils/test_mlstacks_utils.py::test_stack_construction_works_for_stack_deploy": 0.4371748089999983,
-    "tests/unit/utils/test_mlstacks_utils.py::test_stack_exists_works": 0.0230911900001729,
     "tests/unit/utils/test_networking_utils.py::test_find_available_port_works": 0.0015159350000431004,
     "tests/unit/utils/test_networking_utils.py::test_port_available_works": 0.002061952000076417,
     "tests/unit/utils/test_networking_utils.py::test_port_is_open_on_local_host_works": 0.0015869370001837524,

diff --git a/docs/book/.gitbook/assets/Scenario1.png b/docs/book/.gitbook/assets/Scenario1.png
diff --git a/docs/book/.gitbook/assets/Scenario2.png b/docs/book/.gitbook/assets/Scenario2.png
diff --git a/docs/book/.gitbook/assets/Scenario3.2.png b/docs/book/.gitbook/assets/Scenario3.2.png
diff --git a/docs/book/.gitbook/assets/add_org_members.png b/docs/book/.gitbook/assets/add_org_members.png
diff --git a/docs/book/.gitbook/assets/add_team_members.png b/docs/book/.gitbook/assets/add_team_members.png
diff --git a/docs/book/.gitbook/assets/admin_role.png b/docs/book/.gitbook/assets/admin_role.png
diff --git a/docs/book/.gitbook/assets/assign_permissions.png b/docs/book/.gitbook/assets/assign_permissions.png
diff --git a/docs/book/.gitbook/assets/assign_team_to_tenant.png b/docs/book/.gitbook/assets/assign_team_to_tenant.png
diff --git a/docs/book/.gitbook/assets/business-logic-tenants.png b/docs/book/.gitbook/assets/business-logic-tenants.png
diff --git a/docs/book/.gitbook/assets/cloud_architecture_saas_detailed.png b/docs/book/.gitbook/assets/cloud_architecture_saas_detailed.png
diff --git a/docs/book/.gitbook/assets/cloud_architecture_saas_detailed_2.png b/docs/book/.gitbook/assets/cloud_architecture_saas_detailed_2.png
diff --git a/docs/book/.gitbook/assets/cloud_architecture_scenario_1.png b/docs/book/.gitbook/assets/cloud_architecture_scenario_1.png
diff --git a/docs/book/.gitbook/assets/cloud_architecture_scenario_1_1.png b/docs/book/.gitbook/assets/cloud_architecture_scenario_1_1.png
diff --git a/docs/book/.gitbook/assets/cloud_architecture_scenario_2.png b/docs/book/.gitbook/assets/cloud_architecture_scenario_2.png
diff --git a/docs/book/.gitbook/assets/cloud_architecture_scenario_3.png b/docs/book/.gitbook/assets/cloud_architecture_scenario_3.png
diff --git a/docs/book/.gitbook/assets/cloud_architecture_scenario_5.png b/docs/book/.gitbook/assets/cloud_architecture_scenario_5.png
diff --git a/.../assets/cloud_architecture_scenario_4.png → ...oud_architecture_self_hosted_detailed.png b/.../assets/cloud_architecture_scenario_4.png → ...oud_architecture_self_hosted_detailed.png
diff --git a/docs/book/.gitbook/assets/cloudapi_swagger.png b/docs/book/.gitbook/assets/cloudapi_swagger.png
diff --git a/docs/book/.gitbook/assets/core_concepts_oss.png b/docs/book/.gitbook/assets/core_concepts_oss.png
diff --git a/docs/book/.gitbook/assets/create_role_modal.png b/docs/book/.gitbook/assets/create_role_modal.png
diff --git a/docs/book/.gitbook/assets/create_team.png b/docs/book/.gitbook/assets/create_team.png
diff --git a/docs/book/.gitbook/assets/custom_role_settings_page.png b/docs/book/.gitbook/assets/custom_role_settings_page.png
diff --git a/docs/book/.gitbook/assets/new_tenant.png b/docs/book/.gitbook/assets/new_tenant.png
diff --git a/docs/book/.gitbook/assets/new_tenant_modal.png b/docs/book/.gitbook/assets/new_tenant_modal.png
diff --git a/docs/book/.gitbook/assets/org_hierarchy_pro.png b/docs/book/.gitbook/assets/org_hierarchy_pro.png
diff --git a/docs/book/.gitbook/assets/org_members.png b/docs/book/.gitbook/assets/org_members.png
diff --git a/docs/book/.gitbook/assets/org_settings.png b/docs/book/.gitbook/assets/org_settings.png
diff --git a/docs/book/.gitbook/assets/oss_simple_deployment.png b/docs/book/.gitbook/assets/oss_simple_deployment.png
diff --git a/docs/book/.gitbook/assets/pro_deployment_simple.png b/docs/book/.gitbook/assets/pro_deployment_simple.png
diff --git a/docs/book/.gitbook/assets/rbac_admin_permissions.png b/docs/book/.gitbook/assets/rbac_admin_permissions.png
diff --git a/docs/book/.gitbook/assets/role_page.png b/docs/book/.gitbook/assets/role_page.png
diff --git a/docs/book/.gitbook/assets/share_dialog.png b/docs/book/.gitbook/assets/share_dialog.png
diff --git a/docs/book/.gitbook/assets/staging-production-tenants.png b/docs/book/.gitbook/assets/staging-production-tenants.png
diff --git a/docs/book/.gitbook/assets/swagger_docs_zenml.png b/docs/book/.gitbook/assets/swagger_docs_zenml.png
diff --git a/docs/book/.gitbook/assets/team_roles.png b/docs/book/.gitbook/assets/team_roles.png
diff --git a/docs/book/.gitbook/assets/tenant_onboarding.png b/docs/book/.gitbook/assets/tenant_onboarding.png
diff --git a/docs/book/.gitbook/assets/tenant_roles_page.png b/docs/book/.gitbook/assets/tenant_roles_page.png
diff --git a/docs/book/component-guide/artifact-stores/gcp.md b/docs/book/component-guide/artifact-stores/gcp.md
@@ -30,7 +30,6 @@ or [the ZenML GCP Terraform module](../../how-to/stack-deployment/deploy-a-cloud
 for a shortcut on how to deploy & register this stack component.
 {% endhint %}
 
-
 The GCS Artifact Store flavor is provided by the GCP ZenML integration, you need to install it on your local machine to be able to register a GCS Artifact Store and add it to your stack:
 
 ```shell
@@ -51,16 +50,6 @@ zenml stack register custom_stack -a gs_store ... --set
 
 Depending on your use case, however, you may also need to provide additional configuration parameters pertaining to [authentication](gcp.md#authentication-methods) to match your deployment scenario.
 
-#### Infrastructure Deployment
-
-A GCS Artifact Store can be deployed directly from the ZenML CLI:
-
-```shell
-zenml artifact-store deploy gcs_artifact_store --flavor=gcp --provider=gcp ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the [dedicated documentation section](../../how-to/stack-deployment/README.md).
-
 #### Authentication Methods
 
 Integrating and using a GCS Artifact Store in your pipelines is not possible without employing some form of authentication. If you're looking for a quick way to get started locally, you can use the _Implicit Authentication_ method. However, the recommended way to authenticate to the GCP cloud platform is through [a GCP Service Connector](../../how-to/auth-management/gcp-service-connector.md). This is particularly useful if you are configuring ZenML stacks that combine the GCS Artifact Store with other remote stack components also running in GCP.

diff --git a/docs/book/component-guide/artifact-stores/s3.md b/docs/book/component-guide/artifact-stores/s3.md
@@ -50,15 +50,6 @@ zenml stack register custom_stack -a s3_store ... --set
 
 Depending on your use case, however, you may also need to provide additional configuration parameters pertaining to [authentication](s3.md#authentication-methods) or [pass advanced configuration parameters](s3.md#advanced-configuration) to match your S3-compatible service or deployment scenario.
 
-#### Infrastructure Deployment
-
-An S3 Artifact Store can be deployed directly from the ZenML CLI:
-
-```shell
-zenml artifact-store deploy s3-artifact-store --flavor=s3 --provider=aws ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the [dedicated documentation section](../../how-to/stack-deployment/README.md).
 
 #### Authentication Methods
 

diff --git a/docs/book/component-guide/container-registries/aws.md b/docs/book/component-guide/container-registries/aws.md
@@ -49,16 +49,6 @@ To figure out the URI for your registry:
 * Go [here](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints) and choose the region in which you would like to store your container images. Make sure to choose a nearby region for faster access.
 * Once you have both these values, fill in the values in this template `<ACCOUNT_ID>.dkr.ecr.<REGION>.amazonaws.com` to get your container registry URI.
 
-#### Infrastructure Deployment
-
-An AWS ECR Container Registry can be deployed directly from the ZenML CLI:
-
-```shell
-zenml container-registry deploy ecr_container_registry --flavor=aws --provider=aws ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the dedicated documentation section.
-
 ### How to use it
 
 To use the AWS container registry, we need:

diff --git a/docs/book/component-guide/container-registries/gcp.md b/docs/book/component-guide/container-registries/gcp.md
@@ -33,16 +33,6 @@ When using the Google Artifact Registry, you need to:
 * enable it [here](https://console.cloud.google.com/marketplace/product/google/artifactregistry.googleapis.com)
 * go [here](https://console.cloud.google.com/artifacts) and create a `Docker` repository.
 
-### Infrastructure Deployment
-
-A GCP Container Registry can be deployed directly from the ZenML CLI:
-
-```shell
-zenml container-registry deploy gcp_container_registry --flavor=gcp --provider=gcp ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the [dedicated documentation section](../../how-to/stack-deployment/deploy-a-stack-using-mlstacks.md).
-
 ## How to find the registry URI
 
 When using the Google Artifact Registry, the GCP container registry URI should have the following format:
@@ -61,16 +51,6 @@ To figure out the URI for your registry:
 * Go [here](https://console.cloud.google.com/artifacts) and select the repository that you want to use to store Docker images. If you don't have a repository yet, take a look at the [deployment section](gcp.md#how-to-deploy-it).
 * On the top, click the copy button to copy the full repository URL.
 
-#### Infrastructure Deployment
-
-A GCP Container Registry can be deployed directly from the ZenML CLI:
-
-```shell
-zenml container-registry deploy gcp_container_registry --flavor=gcp --provider=gcp ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the dedicated documentation section.
-
 ### How to use it
 
 To use the GCP container registry, we need:

diff --git a/docs/book/component-guide/model-deployers/seldon.md b/docs/book/component-guide/model-deployers/seldon.md
@@ -50,20 +50,6 @@ Since the Seldon Model Deployer is interacting with the Seldon Core model server
 
 In addition to these parameters, the Seldon Core Model Deployer may also require additional configuration to be set up to allow it to authenticate to the remote artifact store or persistent storage service where model artifacts are located. This is covered in the [Managing Seldon Core Authentication](seldon.md#managing-seldon-core-authentication) section.
 
-{% hint style="info" %}
-Configuring Seldon Core in a Kubernetes cluster can be a complex and error-prone process, so we have provided a set of Terraform-based recipes to quickly provision popular combinations of MLOps tools. More information about these recipes can be found in the [MLOps Stack Recipes](https://github.com/zenml-io/mlstacks).
-{% endhint %}
-
-### Infrastructure Deployment
-
-The Seldon Model Deployer can be deployed directly from the ZenML CLI:
-
-```shell
-zenml model-deployer deploy seldon_deployer --flavor=seldon --provider=<YOUR_PROVIDER> ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the [dedicated documentation section](../../how-to/stack-deployment/README.md).
-
 ### Seldon Core Installation Example
 
 The following example briefly shows how you can install Seldon in an EKS Kubernetes cluster. It assumes that the EKS cluster itself is already set up and configured with IAM access. For more information or tutorials for other clouds, check out the [official Seldon Core installation instructions](https://github.com/SeldonIO/seldon-core/tree/master/examples/auth#demo-setup).

diff --git a/docs/book/component-guide/orchestrators/airflow.md b/docs/book/component-guide/orchestrators/airflow.md
@@ -30,18 +30,17 @@ setup is necessary.
 
 There are many options to use a deployed Airflow server:
 
-* Use one of [ZenML's Airflow stack recipes](https://github.com/zenml-io/mlstacks). This is the simplest solution to
-  get ZenML working with Airflow, as the recipe also takes care of additional steps such as installing required Python
-  dependencies in your Airflow server environment.
+* Use [the ZenML GCP Terraform module](../../how-to/stack-deployment/deploy-a-cloud-stack-with-terraform.md)
+  which includes a [Google Cloud Composer](https://cloud.google.com/composer) component.
 * Use a managed deployment of Airflow such as [Google Cloud Composer](https://cloud.google.com/composer)
   , [Amazon MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/),
   or [Astronomer](https://www.astronomer.io/).
 * Deploy Airflow manually. Check out the
   official [Airflow docs](https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html) for more
   information.
 
-If you're not using `mlstacks` to deploy Airflow, there are some additional Python packages that you'll need to
-install in the Python environment of your Airflow server:
+If you're not using the ZenML GCP Terraform module to deploy Airflow, there are some additional Python
+packages that you'll need to install in the Python environment of your Airflow server:
 
 * `pydantic~=2.7.1`: The Airflow DAG files that ZenML creates for you require Pydantic to parse and validate
   configuration files.

diff --git a/docs/book/component-guide/orchestrators/azureml.md b/docs/book/component-guide/orchestrators/azureml.md
@@ -69,8 +69,7 @@ Azure hosting environments and credentials used in local development.
 of service principals on Azure to allow you to connect your cloud components 
 with proper authentication. For this method, you will need to [create a service 
 principal on Azure](https://learn.microsoft.com/en-us/azure/developer/python/sdk/authentication-on-premises-apps?tabs=azure-portal), 
-assign it the correct permissions and use it to [register a ZenML Azure Service 
-Connector](https://docs.zenml.io/how-to/auth-management/azure-service-connector).
+assign it the correct permissions and use it to [register a ZenML Azure Service Connector](../../how-to/auth-management/azure-service-connector.md).
     ```bash
     zenml service-connector register <CONNECTOR_NAME> --type azure -i
     zenml orchestrator connect <ORCHESTRATOR_NAME> -c <CONNECTOR_NAME>

diff --git a/docs/book/component-guide/orchestrators/kubeflow.md b/docs/book/component-guide/orchestrators/kubeflow.md
@@ -88,16 +88,6 @@ If one or more of the deployments are not in the `Running` state, try increasing
 If you're installing Kubeflow Pipelines manually, make sure the Kubernetes service is called exactly `ml-pipeline`. This is a requirement for ZenML to connect to your Kubeflow Pipelines deployment.
 {% endhint %}
 
-#### Infrastructure Deployment
-
-A Kubeflow orchestrator can be deployed directly from the ZenML CLI:
-
-```shell
-zenml orchestrator deploy kubeflow_orchestrator --flavor=kubeflow --provider=<YOUR_PROVIDER> ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the dedicated documentation section.
-
 ### How to use it
 
 To use the Kubeflow orchestrator, we need:

diff --git a/docs/book/component-guide/orchestrators/kubernetes.md b/docs/book/component-guide/orchestrators/kubernetes.md
@@ -28,20 +28,10 @@ You should use the Kubernetes orchestrator if:
 
 ### How to deploy it
 
-The Kubernetes orchestrator requires a Kubernetes cluster in order to run. There are many ways to deploy a Kubernetes cluster using different cloud providers or on your custom infrastructure, and we can't possibly cover all of them, but you can check out our cloud guide
+The Kubernetes orchestrator requires a Kubernetes cluster in order to run. There are many ways to deploy a Kubernetes cluster using different cloud providers or on your custom infrastructure, and we can't possibly cover all of them, but you can check out our [our cloud guide](../../user-guide/cloud-guide/cloud-guide.md).
 
 If the above Kubernetes cluster is deployed remotely on the cloud, then another pre-requisite to use this orchestrator would be to deploy and connect to a [remote ZenML server](../../getting-started/deploying-zenml/README.md).
 
-#### Infrastructure Deployment
-
-A Kubernetes orchestrator can be deployed directly from the ZenML CLI:
-
-```shell
-zenml orchestrator deploy k8s_orchestrator --flavor=kubernetes --provider=<YOUR_PROVIDER> ...
-```
-
-You can pass other configurations specific to the stack components as key-value arguments. If you don't provide a name, a random one is generated for you. For more information about how to work use the CLI for this, please refer to the dedicated documentation section.
-
 ### How to use it
 
 To use the Kubernetes orchestrator, we need: