Merge branch 'main' of github.com:zenml-io/zenml-projects

.github/workflows/production_run_complete_llm.yml

@@ -14,17 +14,18 @@ jobs:
   run-staging-workflow:
     runs-on: ubuntu-latest
     env:
-      ZENML_HOST: ${{ secrets.ZENML_HOST }}
-      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
-      ZENML_PRODUCTION_STACK : 51a49786-b82a-4646-bde7-a460efb0a9c5
+      ZENML_HOST: ${{ secrets.ZENML_PROJECTS_HOST }}
+      ZENML_API_KEY: ${{ secrets.ZENML_PROJECTS_API_KEY }}
+      ZENML_PRODUCTION_STACK: b3951d43-0fb2-4d32-89c5-3399374e7c7e # Set this to your production stack ID
       ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
       ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }}
       ZENML_DEBUG: true
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_LOGGING_VERBOSITY: INFO
       ZENML_PROJECT_SECRET_NAME: llm-complete
       ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True
-      ZENML_ACTION_ID: 23a4d58c-bd2b-47d5-a41d-0a845d2982f8
+      ZENML_EVENT_SOURCE_ID: ae6ae536-d811-4838-a44b-744b768a0f31  # Set this to your preferred event source ID
+      ZENML_SERVICE_ACCOUNT_ID: fef76af2-382f-4ab2-9e6b-5eb85a303f0e  # Set this to your service account ID or delete
 
     steps:
       - name: Check out repository code
@@ -55,4 +56,4 @@ jobs:
       - name: Run pipeline, create pipeline, configure trigger (Production)
         working-directory: ./llm-complete-guide
         run: |
-          python gh_action_rag.py --no-cache --create-template --action-id  ${{ env.ZENML_ACTION_ID }} --config rag_gcp.yaml
+          python gh_action_rag.py --no-cache --create-template ----event-source-id  --service-account-id ${{ env.ZENML_SERVICE_ACCOUNT_ID }} --action-id  ${{ env.ZENML_ACTION_ID }} --config rag_gcp.yaml

.github/workflows/staging_run_complete_llm.yml

@@ -12,9 +12,9 @@ jobs:
   run-staging-workflow:
     runs-on: ubuntu-latest
     env:
-      ZENML_HOST: ${{ secrets.ZENML_HOST }}
-      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
-      ZENML_STAGING_STACK: 51a49786-b82a-4646-bde7-a460efb0a9c5
+      ZENML_HOST: ${{ secrets.ZENML_PROJECTS_HOST }}
+      ZENML_API_KEY: ${{ secrets.ZENML_PROJECTS_API_KEY }}
+      ZENML_STAGING_STACK : 67166d73-a44e-42f9-b67f-011e9afab9b5 # Set this to your staging stack ID
       ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
       ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }}
       ZENML_DEBUG: true
@@ -36,7 +36,7 @@ jobs:
         run: |
           pip3 install -r requirements.txt
           pip3 install -r requirements-argilla.txt
-          zenml integration install gcp -y
+          zenml integration install aws s3 -y
 
       - name: Connect to ZenML server
         working-directory: ./llm-complete-guide

llm-complete-guide/configs/embeddings.yaml

@@ -16,7 +16,7 @@ settings:
       - pandas
       - numpy
       - sentence-transformers>=3
-      - transformers[torch]
+      - transformers[torch]==4.43.1
       - litellm
       - ollama
       - tiktoken

llm-complete-guide/configs/synthetic.yaml

@@ -13,7 +13,7 @@ settings:
       - pandas
       - numpy
       - sentence-transformers>=3
-      - transformers
+      - transformers==4.43.1
       - litellm
       - ollama
       - tiktoken

llm-complete-guide/gh_action_rag.py

@@ -22,6 +22,7 @@
 import click
 import yaml
 from zenml.client import Client
+from zenml.exceptions import ZenKeyError
 
 from pipelines.llm_basic_rag import llm_basic_rag
 
@@ -53,16 +54,23 @@
     help="Specify a configuration file"
 )
 @click.option(
-    "--action-id",
-    "action_id",
+    "--service-account-id",
+    "service_account_id",
     default=None,
-    help="Specify an action ID"
+    help="Specify a service account ID"
+)
+@click.option(
+    "--event-source-id",
+    "event_source_id",
+    default=None,
+    help="Specify an event source ID"
 )
 def main(
     no_cache: bool = False,
     config: Optional[str]= "rag_local_dev.yaml",
     create_template: bool = False,
-    action_id: Optional[str] = None
+    service_account_id: Optional[str] = None,
+    event_source_id: Optional[str] = None
 ):
     """
     Executes the pipeline to train a basic RAG model.
@@ -72,6 +80,8 @@ def main(
         config (str): The path to the configuration file.
         create_template (bool): If `True`, a run template will be created.
         action_id (str): The action ID.
+        service_account_id (str): The service account ID.
+        event_source_id (str): The event source ID.
     """
     client = Client()
     config_path = Path(__file__).parent / "configs" / config
@@ -80,6 +90,7 @@ def main(
         config = yaml.safe_load(file)
 
     if create_template:
+
         # run pipeline
         run = llm_basic_rag.with_options(
             config_path=str(config_path),
@@ -90,14 +101,49 @@ def main(
             name=f"production-llm-complete-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}",
             deployment_id=run.deployment_id
         )
-        # update the action with the new template
-        client.update_action(
-            name_id_or_prefix=UUID(action_id),
-            configuration={
-                "template_id": str(rt.id),
-                "run_config": pop_restricted_configs(config)
-            }
-        )
+
+        try:
+            # Check if an action ahs already be configured for this pipeline
+            action = client.get_action(
+                name_id_or_prefix="LLM Complete (production)",
+                allow_name_prefix_match=True
+            )
+        except ZenKeyError:
+            if not event_source_id:
+                raise RuntimeError("An event source is required for this workflow.")
+
+            if not service_account_id:
+                service_account_id = client.create_service_account(
+                    name="github-action-sa",
+                    description="To allow triggered pipelines to run with M2M authentication."
+                ).id
+
+            action_id = client.create_action(
+                name="LLM Complete (production)",
+                configuration={
+                    "template_id": str(rt.id),
+                    "run_config": pop_restricted_configs(config)
+                },
+                service_account_id=service_account_id,
+                auth_window=0,
+            ).id
+            client.create_trigger(
+                name="Production Trigger LLM-Complete",
+                event_source_id=UUID(event_source_id),
+                event_filter={"event_type": "tag_event"},
+                action_id=action_id,
+                description="Trigger pipeline to reindex everytime the docs are updated through git."
+            )
+        else:
+            # update the action with the new template
+            # here we can assume the trigger is fully set up already
+            client.update_action(
+                name_id_or_prefix=action.id,
+                configuration={
+                    "template_id": str(rt.id),
+                    "run_config": pop_restricted_configs(config)
+                }
+            )
 
     else:
         llm_basic_rag.with_options(

llm-complete-guide/steps/finetune_embeddings.py

@@ -268,7 +268,7 @@ def finetune(
         tf32=False,  # use tf32 precision
         bf16=False,  # use bf16 precision
         batch_sampler=BatchSamplers.NO_DUPLICATES,  # MultipleNegativesRankingLoss benefits from no duplicate samples in a batch
-        eval_strategy="no",  # evaluate after each epoch
+        eval_strategy="epoch",  # evaluate after each epoch
         save_strategy="epoch",  # save after each epoch
         logging_steps=10,  # log every 10 steps
         save_total_limit=3,  # save only the last 3 models