Adding CI pipelines

e2e_samples/fabric_dataops_sample/devops/azure-pipelines-ci-artifacts.yml

@@ -0,0 +1,93 @@
+trigger:
+  branches:
+    include:
+      - <DEV_BRANCH_NAME>
+  paths:
+    include:
+      - $(WORKING_DIR)/*
+
+pr: none
+
+variables:
+  - group: <VARIABLE_GROUP_NAME>
+
+stages:
+  - stage: 'publish_artifacts'
+    displayName: 'Publish Build Artifacts'
+    jobs:
+
+      - job: 'publish_fabric_environment_config_artifacts'
+        displayName: 'Publish Fabric Environment Config Artifacts'
+
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.8'
+              architecture: 'x64'
+
+          - script: |
+              pip install -r requirements.txt
+            displayName: 'Install Requirements'
+            workingDirectory: $(WORKING_DIR)/libraries
+
+          - script: |
+              echo "Linting files in $(WORKING_DIR)..."
+              flake8 .
+            displayName: 'Linting Files'
+            workingDirectory: $(WORKING_DIR)/libraries
+
+          - script: |
+              echo "Running tests in $(WORKING_DIR)/libraries/test..."
+              pytest .
+            displayName: 'Run Tests'
+            workingDirectory: $(WORKING_DIR)/libraries
+
+          - script: |
+              mkdir fabric_env/
+              cp fabric/fabric_environment/environment.yml fabric_env/
+
+              mkdir fabric_env/custom_libraries/
+              cp libraries/src/ddo_transform_standardize.py fabric_env/custom_libraries/
+              cp libraries/src/ddo_transform_transform.py fabric_env/custom_libraries/
+              cp libraries/src/otel_monitor_invoker.py fabric_env/custom_libraries/
+
+            displayName: 'Build Fabric Environment Config Artifacts'
+            workingDirectory: $(WORKING_DIR)
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              PathtoPublish: $(WORKING_DIR)/fabric_env/
+              ArtifactName: 'fabric_env'
+            displayName: 'Publish Fabric Environment Config Files'
+
+      - job: 'publish_adls_artifacts'
+        displayName: 'Publish ADLS Artifacts'
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+
+          - task: AzureCLI@2
+            inputs:
+              azureSubscription: 'fabric-cicd-service-connection'
+              scriptType: 'bash'
+              scriptLocation: 'inlineScript'
+              inlineScript: |
+                FABRIC_BEARER_TOKEN=$(az account get-access-token --resource https://api.fabric.microsoft.com --query accessToken -o tsv)
+
+                echo "##vso[task.setvariable variable=FABRIC_BEARER_TOKEN]$FABRIC_BEARER_TOKEN"
+              displayName: 'Get Fabric Bearer Token'
+
+          - script: |
+              chmod +x devops/scripts/build_adls_files.sh
+              ./devops/scripts/build_adls_files.sh
+            displayName: 'Create artifacts for ADLS'
+            workingDirectory: $(WORKING_DIR)
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              PathtoPublish: '$(WORKING_DIR)/adls/'
+              ArtifactName: 'adls'
+            displayName: 'Publish ADLS Artifacts'

e2e_samples/fabric_dataops_sample/devops/azure-pipelines-ci-qa-cleanup.yaml

@@ -0,0 +1,49 @@
+trigger: none
+
+variables:
+  - group: <VARIABLE_GROUP_NAME>
+  - name: STORAGE_CONTAINER_NAME
+    value: "feature-$(PR_ID)"
+
+jobs:
+  - job: CleanupWorkspace
+    displayName: 'Cleanup Resources after PR Merge or Abandon'
+    pool:
+      vmImage: 'ubuntu-latest'
+    condition: always()
+    steps:
+
+      - script: |
+          echo "Triggering interactive Azure CLI login..."
+          az login --use-device-code
+        displayName: 'Azure CLI - Interactive Login'
+
+      - script: |
+          FABRIC_BEARER_TOKEN=$(az account get-access-token --resource https://api.fabric.microsoft.com --query accessToken -o tsv)
+          AZURE_MANAGEMENT_BEARER_TOKEN=$(az account get-access-token --resource https://management.azure.com --query accessToken -o tsv)
+          AZURE_STORAGE_BEARER_TOKEN=$(az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv)
+
+          # Set the tokens as pipeline variables
+          echo "##vso[task.setvariable variable=FABRIC_BEARER_TOKEN]$FABRIC_BEARER_TOKEN"
+          echo "##vso[task.setvariable variable=AZURE_MANAGEMENT_BEARER_TOKEN]$AZURE_MANAGEMENT_BEARER_TOKEN"
+          echo "##vso[task.setvariable variable=AZURE_STORAGE_BEARER_TOKEN]$AZURE_STORAGE_BEARER_TOKEN"
+
+        displayName: 'Get Access Tokens'
+        name: GetAccessTokens
+
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: '3.8'
+          addToPath: true
+
+      - script: |
+          pip install -r $(WORKING_DIR)/devops/scripts/requirements.txt
+        displayName: 'Install Requirements'
+
+      - script: |
+          echo "Running Cleanup Script"
+          export FABRIC_WORKSPACE_NAME="$FABRIC_WORKSPACE_NAME-$PR_ID"
+          export FABRIC_CONNECTION_NAME="$FABRIC_CONNECTION_NAME-$PR_ID"
+
+          python -u $(WORKING_DIR)/devops/scripts/cleanup_workspace.py --verbose
+        displayName: 'Cleanup Resources'

e2e_samples/fabric_dataops_sample/devops/azure-pipelines-ci-qa.yml

@@ -0,0 +1,163 @@
+trigger: none
+
+pr:
+  branches:
+    include:
+      - <DEV_BRANCH_NAME>
+  paths:
+    include:
+      - $(WORKING_DIR)/fabric/*
+      - $(WORKING_DIR)/libraries/*
+      - $(WORKING_DIR)/config/*
+
+variables:
+  - group: <VARIABLE_GROUP_NAME>
+  - name: FEATURE_BRANCH
+    value: $[replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '')]
+  - name: PR_ID
+    value: "$(System.PullRequest.PullRequestId)"
+  - name: COMMIT_HASH
+    value: "$(Build.SourceVersion)"
+  - name: FABRIC_WORKSPACE_DIRECTORY
+    value: "$(WORKING_DIR)/fabric/workspace"
+  - name: STORAGE_CONTAINER_NAME
+    value: "feature-$(PR_ID)"
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+stages:
+  - stage: BuildLibraries
+    jobs:
+      - job: BuildLibraries
+        displayName: 'Build and Test Libraries'
+        workspace:
+          clean: all
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.8'
+              addToPath: true
+
+          - script: |
+              pip install -r requirements.txt
+            displayName: 'Install Requirements'
+            workingDirectory: $(WORKING_DIR)/libraries
+
+          - script: |
+              echo "Linting files in $(WORKING_DIR)..."
+              flake8 .
+            displayName: 'Linting Files'
+            workingDirectory: $(WORKING_DIR)/libraries
+
+          - script: |
+              echo "Running tests in $(WORKING_DIR)/libraries/test..."
+              pytest .
+            displayName: 'Run Tests'
+            workingDirectory: $(WORKING_DIR)/libraries
+
+  - stage: BuildFabric
+    jobs:
+      - job: BuildFeatureFabricWorkspace
+        displayName: 'Build Feature Fabric Workspace'
+        steps:
+          - script: |
+              echo "Triggering interactive Azure CLI login..."
+              az login --use-device-code
+            displayName: 'Azure CLI - Interactive Login'
+
+          - script: |
+              FABRIC_BEARER_TOKEN=$(az account get-access-token --resource https://api.fabric.microsoft.com --query accessToken -o tsv)
+              AZURE_MANAGEMENT_BEARER_TOKEN=$(az account get-access-token --resource https://management.azure.com --query accessToken -o tsv)
+              AZURE_STORAGE_BEARER_TOKEN=$(az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv)
+
+              # Set the tokens as pipeline variables
+              echo "##vso[task.setvariable variable=FABRIC_BEARER_TOKEN]$FABRIC_BEARER_TOKEN"
+              echo "##vso[task.setvariable variable=AZURE_MANAGEMENT_BEARER_TOKEN]$AZURE_MANAGEMENT_BEARER_TOKEN"
+              echo "##vso[task.setvariable variable=AZURE_STORAGE_BEARER_TOKEN]$AZURE_STORAGE_BEARER_TOKEN"
+
+              echo "##vso[task.setvariable variable=FABRIC_BEARER_TOKEN;isOutput=true]$FABRIC_BEARER_TOKEN"
+            displayName: 'Get Access Tokens'
+            name: GetAccessTokens
+
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.8'
+              addToPath: true
+
+          - script: |
+              pip install -r $(WORKING_DIR)/devops/scripts/requirements.txt
+            displayName: 'Install Requirements'
+
+          - script: |
+              export FABRIC_WORKSPACE_NAME="$FABRIC_WORKSPACE_NAME-$PR_ID"
+              export FABRIC_CONNECTION_NAME="$FABRIC_CONNECTION_NAME-$PR_ID"
+
+              python -u $(WORKING_DIR)/devops/scripts/build_workspace.py --verbose
+            displayName: 'Build Workspace'
+
+          - script: |
+              export FABRIC_WORKSPACE_NAME="$FABRIC_WORKSPACE_NAME-$PR_ID"
+
+              # Get the list of changed files in the current commit or PR
+              CHANGED_FILES=$(git diff --name-only $(System.PullRequest.SourceCommitId) $(System.PullRequest.TargetCommitId))
+
+              # If any of the changed files are in the fabric_environment directory, set the variable UPDATE_PUBLIC_LIBRARIES to true
+              if echo "$CHANGED_FILES" | grep -q "$(WORKING_DIR)/fabric/fabric_environment/"; then
+                echo "Changes detected in fabric_environment. Setting variable UPDATE_PUBLIC_LIBRARIES to true."
+                echo "##vso[task.setvariable variable=UPDATE_PUBLIC_LIBRARIES]true"
+              else
+                echo "No changes detected in fabric_environment."
+              fi
+
+              # If any of the changed files are in the libraries directory, set the variable UPDATE_CUSTOM_LIBRARIES to true
+              if echo "$CHANGED_FILES" | grep -q "$(WORKING_DIR)/libraries/"; then
+                echo "Changes detected in the libraries. Setting variable UPDATE_CUSTOM_LIBRARIES to true."
+                echo "##vso[task.setvariable variable=UPDATE_CUSTOM_LIBRARIES]true"
+              else
+                echo "No changes detected in the libraries."
+              fi
+
+              python -u $(WORKING_DIR)/devops/scripts/update_environment.py --verbose
+            displayName: 'Update Environment'
+
+          - script: |
+              export FABRIC_WORKSPACE_NAME="$FABRIC_WORKSPACE_NAME-$PR_ID"
+
+              chmod +x devops/scripts/build_adls_files.sh
+              ./devops/scripts/build_adls_files.sh
+
+              az storage blob upload-batch \
+                --account-name $STORAGE_ACCOUNT_NAME \
+                -d $STORAGE_CONTAINER_NAME \
+                -s adls \
+                --auth-mode login \
+                --overwrite true \
+                --output none
+
+            displayName: 'Upload ADLS Config Artifacts'
+            workingDirectory: $(WORKING_DIR)
+
+      - job: FabricTests
+        displayName: 'Test Feature Fabric Workspace'
+        dependsOn: BuildFeatureFabricWorkspace
+        variables:
+          FABRIC_BEARER_TOKEN: $[ dependencies.BuildFeatureFabricWorkspace.outputs['GetAccessTokens.FABRIC_BEARER_TOKEN'] ]
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.8'
+              addToPath: true
+
+          - script: |
+              pip install -r requirements.txt
+            displayName: 'Install Requirements'
+            workingDirectory: $(WORKING_DIR)/fabric/
+
+          - script: |
+              export FABRIC_WORKSPACE_NAME="$FABRIC_WORKSPACE_NAME-$PR_ID"
+
+              echo "Running tests in $(WORKING_DIR)/fabric/test..."
+              pytest -o log_cli=true --capture=no test/
+            displayName: 'Run Tests'
+            workingDirectory: $(WORKING_DIR)/fabric/