feature_pipeline.yaml

name: ML Pipelines

on:
  schedule:
    - cron: '0 * * * *'  # Runs every hour
  push:
    branches:
      - main
      
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  ml_pipelines:
    name: ML Pipelines
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v3
        
      - name: Install uv
        uses: astral-sh/setup-uv@v3
        with:
          version: 0.4.30
          
      - name: Get notebook list
        id: ml_pipelines
        run: |
          echo "notebooks=$(ls notebooks/*.ipynb | sort -n | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT

    strategy:
      matrix:
        notebook: ${{ fromJson(steps.notebooks.outputs.notebooks) }}

      - name: Set up Python
        run: uv python install

      - name: Install the project
        run: |
          uv sync --all-extras --dev

      - name: Run pipeline
        run: uv run ipython ${{ matrix.ml_pipelines }}
        env:
          HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}