Merge branch 'main' of https://github.com/DataCloud-project/SIM-PIPE …

…into resource-prediction

simulator/.dockerignore

@@ -0,0 +1,14 @@
+# ignore environment variables files
+.env
+
+# virtual environment
+venv/
+.venv/
+
+# ignore standard files and folders
+__pycache__/
+.git/
+.vscode/
+
+# notebooks
+*.ipynb

simulator/Dockerfile

@@ -0,0 +1,35 @@
+
+# Use an official Python runtime as a parent image
+FROM python:3.11-slim-buster
+
+# Variables
+ENV KWOK_REPO=kubernetes-sigs/kwok
+ENV KWOK_LATEST_RELEASE=v0.0.0
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+
+# Install necessary packages
+RUN apt-get update && \
+    apt-get install -y curl gzip ca-certificates apt-transport-https && \
+    curl -sSL https://get.docker.com | sh && \
+    curl -sSL -o /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64" && \
+    chmod +x /usr/local/bin/kind && \
+    curl -sLO "https://github.com/argoproj/argo-workflows/releases/download/v3.4.13/argo-linux-amd64.gz" && \
+    gzip -d argo-linux-amd64.gz && \
+    mv ./argo-linux-amd64 /usr/local/bin/argo && \
+    chmod +x /usr/local/bin/argo && \
+    curl -sSL -o /usr/local/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubectl" && \
+    chmod +x /usr/local/bin/kubectl && \
+    curl -sSL -o /usr/local/bin/kwokctl "https://github.com/kubernetes-sigs/kwok/releases/download/v0.4.0/kwokctl-linux-amd64" && \
+    chmod +x /usr/local/bin/kwokctl && \
+    curl -sSL -o /usr/local/bin/kwok "https://github.com/kubernetes-sigs/kwok/releases/download/v0.4.0/kwok-linux-amd64" && \    
+    chmod +x /usr/local/bin/kwok && \
+    pip install poetry && \
+    poetry install && \
+    rm -rf /var/lib/apt/lists/*
+
+CMD [ "poetry", "run", "python", "run_simulation.py", "-v"]

simulator/README.md

@@ -0,0 +1,35 @@
+# Simulate pipeline
+Setup a KWOK cluster and submit an argo workflow to the cluster to simulate the pipeline deployment.
+
+# Python poetry
+Run simulation
+
+First install dependencies:
+```bash
+poetry install
+```
+
+Run example simulation:
+```bash
+poetry run python run_simulation.py -v
+```
+
+Pass workflow file to simulate workflow:
+```bash
+poetry run python run_simulation.py -v --workflow <path/to/workflow>
+```
+
+# Docker
+Create and run docker image
+
+```bash
+docker build -f Dockerfile -t kwoksim .
+docker run -v /var/run/docker.sock:/var/run/docker.sock kwoksim
+```
+
+# Troubleshooting
+
+## Known issues
+Experienced problems using Rancher desktop. The program did not successfully start required containers.
+Try using Docker desktop
+

simulator/config/hello-world.yaml

@@ -0,0 +1,17 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: hello-world-
+  labels:
+    workflows.argoproj.io/archive-strategy: "false"
+  annotations:
+    workflows.argoproj.io/description: |
+      This is a simple hello world example.
+spec:
+  entrypoint: whalesay
+  templates:
+  - name: whalesay
+    container:
+      image: docker/whalesay:latest
+      command: [cowsay]
+      args: ["hello world"]

simulator/config/nodes.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Node
+metadata:
+  annotations:
+    node.alpha.kubernetes.io/ttl: "0"
+    kwok.x-k8s.io/node: fake
+  labels:
+    beta.kubernetes.io/arch: arm64
+    beta.kubernetes.io/os: linux
+    kubernetes.io/arch: arm64
+    kubernetes.io/hostname: worker1
+    kubernetes.io/os: linux
+    kubernetes.io/role: worker    
+    node-role.kubernetes.io/worker: ""
+    type: kwok
+  name: worker1
+status:
+  allocatable:
+    cpu: "42"
+    memory: 42Gi
+    pods: "42"
+  capacity:
+    cpu: "42"
+    memory: 42Gi
+    pods: "42"
+  nodeInfo:
+    architecture: arm64
+    kubeProxyVersion: fake
+    kubeletVersion: fake
+    operatingSystem: linux
+
+---
+
+apiVersion: v1
+kind: Node
+metadata:
+  annotations:
+    node.alpha.kubernetes.io/ttl: "0"
+    kwok.x-k8s.io/node: fake
+  labels:
+    beta.kubernetes.io/arch: arm64
+    beta.kubernetes.io/os: linux
+    kubernetes.io/arch: arm64
+    kubernetes.io/hostname: worker2
+    kubernetes.io/os: linux
+    kubernetes.io/role: worker    
+    node-role.kubernetes.io/worker: ""
+    type: kwok
+  name: worker2
+status:
+  allocatable:
+    cpu: "99"
+    memory: 99Gi
+    pods: "99"
+  capacity:
+    cpu: "99"
+    memory: 99Gi
+    pods: "99"
+  nodeInfo:
+    architecture: arm64
+    kubeProxyVersion: fake
+    kubeletVersion: fake
+    operatingSystem: linux

simulator/config/stages.yaml

@@ -0,0 +1,124 @@
+kind: Stage
+apiVersion: kwok.x-k8s.io/v1alpha1
+metadata:
+  name: pod-ready
+spec:
+  resourceRef:
+    apiGroup: v1
+    kind: Pod
+  selector:
+    matchExpressions:
+      - key: '.metadata.deletionTimestamp'
+        operator: 'DoesNotExist'
+      - key: '.status.podIP'
+        operator: 'DoesNotExist'
+  next:
+    finalizers:
+      add:
+        - value: 'kwok.x-k8s.io/fake'
+    statusTemplate: |
+      {{ $now := Now }}
+
+      conditions:
+        - lastTransitionTime: {{ $now }}
+          status: "True"
+          type: Initialized
+        - lastTransitionTime: {{ $now }}
+          status: "True"
+          type: Ready
+        - lastTransitionTime: {{ $now }}
+          status: "True"
+          type: ContainersReady
+        {{ range .spec.readinessGates }}
+        - lastTransitionTime: {{ $now }}
+          status: "True"
+          type: {{ .conditionType }}
+        {{ end }}
+
+      containerStatuses:
+        {{ range .spec.containers }}
+        - image: {{ .image }}
+          name: {{ .name }}
+          ready: true
+          restartCount: 0
+          state:
+            running:
+              startedAt: {{ $now }}
+        {{ end }}
+
+      initContainerStatuses:
+        {{ range .spec.initContainers }}
+        - image: {{ .image }}
+          name: {{ .name }}
+          ready: true
+          restartCount: 0
+          state:
+            terminated:
+              exitCode: 0
+              finishedAt: {{ $now }}
+              reason: Completed
+              startedAt: {{ $now }}
+        {{ end }}
+
+      hostIP: {{ NodeIPWith .spec.nodeName }}
+      podIP: {{ PodIPWith .spec.nodeName ( or .spec.hostNetwork false ) ( or .metadata.uid "" ) ( or .metadata.name "" ) ( or .metadata.namespace "" ) }}
+      phase: Running
+      startTime: {{ $now }}
+---
+kind: Stage
+apiVersion: kwok.x-k8s.io/v1alpha1
+metadata:
+  name: pod-complete
+spec:
+  resourceRef:
+    apiGroup: v1
+    kind: Pod
+  selector:
+    matchExpressions:
+      - key: '.metadata.deletionTimestamp'
+        operator: 'DoesNotExist'
+      - key: '.status.phase'
+        operator: 'In'
+        values:
+          - 'Running'
+      - key: '.metadata.ownerReferences.[].kind'
+        operator: 'In'
+        values:
+          - 'Workflow'
+  next:
+    statusTemplate: |
+      {{ $now := Now }}
+      {{ $root := . }}
+      containerStatuses:
+        {{ range $index, $item := .spec.containers }}
+        {{ $origin := index $root.status.containerStatuses $index }}
+        - image: {{ $item.image }}
+          name: {{ $item.name }}
+          ready: false
+          restartCount: 0
+          started: false
+          state:
+            terminated:
+              exitCode: 0
+              finishedAt: '{{ $now }}'
+              reason: Completed
+              startedAt: '{{ $now }}'
+        {{ end }}
+      phase: Succeeded
+---
+kind: Stage
+apiVersion: kwok.x-k8s.io/v1alpha1
+metadata:
+  name: pod-delete
+spec:
+  resourceRef:
+    apiGroup: v1
+    kind: Pod
+  selector:
+    matchExpressions:
+      - key: '.metadata.deletionTimestamp'
+        operator: 'Exists'
+  next:
+    finalizers:
+      empty: true
+    delete: true