[WIP] Add minio log storage for tekton-results.

developer/ckcp/openshift-operators/minio/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - minio.yaml

developer/ckcp/openshift-operators/minio/minio.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: minio-operator
+  namespace: openshift-operators
+spec:
+  channel: stable
+  installPlanApproval: Automatic
+  name: minio-operator
+  source: certified-operators
+  sourceNamespace: openshift-marketplace

developer/ckcp/openshift-operators/minio/tenant/kustomization.yaml

@@ -0,0 +1,9 @@
+
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: tekton-results
+resources:
+  - namespace.yaml
+  - tenant-config.yaml
+  - tenant.yaml

developer/ckcp/openshift-operators/minio/tenant/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tekton-results

developer/ckcp/openshift-operators/minio/tenant/tenant-config.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: storage-configuration
+  namespace: tekton-results
+type: Opaque
+stringData:
+  config.env: |-
+    export MINIO_ROOT_USER="minio"
+    export MINIO_ROOT_PASSWORD="minio123"
+    export MINIO_STORAGE_CLASS_STANDARD="EC:2"
+    export MINIO_BROWSER="on"

developer/ckcp/openshift-operators/minio/tenant/tenant.yaml

@@ -0,0 +1,71 @@
+
+apiVersion: minio.min.io/v2
+kind: Tenant
+metadata:
+  name: storage
+  namespace: tekton-results
+  labels:
+    app: minio
+  annotations:
+    prometheus.io/path: /minio/v2/metrics/cluster
+    prometheus.io/port: "9000"
+    prometheus.io/scrape: "true"
+spec:
+  exposeServices:
+    console: true
+    minio: true
+  features:
+    bucketDNS: false
+    domains: { }
+  # This desired part doesn't work. Issues:
+  # https://github.com/minio/operator/issues/1345
+  # https://github.com/minio/operator/issues/1346
+  # users:
+  #   - name: storage-user
+  # buckets:
+  #   - name: "tekton-results"
+  #     region: "us-east-1"
+  #     objectLock: true
+  certConfig: { }
+  podManagementPolicy: Parallel
+  configuration:
+    name: storage-configuration
+  env: [ ]
+  serviceMetadata:
+    minioServiceLabels: { }
+    minioServiceAnnotations: { }
+    consoleServiceLabels: { }
+    consoleServiceAnnotations: { }
+  priorityClassName: ""
+  externalCaCertSecret: [ ]
+  externalCertSecret: [ ]
+  externalClientCertSecrets: [ ]
+  image: quay.io/minio/minio:RELEASE.2022-09-17T00-09-45Z
+  imagePullSecret: { }
+  mountPath: /export
+  subPath: ""
+  serviceAccountName: ""
+  pools:
+    - servers: 1
+      name: pool-0
+      volumesPerServer: 2
+      nodeSelector: { }
+      tolerations: [ ]
+      affinity:
+        nodeAffinity: { }
+        podAffinity: { }
+        podAntiAffinity: { }
+      resources: { }
+      volumeClaimTemplate:
+        apiVersion: v1
+        kind: persistentvolumeclaims
+        metadata: { }
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 1Gi
+        status: { }
+      securityContext: {}
+  requestAutoCert: true

developer/ckcp/openshift_dev_setup.sh

@@ -108,6 +108,7 @@ init() {
             "openshift-gitops"
             "cert-manager"
             "ckcp"
+            "minio"
            )
   # get the list of APPS to be installed
   read -ra APPS <<< "$(yq eval '.apps // [] | join(" ")' "$CONFIG")"
@@ -342,6 +343,17 @@ patches:
   KUBECONFIG_KCP="$WORK_DIR/credentials/kubeconfig/kcp/ckcp-ckcp.${ws_name}.${kcp_workspace}.kubeconfig"
 }
 
+install_minio() {
+  echo -n "- Install minio S3 log server: "
+  kubectl apply -k "$CKCP_DIR/openshift-operators/minio" 2>&1 |
+  indent 4
+
+  check_deployments "openshift-operators" "minio-operator" | indent 2
+
+  kubectl apply -k "$CKCP_DIR/openshift-operators/minio/tenant" 2>&1 |
+  indent 4
+}
+
 install_pipeline_service() {
 
   TEKTON_RESULTS_DATABASE_USER="$(yq '.tekton_results_db.user' "$CONFIG")"

operator/gitops/argocd/argo-apps/tekton-results.yaml

@@ -10,7 +10,8 @@ spec:
     namespace: openshift-gitops
     server: https://kubernetes.default.svc
   source:
-    path: operator/gitops/argocd/tekton-results
+    # TODO patch base overlay to dev overlay in the dev script
+    path: operator/gitops/argocd/tekton-results/overlays/dev
     repoURL: https://github.com/openshift-pipelines/pipeline-service.git
     targetRevision: main
   project: default

operator/gitops/argocd/tekton-results/deployment.yaml → operator/gitops/argocd/tekton-results/base/deployment.yaml

@@ -42,27 +42,17 @@ spec:
               value: tekton-results-database-service.tekton-results.svc.cluster.local
             - name: DB_NAME
               value: tekton_results
-          image: quay.io/redhat-appstudio/tekton-results-api@sha256:56471d0c0582e8c4e4c109ef615b66e94a7fba258ab5960e36adce71d5a6e7f1
+          envFrom: []
+          image: docker.io/aandrienko/api-b1b7ffa9ba32f7c3020c3b68830b30a8
+          imagePullPolicy: Always
           name: api
           volumeMounts:
             - mountPath: /etc/tls
               name: tls
               readOnly: true
             - name: config
-              mountPath: /config/env
+              mountPath: "/etc/config/server"
               readOnly: true
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - "ALL"
-            # User 65532 is the nonroot user ID
-            runAsUser: 65532
-            runAsGroup: 65532
-            runAsNonRoot: true
-            seccompProfile:
-              type: RuntimeDefault
-            readOnlyRootFilesystem: true
           resources:
             requests:
               cpu: 100m
@@ -111,31 +101,38 @@ spec:
             - tekton-results-api-service.tekton-results.svc.cluster.local:50051
             - -auth_mode
             - token
-          image: quay.io/redhat-appstudio/tekton-results-watcher@sha256:2a3ae9fca05bfef0195f6d11a2c1b6111288617bad4f7106400330c7fb016d14
+          image: docker.io/aandrienko/watcher-83f971ea227fb24157c0c699b824a628
+          imagePullPolicy: Always
           name: watcher
           volumeMounts:
             - mountPath: /etc/tls
               name: tls
               readOnly: true
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - "ALL"
-            # User 65532 is the nonroot user ID
-            runAsUser: 65533
-            runAsGroup: 65532
-            runAsNonRoot: true
-            seccompProfile:
-              type: RuntimeDefault
-            readOnlyRootFilesystem: true
           resources:
             requests:
               cpu: 100m
               memory: 100Mi
             limits:
               cpu: 500m
               memory: 500Mi
+          env:
+            - name: SYSTEM_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CONFIG_LOGGING_NAME
+              value: tekton-results-config-logging
+            - name: CONFIG_LEADERELECTION_NAME
+              value: tekton-results-config-leader-election
+            - name: CONFIG_OBSERVABILITY_NAME
+              value: tekton-results-config-observability
+            - name: METRICS_DOMAIN
+              value: tekton.dev/results
+          ports:
+            - name: metrics
+              containerPort: 9090
+            - name: profiling
+              containerPort: 8008
       serviceAccountName: tekton-results-watcher
       volumes:
         - name: tls

operator/gitops/argocd/tekton-results/permissions.yaml → operator/gitops/argocd/tekton-results/base/permissions.yaml

@@ -113,6 +113,38 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - "coordination.k8s.io"
+    resources:
+      - "leases"
+    verbs:
+      - "get"
+      - "list"
+      - "create"
+      - "update"
+      - "delete"
+      - "patch"
+      - "watch"
+  - apiGroups:
+      - ""
+    resources:
+      - "pods"
+    verbs:
+      - "get"
+      - "list"
+      - "watch"
+  - apiGroups:
+      - ""
+    resources:
+      - "pods/log"
+    verbs:
+      - "get"
+  - apiGroups:
+      - ""
+    resources:
+      - "namespaces"
+    verbs:
+      - "get"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

operator/gitops/argocd/tekton-results/overlays/dev/kustomization.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+- minio-user-secret.yaml
+
+patchesJson6902:
+  - target:
+      version: v1
+      group: apps
+      kind: Deployment
+      name: tekton-results-api
+      namespace: tekton-results
+    path: s3-configuration-patch.yaml
+  - target:
+      version: v1
+      group: apps
+      kind: Deployment
+      name: tekton-results-api
+      namespace: tekton-results
+    path: minio-tls-patch.yaml
+
+patches:
+  - target:
+      version: v1
+      group: apps
+      kind: Deployment
+      name: tekton-results-api
+      namespace: tekton-results
+    path: minio-user-cred-patch.yaml

operator/gitops/argocd/tekton-results/overlays/dev/minio-tls-patch.yaml

@@ -0,0 +1,16 @@
+# minio tls patch
+- op: add
+  path: "/spec/template/spec/volumes/-"
+  value:
+    name: ca-s3
+    secret:
+      secretName: storage-tls
+      items:
+        - key: public.crt
+          path: s3-cert.crt
+- op: add
+  path: "/spec/template/spec/containers/0/volumeMounts/0"
+  value:
+    name: ca-s3
+    mountPath: /etc/ssl/certs/s3-cert.crt
+    subPath: s3-cert.crt

operator/gitops/argocd/tekton-results/overlays/dev/minio-user-cred-patch.yaml

@@ -0,0 +1,5 @@
+- op: add
+  path: "/spec/template/spec/containers/0/envFrom/-"
+  value:
+    secretRef:
+      name: storage-user

operator/gitops/argocd/tekton-results/overlays/dev/minio-user-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+data:
+  S3_ACCESS_KEY_ID: bWluaW8=
+  S3_SECRET_ACCESS_KEY: bWluaW8xMjM=
+kind: Secret
+metadata:
+  name: storage-user
+  namespace: tekton-results
+type: Opaque

operator/gitops/argocd/tekton-results/overlays/dev/s3-configuration-patch.yaml

@@ -0,0 +1,25 @@
+- op: add
+  path: "/spec/template/spec/containers/0/env/-"
+  value:
+    name: LOG_TYPE
+    value: S3
+- op: add
+  path: "/spec/template/spec/containers/0/env/-"
+  value:
+    name: S3_BUCKET_NAME
+    value: tekton-results
+- op: add
+  path: "/spec/template/spec/containers/0/env/-"
+  value:
+    name: S3_ENDPOINT
+    value: https://minio.tekton-results.svc.cluster.local
+- op: add
+  path: "/spec/template/spec/containers/0/env/-"
+  value:
+    name: S3_HOSTNAME_IMMUTABLE
+    value: "true"
+- op: add
+  path: "/spec/template/spec/containers/0/env/-"
+  value:
+    name: S3_REGION
+    value: us-east-1