From b0088dbcf4620f4bb92fa073cc9a25fe3f4c65fb Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 11 Mar 2022 14:32:21 +0100 Subject: [PATCH 01/69] Initial commit after migration to combined repo Signed-off-by: Sven Trieflinger --- README.md | 72 +- klyshko-mp-spdz/Dockerfile.fake-offline | 45 + klyshko-mp-spdz/deploy.sh | 21 + klyshko-mp-spdz/kii-run.sh | 71 ++ klyshko-mp-spdz/tuple_type_mapping.md | 12 + klyshko-operator/.dockerignore | 4 + klyshko-operator/.gitignore | 25 + klyshko-operator/Dockerfile | 27 + klyshko-operator/Makefile | 203 +++++ klyshko-operator/PROJECT | 37 + .../api/v1alpha1/groupversion_info.go | 27 + .../api/v1alpha1/tuplegenerationjob_types.go | 52 ++ .../api/v1alpha1/tuplegenerationtask_types.go | 42 + .../api/v1alpha1/zz_generated.deepcopy.go | 203 +++++ ...o.carbnyestack.io_tuplegenerationjobs.yaml | 75 ++ ....carbnyestack.io_tuplegenerationtasks.yaml | 52 ++ .../config/crd/kustomization.yaml | 24 + .../config/crd/kustomizeconfig.yaml | 19 + .../cainjection_in_tuplegenerationjobs.yaml | 7 + .../cainjection_in_tuplegenerationtasks.yaml | 7 + .../webhook_in_tuplegenerationjobs.yaml | 16 + .../webhook_in_tuplegenerationtasks.yaml | 16 + .../config/default/kustomization.yaml | 74 ++ .../default/manager_auth_proxy_patch.yaml | 27 + .../config/default/manager_config_patch.yaml | 20 + .../manager/controller_manager_config.yaml | 11 + .../config/manager/kustomization.yaml | 16 + klyshko-operator/config/manager/manager.yaml | 56 ++ .../config/manifests/kustomization.yaml | 27 + .../config/prometheus/kustomization.yaml | 2 + .../config/prometheus/monitor.yaml | 20 + .../rbac/auth_proxy_client_clusterrole.yaml | 9 + .../config/rbac/auth_proxy_role.yaml | 17 + .../config/rbac/auth_proxy_role_binding.yaml | 12 + .../config/rbac/auth_proxy_service.yaml | 15 + .../config/rbac/kustomization.yaml | 18 + .../config/rbac/leader_election_role.yaml | 37 + .../rbac/leader_election_role_binding.yaml | 12 + klyshko-operator/config/rbac/role.yaml | 68 ++ .../config/rbac/role_binding.yaml | 12 + .../config/rbac/service_account.yaml | 5 + .../rbac/tuplegenerationjob_editor_role.yaml | 24 + .../rbac/tuplegenerationjob_viewer_role.yaml | 20 + .../rbac/tuplegenerationtask_editor_role.yaml | 24 + .../rbac/tuplegenerationtask_viewer_role.yaml | 20 + .../config/samples/apollo-vcp.yaml | 6 + .../klyshko_v1alpha1_tuplegenerationjob.yaml | 7 + .../klyshko_v1alpha1_tuplegenerationtask.yaml | 7 + .../config/samples/kustomization.yaml | 5 + .../config/samples/starbuck-vcp.yaml | 6 + .../config/scorecard/bases/config.yaml | 7 + .../config/scorecard/kustomization.yaml | 16 + .../scorecard/patches/basic.config.yaml | 10 + .../config/scorecard/patches/olm.config.yaml | 50 ++ klyshko-operator/controllers/config_utils.go | 47 ++ klyshko-operator/controllers/etcd_keys.go | 90 ++ klyshko-operator/controllers/suite_test.go | 77 ++ .../tuplegenerationjob_controller.go | 303 +++++++ .../tuplegenerationtask_controller.go | 123 +++ klyshko-operator/deploy.sh | 48 ++ klyshko-operator/go.mod | 14 + klyshko-operator/go.sum | 784 ++++++++++++++++++ klyshko-operator/hack/boilerplate.go.txt | 15 + klyshko-operator/main.go | 113 +++ 64 files changed, 3330 insertions(+), 1 deletion(-) create mode 100644 klyshko-mp-spdz/Dockerfile.fake-offline create mode 100755 klyshko-mp-spdz/deploy.sh create mode 100755 klyshko-mp-spdz/kii-run.sh create mode 100644 klyshko-mp-spdz/tuple_type_mapping.md create mode 100644 klyshko-operator/.dockerignore create mode 100644 klyshko-operator/.gitignore create mode 100644 klyshko-operator/Dockerfile create mode 100644 klyshko-operator/Makefile create mode 100644 klyshko-operator/PROJECT create mode 100644 klyshko-operator/api/v1alpha1/groupversion_info.go create mode 100644 klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go create mode 100644 klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go create mode 100644 klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go create mode 100644 klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml create mode 100644 klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml create mode 100644 klyshko-operator/config/crd/kustomization.yaml create mode 100644 klyshko-operator/config/crd/kustomizeconfig.yaml create mode 100644 klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationjobs.yaml create mode 100644 klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationtasks.yaml create mode 100644 klyshko-operator/config/crd/patches/webhook_in_tuplegenerationjobs.yaml create mode 100644 klyshko-operator/config/crd/patches/webhook_in_tuplegenerationtasks.yaml create mode 100644 klyshko-operator/config/default/kustomization.yaml create mode 100644 klyshko-operator/config/default/manager_auth_proxy_patch.yaml create mode 100644 klyshko-operator/config/default/manager_config_patch.yaml create mode 100644 klyshko-operator/config/manager/controller_manager_config.yaml create mode 100644 klyshko-operator/config/manager/kustomization.yaml create mode 100644 klyshko-operator/config/manager/manager.yaml create mode 100644 klyshko-operator/config/manifests/kustomization.yaml create mode 100644 klyshko-operator/config/prometheus/kustomization.yaml create mode 100644 klyshko-operator/config/prometheus/monitor.yaml create mode 100644 klyshko-operator/config/rbac/auth_proxy_client_clusterrole.yaml create mode 100644 klyshko-operator/config/rbac/auth_proxy_role.yaml create mode 100644 klyshko-operator/config/rbac/auth_proxy_role_binding.yaml create mode 100644 klyshko-operator/config/rbac/auth_proxy_service.yaml create mode 100644 klyshko-operator/config/rbac/kustomization.yaml create mode 100644 klyshko-operator/config/rbac/leader_election_role.yaml create mode 100644 klyshko-operator/config/rbac/leader_election_role_binding.yaml create mode 100644 klyshko-operator/config/rbac/role.yaml create mode 100644 klyshko-operator/config/rbac/role_binding.yaml create mode 100644 klyshko-operator/config/rbac/service_account.yaml create mode 100644 klyshko-operator/config/rbac/tuplegenerationjob_editor_role.yaml create mode 100644 klyshko-operator/config/rbac/tuplegenerationjob_viewer_role.yaml create mode 100644 klyshko-operator/config/rbac/tuplegenerationtask_editor_role.yaml create mode 100644 klyshko-operator/config/rbac/tuplegenerationtask_viewer_role.yaml create mode 100644 klyshko-operator/config/samples/apollo-vcp.yaml create mode 100644 klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml create mode 100644 klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml create mode 100644 klyshko-operator/config/samples/kustomization.yaml create mode 100644 klyshko-operator/config/samples/starbuck-vcp.yaml create mode 100644 klyshko-operator/config/scorecard/bases/config.yaml create mode 100644 klyshko-operator/config/scorecard/kustomization.yaml create mode 100644 klyshko-operator/config/scorecard/patches/basic.config.yaml create mode 100644 klyshko-operator/config/scorecard/patches/olm.config.yaml create mode 100644 klyshko-operator/controllers/config_utils.go create mode 100644 klyshko-operator/controllers/etcd_keys.go create mode 100644 klyshko-operator/controllers/suite_test.go create mode 100644 klyshko-operator/controllers/tuplegenerationjob_controller.go create mode 100644 klyshko-operator/controllers/tuplegenerationtask_controller.go create mode 100755 klyshko-operator/deploy.sh create mode 100644 klyshko-operator/go.mod create mode 100644 klyshko-operator/go.sum create mode 100644 klyshko-operator/hack/boilerplate.go.txt create mode 100644 klyshko-operator/main.go diff --git a/README.md b/README.md index c1f0890..d5a7fac 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,76 @@ Klyshko is a kubernetes-native open source correlated randomness generator service for Secure Multiparty Computation in the offline/online model and part of [Carbyne Stack](https://github.com/carbynestack). +> **DISCLAIMER**: Carbyne Stack Klyshko is in *proof-of-concept* stage. The +> software is not ready for production use. It has neither been developed nor +> tested for a specific use case. + +## Namesake + +_Klyshko_ is one of the inventors of _spontaneous parametric down-conversion_ +(SPDC). SPDC is an important process in quantum optics, used especially as a +source of entangled photon pairs, and of single photons +(see [Wikipedia](https://en.wikipedia.org/wiki/Spontaneous_parametric_down-conversion)). +The analogy to the _Klyshko_ service is that secret shared tuples are +correlated and thus kind of "entangled" and that the microservice is the +implementation of the process that creates the tuples. + +## Klyshko Integration Interface + +> **IMPORTANT**: This is an initial incomplete version of the KII that is +> subject to change without notice. For the time being it is very much +> influenced by the CRGs provided as part of the +> [MP-SPDZ](https://github.com/data61/MP-SPDZ) project. + +_Klyshko_ has been designed to allow for easy integration of different +correlated randomness generators (CRGs). Integration is done by means of +providing a docker image containing the CRG that implements the +_Klyshko Integration Interface_ (KII). + +> **TIP**: For an example of how to integrate the +> [MP-SPDZ](https://github.com/data61/MP-SPDZ) CRG producing _fake_ tuples see +> the [klyshko-mp-spdz](klyshko-mp-spdz) module. + +### Entrypoint + +The CRG docker image must contain a `kii-run.sh` script in the working +directory that performs the tuple generation process. The script must terminate +with a non-zero exit code in case the tuples can not be generated. + +### Environment Variables + +The following environment variables are passed into CRG containers to control +the tuple generation and provisioning process. + +#### Input + +- `KII_JOB_ID`: The Type 4 UUID used as a job identifier. This is the same +among all VCPs in the VC. +- `KII_TUPLES_PER_JOB`: The number of tuples to be generated. The CRG should +try to match the requested number but is not required to do so. +- `KII_PLAYER_NUMBER`: The 0-based number of the local VCP. +- `KII_PLAYER_COUNT`: The overall number of VCPs in the VC. +- `KII_TUPLE_TYPE`: The tuple type to generate. Must be one of + - `bit_gfp`, `bit_gf2n` + - `inputmask_gfp`, `inputmask_gf2n` + - `inversetuple_gfp`, `inversetuple_gf2n` + - `squaretuple_gfp`, `squaretuple_gf2n` + - `multiplicationtriple_gfp`, `multiplicationtriple_gf2n` + +#### Output + +- `KII_TUPLE_FILE`: The file the generated tuples must be written to. + +### Prime + +The prime to be used for generating prime field tuples is provided in the file +`/etc/kii/params/prime`. + +### MAC Key Shares + +The MAC key shares for prime and binary fields are made available as files +`mac_key_share_p` and `mac_key_share_2` in folder `/etc/kii/secret-params`. + ## License Carbyne Stack *Klyshko Correlated Randomness Generation Subsystem* is @@ -20,4 +90,4 @@ the Carbyne Stack repository. Please see the Carbyne Stack [Contributor's Guide](https://github.com/carbynestack/carbynestack/blob/master/CONTRIBUTING.md) -. \ No newline at end of file +. diff --git a/klyshko-mp-spdz/Dockerfile.fake-offline b/klyshko-mp-spdz/Dockerfile.fake-offline new file mode 100644 index 0000000..ff97a84 --- /dev/null +++ b/klyshko-mp-spdz/Dockerfile.fake-offline @@ -0,0 +1,45 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +FROM ubuntu:20.04 + +# Install prerequisites +ENV TZ=Europe/Berlin +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN apt-get update && apt-get install -y --no-install-recommends \ + automake \ + build-essential \ + git \ + libboost-dev \ + libboost-thread-dev \ + libntl-dev \ + libsodium-dev \ + libssl-dev \ + libtool \ + m4 \ + python3 \ + texinfo \ + yasm \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# https://github.com/data61/MP-SPDZ +ARG MP_SPDZ_REPO="https://github.com/carbynestack/MP-SPDZ" +# v0.2.8 +ARG MP_SPDZ_VERSION="add-prng-seed-option" + +# Checkout MP-SPDZ repository at required version +RUN git clone --recurse-submodules ${MP_SPDZ_REPO} mp-spdz-${MP_SPDZ_VERSION} +WORKDIR mp-spdz-${MP_SPDZ_VERSION} +RUN git checkout ${MP_SPDZ_VERSION} + +# Build fake offline phase executable +RUN echo "MY_CFLAGS += -DINSECURE" >> CONFIG.mine +RUN make -j 8 tldr +RUN make -j 8 Fake-Offline.x + +# Copy resources +COPY kii-run.sh . diff --git a/klyshko-mp-spdz/deploy.sh b/klyshko-mp-spdz/deploy.sh new file mode 100755 index 0000000..fecedd4 --- /dev/null +++ b/klyshko-mp-spdz/deploy.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +declare -a CLUSTERS=("starbuck" "apollo") + +echo -e "${GREEN}Building code and image${NC}" +docker build -t carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT . -f Dockerfile.fake-offline + +echo -e "${GREEN}Loading docker images into cluster registries${NC}" +for c in "${CLUSTERS[@]}" +do + echo -e "${GREEN}Loading docker image into $c${NC}" + kind load docker-image carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT --name "$c" +done \ No newline at end of file diff --git a/klyshko-mp-spdz/kii-run.sh b/klyshko-mp-spdz/kii-run.sh new file mode 100755 index 0000000..86af279 --- /dev/null +++ b/klyshko-mp-spdz/kii-run.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Fail, if any command fails +set -e + +# Setup offline executable command line arguments dictionary +n=${KII_TUPLES_PER_JOB} +pn=${KII_PLAYER_NUMBER} +declare -A argsByType=( + ["bit_gfp"]="--nbits 0,${n}" + ["bit_gf2n"]="--nbits ${n},0" + ["inputmask_gfp"]="--ntriples 0,${n}" + ["inputmask_gf2n"]="--ntriples ${n},0" + ["inversetuple_gfp"]="--ninverses ${n}" + ["inversetuple_gf2n"]="--ninverses ${n}" + ["squaretuple_gfp"]="--nsquares 0,${n}" + ["squaretuple_gf2n"]="--nsquares ${n},0" + ["multiplicationtriple_gfp"]="--ntriples 0,${n}" + ["multiplicationtriple_gf2n"]="--ntriples ${n},0" +) +declare -A folderByType=( + ["bit_gfp"]="2-p-128/Bits-p-P${pn}" + ["bit_gf2n"]="2-2-128/Bits-2-P${pn}" + ["inputmask_gfp"]="2-p-128/Triples-p-P${pn}" + ["inputmask_gf2n"]="2-2-128/Triples-2-P${pn}" + ["inversetuple_gfp"]="2-p-128/Inverses-p-P${pn}" + ["inversetuple_gf2n"]="2-2-128/Inverses-2-P${pn}" + ["squaretuple_gfp"]="2-p-128/Squares-p-P${pn}" + ["squaretuple_gf2n"]="2-2-128/Squares-2-P${pn}" + ["multiplicationtriple_gfp"]="2-p-128/Triples-p-P${pn}" + ["multiplicationtriple_gf2n"]="2-2-128/Triples-2-P${pn}" +) + +# Provide required parameters in MP-SPDZ "Player-Data" folder +prime=$(cat /etc/kii/params/prime) +declare fields=("p" "2") +for f in "${fields[@]}" +do + + folder="Player-Data/${KII_PLAYER_COUNT}-${f}-128" + mkdir -p "${folder}" + echo "Providing parameters for field ${f}-128 in folder ${folder}" + + # Write MAC key shares + for pn in $(seq 0 $((KII_PLAYER_COUNT-1))) + do + macKeyShareFile="${folder}/Player-MAC-Keys-${f}-P${pn}" + if [[ ${pn} -eq ${KII_PLAYER_NUMBER} ]]; then + src="/etc/kii/secret-params" + else + src="/etc/kii/extra-params" + fi + macKeyShare=$(cat "${src}/mac_key_share_${f}") + echo "${KII_PLAYER_COUNT} ${macKeyShare}" > "${macKeyShareFile}" + echo "MAC key share for player ${pn} written to ${macKeyShareFile}" + done + +done + +# Execute offline phase +cmd="./Fake-Offline.x -d 0 --prime ${prime} --prngseed ${KII_JOB_ID} ${argsByType[${KII_TUPLE_TYPE}]} ${KII_PLAYER_COUNT}" +eval "$cmd" + +# Copy generated tuples to path expected by KII +cp "Player-Data/${folderByType[${KII_TUPLE_TYPE}]}" "${KII_TUPLE_FILE}" diff --git a/klyshko-mp-spdz/tuple_type_mapping.md b/klyshko-mp-spdz/tuple_type_mapping.md new file mode 100644 index 0000000..3d2540f --- /dev/null +++ b/klyshko-mp-spdz/tuple_type_mapping.md @@ -0,0 +1,12 @@ +| Type | Flag | Folder | Header Length | Verified | +|-----------------------------|-----------------|-----------------------|---------------|----------| +| bit_gfp | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | yes | +| bit_gf2n | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | yes | +| inputmask_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | +| inputmask_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | +| inversetuple_gfp | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | yes | +| inversetuple_gf2n | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | yes | +| squaretuple_gfp | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | yes | +| squaretuple_gf2n | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | yes | +| multiplicationtriple_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | +| multiplicationtriple_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | \ No newline at end of file diff --git a/klyshko-operator/.dockerignore b/klyshko-operator/.dockerignore new file mode 100644 index 0000000..0f04682 --- /dev/null +++ b/klyshko-operator/.dockerignore @@ -0,0 +1,4 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ diff --git a/klyshko-operator/.gitignore b/klyshko-operator/.gitignore new file mode 100644 index 0000000..c0a7a54 --- /dev/null +++ b/klyshko-operator/.gitignore @@ -0,0 +1,25 @@ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin +testbin/* + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Kubernetes Generated files - skip generated files, except for vendored files + +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +*.swp +*.swo +*~ diff --git a/klyshko-operator/Dockerfile b/klyshko-operator/Dockerfile new file mode 100644 index 0000000..4152680 --- /dev/null +++ b/klyshko-operator/Dockerfile @@ -0,0 +1,27 @@ +# Build the manager binary +FROM golang:1.16 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY api/ api/ +COPY controllers/ controllers/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/klyshko-operator/Makefile b/klyshko-operator/Makefile new file mode 100644 index 0000000..65eef2c --- /dev/null +++ b/klyshko-operator/Makefile @@ -0,0 +1,203 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 0.0.1 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# carbnyestack.io/klyshko-go-bundle:$VERSION and carbnyestack.io/klyshko-go-catalog:$VERSION. +IMAGE_TAG_BASE ?= carbnyestack.io/klyshko-go + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# Image URL to use all building/pushing image targets +IMG ?= controller:latest +# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) +CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.21 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +fmt: ## Run go fmt against code. + go fmt ./... + +vet: ## Run go vet against code. + go vet ./... + +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + +##@ Build + +build: generate fmt vet ## Build manager binary. + go build -o bin/manager main.go + +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +docker-build: test ## Build docker image with the manager. + docker build -t ${IMG} . + +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +##@ Deployment + +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl delete -f - + +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/default | kubectl delete -f - + + +CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +controller-gen: ## Download controller-gen locally if necessary. + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1) + +KUSTOMIZE = $(shell pwd)/bin/kustomize +kustomize: ## Download kustomize locally if necessary. + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) + +ENVTEST = $(shell pwd)/bin/setup-envtest +envtest: ## Download envtest-setup locally if necessary. + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) + +# go-get-tool will 'go get' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-get-tool +@[ -f $(1) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +go mod init tmp ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ +rm -rf $$TMP_DIR ;\ +} +endef + +.PHONY: bundle +bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files. + operator-sdk generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + operator-sdk bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = ./bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$${OS}-$${ARCH}-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMG=$(CATALOG_IMG) diff --git a/klyshko-operator/PROJECT b/klyshko-operator/PROJECT new file mode 100644 index 0000000..fdf0129 --- /dev/null +++ b/klyshko-operator/PROJECT @@ -0,0 +1,37 @@ +domain: carbnyestack.io +layout: +- go.kubebuilder.io/v3 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: klyshko-go +repo: github.com/carbynestack/klyshko +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: carbnyestack.io + group: klyshko + kind: Job + path: github.com/carbynestack/klyshko/api/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: carbnyestack.io + group: klyshko + kind: TupleGenerationJob + path: github.com/carbynestack/klyshko/api/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: carbnyestack.io + group: klyshko + kind: TupleGenerationTask + path: github.com/carbynestack/klyshko/api/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/klyshko-operator/api/v1alpha1/groupversion_info.go b/klyshko-operator/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..869cb5b --- /dev/null +++ b/klyshko-operator/api/v1alpha1/groupversion_info.go @@ -0,0 +1,27 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package v1alpha1 contains API Schema definitions for the klyshko v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=klyshko.carbnyestack.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "klyshko.carbnyestack.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go new file mode 100644 index 0000000..b4894f9 --- /dev/null +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -0,0 +1,52 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type TupleGenerationJobSpec struct { + + // +kubebuilder:validation:Enum=multiplicationtriple_gfp + Type string `json:"type"` + + //+kubebuilder:validation:Minimum=0 + //+kubebuilder:validation:ExclusiveMinimum=true + Count int32 `json:"count"` +} + +type TupleGenerationJobStatus struct { +} + +//+kubebuilder:object:root=true +//+kubebuilder:resource:shortName=tgj;tgjob +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="Tuple Type",type=string,JSONPath=`.spec.type` +//+kubebuilder:printcolumn:name="Tuple Count",type=string,JSONPath=`.spec.count` +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +type TupleGenerationJob struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TupleGenerationJobSpec `json:"spec,omitempty"` + Status TupleGenerationJobStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +type TupleGenerationJobList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TupleGenerationJob `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TupleGenerationJob{}, &TupleGenerationJobList{}) +} diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go new file mode 100644 index 0000000..0130403 --- /dev/null +++ b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go @@ -0,0 +1,42 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type TupleGenerationTaskSpec struct { +} + +type TupleGenerationTaskStatus struct { +} + +//+kubebuilder:object:root=true +//+kubebuilder:resource:shortName=tgt;tgtask +//+kubebuilder:subresource:status + +type TupleGenerationTask struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TupleGenerationTaskSpec `json:"spec,omitempty"` + Status TupleGenerationTaskStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +type TupleGenerationTaskList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TupleGenerationTask `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TupleGenerationTask{}, &TupleGenerationTaskList{}) +} diff --git a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..4b51bd6 --- /dev/null +++ b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,203 @@ +// +build !ignore_autogenerated + +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationJob) DeepCopyInto(out *TupleGenerationJob) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationJob. +func (in *TupleGenerationJob) DeepCopy() *TupleGenerationJob { + if in == nil { + return nil + } + out := new(TupleGenerationJob) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TupleGenerationJob) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationJobList) DeepCopyInto(out *TupleGenerationJobList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TupleGenerationJob, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationJobList. +func (in *TupleGenerationJobList) DeepCopy() *TupleGenerationJobList { + if in == nil { + return nil + } + out := new(TupleGenerationJobList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TupleGenerationJobList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationJobSpec) DeepCopyInto(out *TupleGenerationJobSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationJobSpec. +func (in *TupleGenerationJobSpec) DeepCopy() *TupleGenerationJobSpec { + if in == nil { + return nil + } + out := new(TupleGenerationJobSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationJobStatus) DeepCopyInto(out *TupleGenerationJobStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationJobStatus. +func (in *TupleGenerationJobStatus) DeepCopy() *TupleGenerationJobStatus { + if in == nil { + return nil + } + out := new(TupleGenerationJobStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationTask) DeepCopyInto(out *TupleGenerationTask) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationTask. +func (in *TupleGenerationTask) DeepCopy() *TupleGenerationTask { + if in == nil { + return nil + } + out := new(TupleGenerationTask) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TupleGenerationTask) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationTaskList) DeepCopyInto(out *TupleGenerationTaskList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TupleGenerationTask, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationTaskList. +func (in *TupleGenerationTaskList) DeepCopy() *TupleGenerationTaskList { + if in == nil { + return nil + } + out := new(TupleGenerationTaskList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TupleGenerationTaskList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationTaskSpec) DeepCopyInto(out *TupleGenerationTaskSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationTaskSpec. +func (in *TupleGenerationTaskSpec) DeepCopy() *TupleGenerationTaskSpec { + if in == nil { + return nil + } + out := new(TupleGenerationTaskSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationTaskStatus) DeepCopyInto(out *TupleGenerationTaskStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationTaskStatus. +func (in *TupleGenerationTaskStatus) DeepCopy() *TupleGenerationTaskStatus { + if in == nil { + return nil + } + out := new(TupleGenerationTaskStatus) + in.DeepCopyInto(out) + return out +} diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml new file mode 100644 index 0000000..a333360 --- /dev/null +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -0,0 +1,75 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null + name: tuplegenerationjobs.klyshko.carbnyestack.io +spec: + group: klyshko.carbnyestack.io + names: + kind: TupleGenerationJob + listKind: TupleGenerationJobList + plural: tuplegenerationjobs + shortNames: + - tgj + - tgjob + singular: tuplegenerationjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.type + name: Tuple Type + type: string + - jsonPath: .spec.count + name: Tuple Count + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + count: + exclusiveMinimum: true + format: int32 + minimum: 0 + type: integer + type: + enum: + - multiplicationtriple_gfp + type: string + required: + - count + - type + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml new file mode 100644 index 0000000..4c3b3e1 --- /dev/null +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml @@ -0,0 +1,52 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null + name: tuplegenerationtasks.klyshko.carbnyestack.io +spec: + group: klyshko.carbnyestack.io + names: + kind: TupleGenerationTask + listKind: TupleGenerationTaskList + plural: tuplegenerationtasks + shortNames: + - tgt + - tgtask + singular: tuplegenerationtask + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/klyshko-operator/config/crd/kustomization.yaml b/klyshko-operator/config/crd/kustomization.yaml new file mode 100644 index 0000000..f35a6da --- /dev/null +++ b/klyshko-operator/config/crd/kustomization.yaml @@ -0,0 +1,24 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +- bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_tuplegenerationjobs.yaml +#- patches/webhook_in_tuplegenerationtasks.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_tuplegenerationjobs.yaml +#- patches/cainjection_in_tuplegenerationtasks.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/klyshko-operator/config/crd/kustomizeconfig.yaml b/klyshko-operator/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000..ec5c150 --- /dev/null +++ b/klyshko-operator/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationjobs.yaml new file mode 100644 index 0000000..50fe5cc --- /dev/null +++ b/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationjobs.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: tuplegenerationjobs.klyshko.carbnyestack.io diff --git a/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationtasks.yaml b/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationtasks.yaml new file mode 100644 index 0000000..f975f18 --- /dev/null +++ b/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationtasks.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: tuplegenerationtasks.klyshko.carbnyestack.io diff --git a/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationjobs.yaml new file mode 100644 index 0000000..763f279 --- /dev/null +++ b/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationjobs.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: tuplegenerationjobs.klyshko.carbnyestack.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationtasks.yaml b/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationtasks.yaml new file mode 100644 index 0000000..9d770ea --- /dev/null +++ b/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationtasks.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: tuplegenerationtasks.klyshko.carbnyestack.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/klyshko-operator/config/default/kustomization.yaml b/klyshko-operator/config/default/kustomization.yaml new file mode 100644 index 0000000..99d84d5 --- /dev/null +++ b/klyshko-operator/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: klyshko-go-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: klyshko-go- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +- manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/klyshko-operator/config/default/manager_auth_proxy_patch.yaml b/klyshko-operator/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 0000000..4e2232f --- /dev/null +++ b/klyshko-operator/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,27 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + protocol: TCP + name: https + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/klyshko-operator/config/default/manager_config_patch.yaml b/klyshko-operator/config/default/manager_config_patch.yaml new file mode 100644 index 0000000..6c40015 --- /dev/null +++ b/klyshko-operator/config/default/manager_config_patch.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--config=controller_manager_config.yaml" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config diff --git a/klyshko-operator/config/manager/controller_manager_config.yaml b/klyshko-operator/config/manager/controller_manager_config.yaml new file mode 100644 index 0000000..3eb1d02 --- /dev/null +++ b/klyshko-operator/config/manager/controller_manager_config.yaml @@ -0,0 +1,11 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: 02e9e224.carbnyestack.io diff --git a/klyshko-operator/config/manager/kustomization.yaml b/klyshko-operator/config/manager/kustomization.yaml new file mode 100644 index 0000000..352c591 --- /dev/null +++ b/klyshko-operator/config/manager/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: carbynestack/klyshko-operator + newTag: v0.0.1 diff --git a/klyshko-operator/config/manager/manager.yaml b/klyshko-operator/config/manager/manager.yaml new file mode 100644 index 0000000..2026244 --- /dev/null +++ b/klyshko-operator/config/manager/manager.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + securityContext: + runAsNonRoot: true + containers: + - command: + - /manager + args: + - --leader-elect + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/klyshko-operator/config/manifests/kustomization.yaml b/klyshko-operator/config/manifests/kustomization.yaml new file mode 100644 index 0000000..1dee91b --- /dev/null +++ b/klyshko-operator/config/manifests/kustomization.yaml @@ -0,0 +1,27 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/klyshko-go.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard + +# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. +# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. +# These patches remove the unnecessary "cert" volume and its manager container volumeMount. +#patchesJson6902: +#- target: +# group: apps +# version: v1 +# kind: Deployment +# name: controller-manager +# namespace: system +# patch: |- +# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/containers/1/volumeMounts/0 +# # Remove the "cert" volume, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing volumes in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/volumes/0 diff --git a/klyshko-operator/config/prometheus/kustomization.yaml b/klyshko-operator/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..ed13716 --- /dev/null +++ b/klyshko-operator/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/klyshko-operator/config/prometheus/monitor.yaml b/klyshko-operator/config/prometheus/monitor.yaml new file mode 100644 index 0000000..d19136a --- /dev/null +++ b/klyshko-operator/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/klyshko-operator/config/rbac/auth_proxy_client_clusterrole.yaml b/klyshko-operator/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 0000000..51a75db --- /dev/null +++ b/klyshko-operator/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/klyshko-operator/config/rbac/auth_proxy_role.yaml b/klyshko-operator/config/rbac/auth_proxy_role.yaml new file mode 100644 index 0000000..80e1857 --- /dev/null +++ b/klyshko-operator/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/klyshko-operator/config/rbac/auth_proxy_role_binding.yaml b/klyshko-operator/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 0000000..ec7acc0 --- /dev/null +++ b/klyshko-operator/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/klyshko-operator/config/rbac/auth_proxy_service.yaml b/klyshko-operator/config/rbac/auth_proxy_service.yaml new file mode 100644 index 0000000..71f1797 --- /dev/null +++ b/klyshko-operator/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/klyshko-operator/config/rbac/kustomization.yaml b/klyshko-operator/config/rbac/kustomization.yaml new file mode 100644 index 0000000..731832a --- /dev/null +++ b/klyshko-operator/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/klyshko-operator/config/rbac/leader_election_role.yaml b/klyshko-operator/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..4190ec8 --- /dev/null +++ b/klyshko-operator/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/klyshko-operator/config/rbac/leader_election_role_binding.yaml b/klyshko-operator/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..1d1321e --- /dev/null +++ b/klyshko-operator/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/klyshko-operator/config/rbac/role.yaml b/klyshko-operator/config/rbac/role.yaml new file mode 100644 index 0000000..971d5ef --- /dev/null +++ b/klyshko-operator/config/rbac/role.yaml @@ -0,0 +1,68 @@ + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs/finalizers + verbs: + - update +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs/status + verbs: + - get + - patch + - update +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks/finalizers + verbs: + - update +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks/status + verbs: + - get + - patch + - update diff --git a/klyshko-operator/config/rbac/role_binding.yaml b/klyshko-operator/config/rbac/role_binding.yaml new file mode 100644 index 0000000..2070ede --- /dev/null +++ b/klyshko-operator/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/klyshko-operator/config/rbac/service_account.yaml b/klyshko-operator/config/rbac/service_account.yaml new file mode 100644 index 0000000..7cd6025 --- /dev/null +++ b/klyshko-operator/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller-manager + namespace: system diff --git a/klyshko-operator/config/rbac/tuplegenerationjob_editor_role.yaml b/klyshko-operator/config/rbac/tuplegenerationjob_editor_role.yaml new file mode 100644 index 0000000..298d33a --- /dev/null +++ b/klyshko-operator/config/rbac/tuplegenerationjob_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit tuplegenerationjobs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tuplegenerationjob-editor-role +rules: +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs/status + verbs: + - get diff --git a/klyshko-operator/config/rbac/tuplegenerationjob_viewer_role.yaml b/klyshko-operator/config/rbac/tuplegenerationjob_viewer_role.yaml new file mode 100644 index 0000000..b7957cc --- /dev/null +++ b/klyshko-operator/config/rbac/tuplegenerationjob_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view tuplegenerationjobs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tuplegenerationjob-viewer-role +rules: +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs + verbs: + - get + - list + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs/status + verbs: + - get diff --git a/klyshko-operator/config/rbac/tuplegenerationtask_editor_role.yaml b/klyshko-operator/config/rbac/tuplegenerationtask_editor_role.yaml new file mode 100644 index 0000000..f8350ba --- /dev/null +++ b/klyshko-operator/config/rbac/tuplegenerationtask_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit tuplegenerationtasks. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tuplegenerationtask-editor-role +rules: +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks/status + verbs: + - get diff --git a/klyshko-operator/config/rbac/tuplegenerationtask_viewer_role.yaml b/klyshko-operator/config/rbac/tuplegenerationtask_viewer_role.yaml new file mode 100644 index 0000000..848d066 --- /dev/null +++ b/klyshko-operator/config/rbac/tuplegenerationtask_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view tuplegenerationtasks. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tuplegenerationtask-viewer-role +rules: +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks + verbs: + - get + - list + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks/status + verbs: + - get diff --git a/klyshko-operator/config/samples/apollo-vcp.yaml b/klyshko-operator/config/samples/apollo-vcp.yaml new file mode 100644 index 0000000..5e060b8 --- /dev/null +++ b/klyshko-operator/config/samples/apollo-vcp.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vcp-config +data: + playerId: "0" \ No newline at end of file diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml new file mode 100644 index 0000000..c56324e --- /dev/null +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -0,0 +1,7 @@ +apiVersion: klyshko.carbnyestack.io/v1alpha1 +kind: TupleGenerationJob +metadata: + name: tuplegenerationjob-sample +spec: + type: multiplicationtriple_gfp + count: 100000 diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml new file mode 100644 index 0000000..a1c9324 --- /dev/null +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml @@ -0,0 +1,7 @@ +apiVersion: klyshko.carbnyestack.io/v1alpha1 +kind: TupleGenerationTask +metadata: + name: tuplegenerationtask-sample +spec: + # Add fields here + foo: bar diff --git a/klyshko-operator/config/samples/kustomization.yaml b/klyshko-operator/config/samples/kustomization.yaml new file mode 100644 index 0000000..0b61aa2 --- /dev/null +++ b/klyshko-operator/config/samples/kustomization.yaml @@ -0,0 +1,5 @@ +## Append samples you want in your CSV to this file as resources ## +resources: +- klyshko_v1alpha1_tuplegenerationjob.yaml +- klyshko_v1alpha1_tuplegenerationtask.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/klyshko-operator/config/samples/starbuck-vcp.yaml b/klyshko-operator/config/samples/starbuck-vcp.yaml new file mode 100644 index 0000000..1761940 --- /dev/null +++ b/klyshko-operator/config/samples/starbuck-vcp.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vcp-config +data: + playerId: "1" \ No newline at end of file diff --git a/klyshko-operator/config/scorecard/bases/config.yaml b/klyshko-operator/config/scorecard/bases/config.yaml new file mode 100644 index 0000000..c770478 --- /dev/null +++ b/klyshko-operator/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/klyshko-operator/config/scorecard/kustomization.yaml b/klyshko-operator/config/scorecard/kustomization.yaml new file mode 100644 index 0000000..50cd2d0 --- /dev/null +++ b/klyshko-operator/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +#+kubebuilder:scaffold:patchesJson6902 diff --git a/klyshko-operator/config/scorecard/patches/basic.config.yaml b/klyshko-operator/config/scorecard/patches/basic.config.yaml new file mode 100644 index 0000000..c04db31 --- /dev/null +++ b/klyshko-operator/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: basic + test: basic-check-spec-test diff --git a/klyshko-operator/config/scorecard/patches/olm.config.yaml b/klyshko-operator/config/scorecard/patches/olm.config.yaml new file mode 100644 index 0000000..122f703 --- /dev/null +++ b/klyshko-operator/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/klyshko-operator/controllers/config_utils.go b/klyshko-operator/controllers/config_utils.go new file mode 100644 index 0000000..e6573b7 --- /dev/null +++ b/klyshko-operator/controllers/config_utils.go @@ -0,0 +1,47 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "errors" + "fmt" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "math" + "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" +) + +func localPlayerID(ctx context.Context, client *client.Client, namespace string) (uint, error) { + + // Get VCP configuration config map + name := types.NamespacedName{ + Namespace: namespace, + Name: "vcp-config", + } + cfm := &v1.ConfigMap{} + err := (*client).Get(ctx, name, cfm) + if err != nil { + return 0, errors.Unwrap(fmt.Errorf("VCP configuration not found: %w", err)) + } + + // Extract playerId + if playerId, ok := cfm.Data["playerId"]; ok { + pid, err := strconv.Atoi(playerId) + if err != nil { + return 0, err + } + if pid < 0 || pid > math.MaxUint32 { + return 0, fmt.Errorf("invalid playerId '%d'- must be in range [0,%d]", pid, math.MaxUint32) + } + return uint(pid), nil + } else { + return 0, errors.New("invalid VCP configuration - missing playerId") + } +} diff --git a/klyshko-operator/controllers/etcd_keys.go b/klyshko-operator/controllers/etcd_keys.go new file mode 100644 index 0000000..dd74eb0 --- /dev/null +++ b/klyshko-operator/controllers/etcd_keys.go @@ -0,0 +1,90 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "fmt" + "k8s.io/apimachinery/pkg/types" + "math" + "regexp" + "strconv" +) + +type Key interface { + ToEtcdKey() string +} + +type RosterKey struct { + types.NamespacedName +} + +func (k RosterKey) ToEtcdKey() string { + return rosterKey + "/" + k.Namespace + "/" + k.Name +} + +func (k RosterKey) String() string { + return k.ToEtcdKey() +} + +type RosterEntryKey struct { + RosterKey + PlayerID uint +} + +func (k RosterEntryKey) ToEtcdKey() string { + return rosterKey + "/" + k.Namespace + "/" + k.Name + "/" + strconv.Itoa(int(k.PlayerID)) +} + +func (k RosterEntryKey) String() string { + return k.ToEtcdKey() +} + +const rosterKey = "/klyshko/roster" + +var etcdRosterKeyPattern = regexp.MustCompile("^" + rosterKey + "/(?P(\\w|-)+)/(?P(\\w|-)+)(?:/(?P\\d+))?$") + +func etcdKeyParts(s string) map[string]string { + match := etcdRosterKeyPattern.FindStringSubmatch(s) + if match == nil { + return nil + } + result := make(map[string]string) + for i, name := range etcdRosterKeyPattern.SubexpNames() { + if i != 0 && name != "" && match[i] != "" { + result[name] = match[i] + } + } + return result +} + +func ParseKey(s string) (Key, error) { + parts := etcdKeyParts(s) + if parts == nil { + return nil, fmt.Errorf("not a key: %v", s) + } + name := types.NamespacedName{Name: parts["jobName"], Namespace: parts["namespace"]} + if playerID, ok := parts["localPlayerID"]; ok { + pid, err := strconv.Atoi(playerID) + if err != nil { + return nil, fmt.Errorf("invalid playerId '%v' - not an integer: %w", playerID, err) + } + if pid < 0 || pid > math.MaxUint32 { + return nil, fmt.Errorf("invalid playerId '%d' - must be in range [0,%d]", pid, math.MaxUint32) + } + return RosterEntryKey{ + RosterKey: RosterKey{ + name, + }, + PlayerID: uint(pid), + }, nil + } else { + return RosterKey{ + name, + }, nil + } +} diff --git a/klyshko-operator/controllers/suite_test.go b/klyshko-operator/controllers/suite_test.go new file mode 100644 index 0000000..9f2b3a8 --- /dev/null +++ b/klyshko-operator/controllers/suite_test.go @@ -0,0 +1,77 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Controller Suite", + []Reporter{printer.NewlineReporter{}}) +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = klyshkov1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = klyshkov1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = klyshkov1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go new file mode 100644 index 0000000..a15feaf --- /dev/null +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -0,0 +1,303 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "strconv" + "time" +) + +type TupleGenerationJobReconciler struct { + client.Client + Scheme *runtime.Scheme + EtcdClient *clientv3.Client + rosterWatcherCh clientv3.WatchChan +} + +func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Scheme, etcdClient *clientv3.Client) *TupleGenerationJobReconciler { + r := new(TupleGenerationJobReconciler) + r.Client = client + r.Scheme = scheme + r.EtcdClient = etcdClient + r.rosterWatcherCh = etcdClient.Watch(context.Background(), rosterKey, clientv3.WithPrefix()) // TODO Close channel? + go r.handleWatchEvents() + return r +} + +func (r *TupleGenerationJobReconciler) handleWatchEvents() { + ctx := context.Background() + for watchResponse := range r.rosterWatcherCh { + for _, ev := range watchResponse.Events { + r.handleWatchEvent(ctx, ev) + } + } +} + +func (r *TupleGenerationJobReconciler) handleWatchEvent(ctx context.Context, ev *clientv3.Event) { + logger := log.FromContext(ctx) + key, err := ParseKey(string(ev.Kv.Key)) + if err != nil { + logger.Error(err, "unexpected etcd watch event", "Event.Key", ev.Kv.Key) + } + logger.Info("processing roster event", "Key", key) + + switch k := key.(type) { + case RosterEntryKey: + // Skip if update is for local task + local, err := isLocalTaskKey(ctx, &r.Client, k) + if err != nil { + logger.Error(err, "failed to check task type") + return + } + if local { + return + } + r.handleRemoteTaskUpdate(ctx, k, ev) + case RosterKey: + r.handleJobUpdate(ctx, k, ev) + } +} + +func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key RosterKey, ev *clientv3.Event) { + logger := log.FromContext(ctx).WithValues("Key", key) + switch ev.Type { + case mvccpb.PUT: + // Get job spec from etcd K/V pair + jobSpec := &klyshkov1alpha1.TupleGenerationJobSpec{} + err := json.Unmarshal(ev.Kv.Value, jobSpec) + if err != nil { + logger.Error(err, "failed to unmarshal spec") + return + } + // TODO Create or update depending on whether Job already exists + err = r.createJob(ctx, key.NamespacedName, jobSpec) + if err != nil { + logger.Error(err, "failed to create job") + return + } + logger.Info("job created") + case mvccpb.DELETE: + // Delete job iff exists + found := &klyshkov1alpha1.TupleGenerationJob{} + err := r.Client.Get(ctx, key.NamespacedName, found) + if err != nil { + if apierrors.IsNotFound(err) { + logger.Error(err, "job does not exist - ignoring") + return + } + logger.Error(err, "failed to read job resource") + return + } + err = r.Delete(ctx, found) + if err != nil { + logger.Error(err, "job deletion failed") + return + } + logger.Info("job deleted") + } +} + +func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Context, key RosterEntryKey, ev *clientv3.Event) { + taskName := types.NamespacedName{ + Namespace: key.Namespace, + Name: key.Name + "-" + strconv.Itoa(int(key.PlayerID)), + } + logger := log.FromContext(ctx).WithValues("Task.Key", key) + + // Lookup job + job := &klyshkov1alpha1.TupleGenerationJob{} + err := r.Get(ctx, key.NamespacedName, job) + if err != nil { + logger.Error(err, "failed to read job resource") + return + } + + switch ev.Type { + case mvccpb.PUT: + // Create local proxy for remote task + // TODO Check if task already exists; then update or create + task, err := r.taskForJob(job, key.PlayerID) + if err != nil { + logger.Error(err, "failed to define proxy task") + return + } + err = r.Create(ctx, task) + if err != nil { + logger.Error(err, "failed to create proxy task") + return + } + case mvccpb.DELETE: + // Delete task for job iff exists + task := &klyshkov1alpha1.TupleGenerationTask{} + err = r.Get(ctx, taskName, task) + if err != nil { + if apierrors.IsNotFound(err) { + logger.Error(err, "proxy task does not exist - ignoring") + return + } + logger.Error(err, "failed to read proxy task resource") + return + } + err = r.Delete(ctx, task) + if err != nil { + logger.Error(err, "proxy task deletion failed") + return + } + logger.Info("proxy task deleted") + } +} + +func (r *TupleGenerationJobReconciler) createJob(ctx context.Context, name types.NamespacedName, jobSpec *klyshkov1alpha1.TupleGenerationJobSpec) error { + logger := log.FromContext(ctx).WithValues("Job.Name", name) + found := &klyshkov1alpha1.TupleGenerationJob{} + err := r.Client.Get(ctx, name, found) + if err != nil { + if apierrors.IsNotFound(err) { + job := &klyshkov1alpha1.TupleGenerationJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: name.Name, + Namespace: name.Namespace, + }, + Spec: *jobSpec, + } + logger.Info("creating a new job") + return r.Create(ctx, job) + } + return err + } + return nil +} + +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch + +func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + jobKey := RosterKey{ + req.NamespacedName, + } + logger := log.FromContext(ctx).WithValues("Job.Key", jobKey) + + // Cleanup if job has been deleted + job := &klyshkov1alpha1.TupleGenerationJob{} + err := r.Get(ctx, req.NamespacedName, job) + if err != nil { + if apierrors.IsNotFound(err) { + // Job resource not available -> has been deleted + _, err := r.EtcdClient.Delete(ctx, jobKey.ToEtcdKey()) + if err != nil { + logger.Error(err, "failed to delete roster") + return ctrl.Result{}, err + } + logger.Info("roster deleted") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + logger.Error(err, "failed to read job resource") + return ctrl.Result{}, err + } else { + logger.Info("job exists already") + } + + // Create roster if not existing (no etcd transaction needed as remote job creation is triggered by roster creation) + resp, err := r.EtcdClient.Get(ctx, jobKey.ToEtcdKey()) + if err != nil { + logger.Error(err, "failed to fetch roster", "Roster.Key", jobKey) + return ctrl.Result{}, err + } + if resp.Count == 0 { + encoded, err := json.Marshal(job.Spec) + if err != nil { + fmt.Println(err, "failed to marshal specification") + return ctrl.Result{}, err + } + _, err = r.EtcdClient.Put(ctx, jobKey.ToEtcdKey(), string(encoded)) + if err != nil { + logger.Error(err, "failed to create roster") + return ctrl.Result{}, err + } + logger.Info("roster created") + } else { + logger.Info("roster exists already") + } + + // Create local task if not existing + playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) + if err != nil { + logger.Error(err, "can't read playerId from VCP configuration") + return ctrl.Result{RequeueAfter: 60 * time.Second}, err + } + task := &klyshkov1alpha1.TupleGenerationTask{} + err = r.Get(ctx, types.NamespacedName{ + Namespace: job.Namespace, + Name: r.taskName(job, playerID), + }, task) + if err != nil { + if apierrors.IsNotFound(err) { + // Create new local task for job + task, err = r.taskForJob(job, playerID) + if err != nil { + logger.Error(err, "failed to define local task", "Task.Name", task.Name) + return ctrl.Result{}, err + } + err = r.Create(ctx, task) + if err != nil { + logger.Error(err, "failed to create local task", "Task.Name", task.Name) + return ctrl.Result{}, err + } + logger.Info("local task created", "Task.Name", task.Name) + return ctrl.Result{Requeue: true}, nil + } + // Error reading resource, requeue + logger.Error(err, "failed to read task resource") + return ctrl.Result{}, err + } else { + logger.Info("local task exists already") + } + + logger.Info("desired state reached") + return ctrl.Result{}, nil +} + +func (r *TupleGenerationJobReconciler) taskName(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) string { + return job.Name + "-" + strconv.Itoa(int(playerID)) +} + +func (r *TupleGenerationJobReconciler) taskForJob(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) (*klyshkov1alpha1.TupleGenerationTask, error) { + task := &klyshkov1alpha1.TupleGenerationTask{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.taskName(job, playerID), + Namespace: job.Namespace, + }, + Spec: klyshkov1alpha1.TupleGenerationTaskSpec{}, + Status: klyshkov1alpha1.TupleGenerationTaskStatus{}, + } + err := ctrl.SetControllerReference(job, task, r.Scheme) + return task, err +} + +func (r *TupleGenerationJobReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&klyshkov1alpha1.TupleGenerationJob{}). + Owns(&klyshkov1alpha1.TupleGenerationTask{}). + Complete(r) +} diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go new file mode 100644 index 0000000..4730fe7 --- /dev/null +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -0,0 +1,123 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "fmt" + clientv3 "go.etcd.io/etcd/client/v3" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" +) + +type TupleGenerationTaskReconciler struct { + client.Client + Scheme *runtime.Scheme + EtcdClient *clientv3.Client +} + +func (r *TupleGenerationTaskReconciler) taskKeyFromName(namespace string, name string) (*RosterEntryKey, error) { + p := strings.Split(name, "-") + pid := p[len(p)-1] + jobName := strings.Join(p[:len(p)-1], "-") + key, err := ParseKey(rosterKey + "/" + namespace + "/" + jobName + "/" + pid) + if err != nil { + return nil, err + } + rev, ok := key.(RosterEntryKey) + if !ok { + return nil, fmt.Errorf("not a task key: %v", key) + } + return &rev, nil +} + +func isLocalTaskKey(ctx context.Context, client *client.Client, key RosterEntryKey) (bool, error) { + playerID, err := localPlayerID(ctx, client, key.Namespace) + if err != nil { + return false, fmt.Errorf("can't read local player ID: %w", err) + } + return uint(playerID) == key.PlayerID, nil +} + +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/finalizers,verbs=update + +func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithValues("Task.Name", req.Name) + taskKey, err := r.taskKeyFromName(req.Namespace, req.Name) + if err != nil { + logger.Error(err, "failed to get task key") + return ctrl.Result{}, err + } + + // Skip if remote task + local, err := isLocalTaskKey(ctx, &r.Client, *taskKey) + if err != nil { + logger.Error(err, "failed to check task type") + return ctrl.Result{}, err + } + if !local { + return ctrl.Result{}, nil + } + + // Cleanup if task has been deleted + task := &klyshkov1alpha1.TupleGenerationTask{} + err = r.Get(ctx, req.NamespacedName, task) + if err != nil { + if apierrors.IsNotFound(err) { + // Task resource not available -> has been deleted + _, err := r.EtcdClient.Delete(ctx, taskKey.ToEtcdKey()) + if err != nil { + logger.Error(err, "failed to delete roster entry") + return ctrl.Result{}, err + } + logger.Info("roster entry deleted") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + logger.Error(err, "failed to read task resource") + return ctrl.Result{}, err + } else { + logger.Info("task exists already") + } + + // Create roster entry if not existing + resp, err := r.EtcdClient.Get(ctx, taskKey.ToEtcdKey()) + if err != nil { + logger.Error(err, "failed to fetch roster entry", "RosterEntry.Key", taskKey) + return ctrl.Result{}, err + } + if resp.Count == 0 { + _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), "") + if err != nil { + logger.Error(err, "failed to create roster entry") + return ctrl.Result{}, err + } + logger.Info("roster entry created") + } else { + logger.Info("roster entry exists already") + } + + // TODO Create POD using KII + + return ctrl.Result{}, nil +} + +func (r *TupleGenerationTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&klyshkov1alpha1.TupleGenerationTask{}). + Complete(r) +} diff --git a/klyshko-operator/deploy.sh b/klyshko-operator/deploy.sh new file mode 100755 index 0000000..5e9d673 --- /dev/null +++ b/klyshko-operator/deploy.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +declare -a CLUSTERS=("starbuck" "apollo") + +echo -e "${GREEN}Undeploying from clusters${NC}" +for c in "${CLUSTERS[@]}" +do + echo -e "${GREEN}Undeploying from $c${NC}" + kubectl config use-context "kind-$c" + if [ "$c" == "apollo" ]; then + kubectl delete -f config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml + fi + kubectl delete --all tgj + kubectl delete --all tgt + make undeploy IMG="carbynestack/klyshko-operator:v0.0.1" +done + +echo -e "${GREEN}Cleaning up etcd${NC}" +bin/etcdctl --endpoints 172.18.1.129:2379 del "/klyshko/roster" --prefix + +echo -e "${GREEN}Building code and image${NC}" +make docker-build IMG="carbynestack/klyshko-operator:v0.0.1" + +echo -e "${GREEN}Loading docker images into cluster registries${NC}" +for c in "${CLUSTERS[@]}" +do + echo -e "${GREEN}Loading docker image into $c${NC}" + kind load docker-image carbynestack/klyshko-operator:v0.0.1 --name "$c" +done + +for c in "${CLUSTERS[@]}" +do + echo -e "${GREEN}Deploying in $c${NC}" + kubectl config use-context "kind-$c" + kubectl apply -f "config/samples/$c-vcp.yaml" + make deploy IMG="carbynestack/klyshko-operator:v0.0.1" + if [ "$c" == "apollo" ]; then + kubectl apply -f config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml + fi +done \ No newline at end of file diff --git a/klyshko-operator/go.mod b/klyshko-operator/go.mod new file mode 100644 index 0000000..164a315 --- /dev/null +++ b/klyshko-operator/go.mod @@ -0,0 +1,14 @@ +module github.com/carbynestack/klyshko + +go 1.16 + +require ( + github.com/onsi/ginkgo v1.16.4 + github.com/onsi/gomega v1.13.0 + go.etcd.io/etcd/api/v3 v3.5.2 + go.etcd.io/etcd/client/v3 v3.5.2 + k8s.io/api v0.21.2 + k8s.io/apimachinery v0.21.2 + k8s.io/client-go v0.21.2 + sigs.k8s.io/controller-runtime v0.9.2 +) diff --git a/klyshko-operator/go.sum b/klyshko-operator/go.sum new file mode 100644 index 0000000..b3f9391 --- /dev/null +++ b/klyshko-operator/go.sum @@ -0,0 +1,784 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= +github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI= +go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE= +go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v3 v3.5.2 h1:WdnejrUtQC4nCxK0/dLTMqKOB+U5TP/2Ya0BJL+1otA= +go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= +golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= +k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= +k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE= +k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA= +k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= +k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= +k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw= +k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0= +k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= +k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= +k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4= +k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s= +k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/controller-runtime v0.9.2 h1:MnCAsopQno6+hI9SgJHKddzXpmv2wtouZz6931Eax+Q= +sigs.k8s.io/controller-runtime v0.9.2/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/klyshko-operator/hack/boilerplate.go.txt b/klyshko-operator/hack/boilerplate.go.txt new file mode 100644 index 0000000..29c55ec --- /dev/null +++ b/klyshko-operator/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go new file mode 100644 index 0000000..4e940f8 --- /dev/null +++ b/klyshko-operator/main.go @@ -0,0 +1,113 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package main + +import ( + "flag" + "os" + "time" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + clientv3 "go.etcd.io/etcd/client/v3" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/carbynestack/klyshko/controllers" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(klyshkov1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "02e9e224.carbnyestack.io", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + etcdClient, err := clientv3.New(clientv3.Config{ + Endpoints: []string{"172.18.1.129:2379"}, + DialTimeout: 5 * time.Second, + }) + if err != nil { + setupLog.Error(err, "unable to create etcd client", "controller", "TupleGenerationJob") + os.Exit(1) + } + defer etcdClient.Close() + + if err = controllers.NewTupleGenerationJobReconciler(mgr.GetClient(), mgr.GetScheme(), etcdClient).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationJob") + os.Exit(1) + } + + if err = (&controllers.TupleGenerationTaskReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + EtcdClient: etcdClient, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationTask") + os.Exit(1) + } + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} From 71e11050a81ea5f0062ed24a627eae17a0c8973c Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 11 Mar 2022 17:18:20 +0100 Subject: [PATCH 02/69] Launch generator pod Signed-off-by: Sven Trieflinger --- deploy.sh | 19 +++ .../api/v1alpha1/tuplegenerationjob_types.go | 1 + ...o.carbnyestack.io_tuplegenerationjobs.yaml | 3 + klyshko-operator/config/rbac/role.yaml | 12 ++ .../samples/engine-params-extra.yaml.template | 15 ++ .../engine-params-secret.yaml.template | 16 ++ .../config/samples/engine-params.yaml | 13 ++ .../klyshko_v1alpha1_tuplegenerationjob.yaml | 1 + .../tuplegenerationtask_controller.go | 151 +++++++++++++++++- klyshko-operator/deploy.sh | 10 ++ klyshko-operator/go.mod | 1 + klyshko-operator/go.sum | 3 +- 12 files changed, 243 insertions(+), 2 deletions(-) create mode 100755 deploy.sh create mode 100644 klyshko-operator/config/samples/engine-params-extra.yaml.template create mode 100644 klyshko-operator/config/samples/engine-params-secret.yaml.template create mode 100644 klyshko-operator/config/samples/engine-params.yaml diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..31368b6 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +kubectl config use-context "kind-apollo" +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install test-etcd --set auth.rbac.enabled=false --set service.type=LoadBalancer bitnami/etcd + +( +cd klyshko-mp-spdz +./deploy.sh +) +( +cd klyshko-operator +./deploy.sh +) \ No newline at end of file diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go index b4894f9..1bc9745 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -12,6 +12,7 @@ import ( ) type TupleGenerationJobSpec struct { + ID string `json:"id"` // +kubebuilder:validation:Enum=multiplicationtriple_gfp Type string `json:"type"` diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml index a333360..ec499e2 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -52,12 +52,15 @@ spec: format: int32 minimum: 0 type: integer + id: + type: string type: enum: - multiplicationtriple_gfp type: string required: - count + - id - type type: object status: diff --git a/klyshko-operator/config/rbac/role.yaml b/klyshko-operator/config/rbac/role.yaml index 971d5ef..1f55cc8 100644 --- a/klyshko-operator/config/rbac/role.yaml +++ b/klyshko-operator/config/rbac/role.yaml @@ -14,6 +14,18 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - klyshko.carbnyestack.io resources: diff --git a/klyshko-operator/config/samples/engine-params-extra.yaml.template b/klyshko-operator/config/samples/engine-params-extra.yaml.template new file mode 100644 index 0000000..9f5215f --- /dev/null +++ b/klyshko-operator/config/samples/engine-params-extra.yaml.template @@ -0,0 +1,15 @@ +# +# Copyright (c) 2021 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params.extra +data: + mac_key_share_p: | + MAC_KEY_SHARE_P + mac_key_share_2: | + MAC_KEY_SHARE_2 \ No newline at end of file diff --git a/klyshko-operator/config/samples/engine-params-secret.yaml.template b/klyshko-operator/config/samples/engine-params-secret.yaml.template new file mode 100644 index 0000000..b2ee084 --- /dev/null +++ b/klyshko-operator/config/samples/engine-params-secret.yaml.template @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Secret +metadata: + name: io.carbynestack.engine.params.secret +type: Opaque +data: + mac_key_share_p: | + MAC_KEY_SHARE_P + mac_key_share_2: | + MAC_KEY_SHARE_2 \ No newline at end of file diff --git a/klyshko-operator/config/samples/engine-params.yaml b/klyshko-operator/config/samples/engine-params.yaml new file mode 100644 index 0000000..cbf2847 --- /dev/null +++ b/klyshko-operator/config/samples/engine-params.yaml @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params +data: + prime: | + 198766463529478683931867765928436695041 \ No newline at end of file diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index c56324e..ade065d 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -3,5 +3,6 @@ kind: TupleGenerationJob metadata: name: tuplegenerationjob-sample spec: + id: e8e54706-edc9-4a0c-b6bd-14368daf4ebe type: multiplicationtriple_gfp count: 100000 diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 4730fe7..06cde40 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -11,7 +11,10 @@ import ( "context" "fmt" clientv3 "go.etcd.io/etcd/client/v3" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "strings" "k8s.io/apimachinery/pkg/runtime" @@ -54,6 +57,7 @@ func isLocalTaskKey(ctx context.Context, client *client.Client, key RosterEntryK //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/status,verbs=get;update;patch //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues("Task.Name", req.Name) @@ -111,7 +115,19 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. logger.Info("roster entry exists already") } - // TODO Create POD using KII + // Lookup job + job := &klyshkov1alpha1.TupleGenerationJob{} + err = r.Get(ctx, taskKey.NamespacedName, job) + if err != nil { + logger.Error(err, "failed to lookup job") + return ctrl.Result{}, err + } + + // Create Pod + err = r.createCRGPod(ctx, *taskKey, job, task) + if err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, nil } @@ -119,5 +135,138 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. func (r *TupleGenerationTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&klyshkov1alpha1.TupleGenerationTask{}). + Owns(&v1.Pod{}). Complete(r) } + +func (r *TupleGenerationTaskReconciler) createCRGPod(ctx context.Context, key RosterEntryKey, job *klyshkov1alpha1.TupleGenerationJob, task *klyshkov1alpha1.TupleGenerationTask) error { + logger := log.FromContext(ctx).WithValues("Task.Key", key) + found := &v1.Pod{} + err := r.Get(ctx, key.NamespacedName, found) + if err == nil { + logger.Info("pod already exists") + return nil + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.Name, + Namespace: task.Namespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "generator", + Image: "carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT", // TODO Read from config + Command: []string{ + "/bin/bash", + "-c", + }, + Args: []string{ + "./kii-run.sh", + }, + Env: []v1.EnvVar{ + { + Name: "KII_JOB_ID", + Value: job.Spec.ID, + }, + { + Name: "KII_PLAYER_COUNT", + Value: "2", // TODO Read from config + }, + { + Name: "KII_TUPLE_TYPE", + Value: job.Spec.Type, + }, + { + Name: "KII_TUPLES_PER_JOB", + Value: fmt.Sprint(job.Spec.Count), + }, + { + Name: "KII_PLAYER_NUMBER", + Value: fmt.Sprint(key.PlayerID), + }, + { + Name: "KII_SHARED_FOLDER", + Value: "/kii", // TODO Read from config + }, + { + Name: "KII_TUPLE_FILE", + Value: "/kii/tuples", + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kii", + MountPath: "/kii", + }, + { + Name: "params", + ReadOnly: true, + MountPath: "/etc/kii/params", + }, + { + Name: "secret-params", + ReadOnly: true, + MountPath: "/etc/kii/secret-params", + }, + { + Name: "extra-params", + ReadOnly: true, + MountPath: "/etc/kii/extra-params", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + ShareProcessNamespace: pointer.Bool(true), + Volumes: []v1.Volume{ + { + Name: "kii", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "params", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "io.carbynestack.engine.params", + }, + }, + }, + }, + { + Name: "secret-params", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "io.carbynestack.engine.params.secret", + }, + }, + }, + { + Name: "extra-params", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "io.carbynestack.engine.params.extra", + }, + }, + }, + }, + }, + }, + } + logger.Info("creating pod", "Pod", pod) + err = ctrl.SetControllerReference(task, pod, r.Scheme) + if err != nil { + logger.Error(err, "setting owner reference failed") + return err + } + err = r.Create(ctx, pod) + if err != nil { + logger.Error(err, "pod creation failed") + return err + } + return nil +} diff --git a/klyshko-operator/deploy.sh b/klyshko-operator/deploy.sh index 5e9d673..f0f0e4b 100755 --- a/klyshko-operator/deploy.sh +++ b/klyshko-operator/deploy.sh @@ -41,6 +41,16 @@ do echo -e "${GREEN}Deploying in $c${NC}" kubectl config use-context "kind-$c" kubectl apply -f "config/samples/$c-vcp.yaml" + MAC_KEY_SHARE_P=$([ "$c" == "apollo" ] && echo "-88222337191559387830816715872691188861" | base64 || echo "1113507028231509545156335486838233835" | base64) + MAC_KEY_SHARE_2=$([ "$c" == "apollo" ] && echo "f0cf6099e629fd0bda2de3f9515ab72b" | base64 || echo "c347ce3d9e165e4e85221f9da7591d98" | base64) + sed -e "s/MAC_KEY_SHARE_P/${MAC_KEY_SHARE_P}/" -e "s/MAC_KEY_SHARE_2/${MAC_KEY_SHARE_2}/" config/samples/engine-params-secret.yaml.template > "/tmp/$c-engine-params-secret.yaml" + EXTRA_MAC_KEY_SHARE_P=$([ "$c" == "starbuck" ] && echo "-88222337191559387830816715872691188861" || echo "1113507028231509545156335486838233835") + EXTRA_MAC_KEY_SHARE_2=$([ "$c" == "starbuck" ] && echo "f0cf6099e629fd0bda2de3f9515ab72b" || echo "c347ce3d9e165e4e85221f9da7591d98") + sed -e "s/MAC_KEY_SHARE_P/${EXTRA_MAC_KEY_SHARE_P}/" -e "s/MAC_KEY_SHARE_2/${EXTRA_MAC_KEY_SHARE_2}/" config/samples/engine-params-extra.yaml.template > "/tmp/$c-engine-params-extra.yaml" + kubectl apply -f /tmp/$c-engine-params-secret.yaml + kubectl apply -f /tmp/$c-engine-params-extra.yaml + kubectl apply -f config/samples/engine-params.yaml + make deploy IMG="carbynestack/klyshko-operator:v0.0.1" if [ "$c" == "apollo" ]; then kubectl apply -f config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml diff --git a/klyshko-operator/go.mod b/klyshko-operator/go.mod index 164a315..e1626eb 100644 --- a/klyshko-operator/go.mod +++ b/klyshko-operator/go.mod @@ -10,5 +10,6 @@ require ( k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 sigs.k8s.io/controller-runtime v0.9.2 ) diff --git a/klyshko-operator/go.sum b/klyshko-operator/go.sum index b3f9391..662c1a7 100644 --- a/klyshko-operator/go.sum +++ b/klyshko-operator/go.sum @@ -768,8 +768,9 @@ k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s= k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From d23f3aa71cb3ff92d6b85bab4b97573b8f3cb16d Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 8 Apr 2022 15:18:55 +0200 Subject: [PATCH 03/69] Upload tuples and track task status Signed-off-by: Sven Trieflinger --- deploy.sh | 4 + .../api/v1alpha1/tuplegenerationtask_types.go | 31 ++ ....carbnyestack.io_tuplegenerationtasks.yaml | 14 +- klyshko-operator/config/rbac/role.yaml | 12 + .../klyshko_v1alpha1_tuplegenerationjob.yaml | 2 +- ..._v1alpha1_tuplegenerationjob.yaml.template | 8 + klyshko-operator/controllers/castor.go | 36 +++ .../tuplegenerationjob_controller.go | 51 +++- .../tuplegenerationtask_controller.go | 274 +++++++++++++++++- klyshko-operator/deploy.sh | 2 + klyshko-operator/go.mod | 3 +- klyshko-provisioner/Dockerfile | 22 ++ klyshko-provisioner/deploy.sh | 21 ++ klyshko-provisioner/kii-provisioner-run.sh | 49 ++++ 14 files changed, 497 insertions(+), 32 deletions(-) create mode 100644 klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template create mode 100644 klyshko-operator/controllers/castor.go create mode 100644 klyshko-provisioner/Dockerfile create mode 100755 klyshko-provisioner/deploy.sh create mode 100755 klyshko-provisioner/kii-provisioner-run.sh diff --git a/deploy.sh b/deploy.sh index 31368b6..66a1a6e 100755 --- a/deploy.sh +++ b/deploy.sh @@ -14,6 +14,10 @@ cd klyshko-mp-spdz ./deploy.sh ) ( +cd klyshko-provisioner +./deploy.sh +) +( cd klyshko-operator ./deploy.sh ) \ No newline at end of file diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go index 0130403..5715287 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go @@ -8,18 +8,49 @@ SPDX-License-Identifier: Apache-2.0 package v1alpha1 import ( + "encoding/json" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type TupleGenerationTaskState string + +const ( + Launching TupleGenerationTaskState = "Launching" + Generating = "Generating" + Provisioning = "Provisioning" + Completed = "Completed" + Failed = "Failed" +) + +func (s TupleGenerationTaskState) IsValid() bool { + switch s { + case Launching, Generating, Provisioning, Completed, Failed: + return true + default: + return false + } +} + type TupleGenerationTaskSpec struct { } type TupleGenerationTaskStatus struct { + State TupleGenerationTaskState `json:"state"` +} + +func ParseFromJSON(data []byte) (*TupleGenerationTaskStatus, error) { + status := &TupleGenerationTaskStatus{} + if err := json.Unmarshal(data, status); err != nil { + return status, err + } + return status, nil } //+kubebuilder:object:root=true //+kubebuilder:resource:shortName=tgt;tgtask //+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state` +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" type TupleGenerationTask struct { metav1.TypeMeta `json:",inline"` diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml index 4c3b3e1..5ffa1e2 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml @@ -19,7 +19,14 @@ spec: singular: tuplegenerationtask scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: properties: @@ -38,6 +45,11 @@ spec: spec: type: object status: + properties: + state: + type: string + required: + - state type: object type: object served: true diff --git a/klyshko-operator/config/rbac/role.yaml b/klyshko-operator/config/rbac/role.yaml index 1f55cc8..8c44b9b 100644 --- a/klyshko-operator/config/rbac/role.yaml +++ b/klyshko-operator/config/rbac/role.yaml @@ -14,6 +14,18 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index ade065d..8d71a7a 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -3,6 +3,6 @@ kind: TupleGenerationJob metadata: name: tuplegenerationjob-sample spec: - id: e8e54706-edc9-4a0c-b6bd-14368daf4ebe + id: fa992993-080a-4328-8416-3ec089adfdfe type: multiplicationtriple_gfp count: 100000 diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template new file mode 100644 index 0000000..abc6b67 --- /dev/null +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template @@ -0,0 +1,8 @@ +apiVersion: klyshko.carbnyestack.io/v1alpha1 +kind: TupleGenerationJob +metadata: + name: tuplegenerationjob-sample +spec: + id: JOB_ID + type: multiplicationtriple_gfp + count: 100000 diff --git a/klyshko-operator/controllers/castor.go b/klyshko-operator/controllers/castor.go new file mode 100644 index 0000000..3bc3fd4 --- /dev/null +++ b/klyshko-operator/controllers/castor.go @@ -0,0 +1,36 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "fmt" + "github.com/google/uuid" + "io/ioutil" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func activateTupleChunk(ctx context.Context, chunkId uuid.UUID) error { + logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkId) + client := &http.Client{} + url := fmt.Sprintf("http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", chunkId) // TODO Make servername configurable / use discovery + logger.Info("activating tuple chunk with castor URL", "URL", url) + req, err := http.NewRequest(http.MethodPut, url, nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + logger.Info("response from castor", "Status", resp.Status, "Body", body) // TODO Body is Base64 encoded :-( + return err +} diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index a15feaf..f261adb 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -116,10 +116,6 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key } func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Context, key RosterEntryKey, ev *clientv3.Event) { - taskName := types.NamespacedName{ - Namespace: key.Namespace, - Name: key.Name + "-" + strconv.Itoa(int(key.PlayerID)), - } logger := log.FromContext(ctx).WithValues("Task.Key", key) // Lookup job @@ -130,19 +126,44 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex return } + taskName := types.NamespacedName{ + Namespace: key.Namespace, + Name: r.taskName(job, key.PlayerID), + } + switch ev.Type { case mvccpb.PUT: - // Create local proxy for remote task - // TODO Check if task already exists; then update or create - task, err := r.taskForJob(job, key.PlayerID) - if err != nil { - logger.Error(err, "failed to define proxy task") - return - } - err = r.Create(ctx, task) - if err != nil { - logger.Error(err, "failed to create proxy task") - return + found := &klyshkov1alpha1.TupleGenerationTask{} + if err := r.Client.Get(ctx, taskName, found); err == nil { + // Update local proxy task status + status, err := klyshkov1alpha1.ParseFromJSON(ev.Kv.Value) + if err != nil { + logger.Error(err, "extracting state from roster entry failed") + return + } + found.Status = *status + err = r.Client.Status().Update(ctx, found) + if err != nil { + logger.Error(err, "failed to update proxy task") + return + } + } else { + if !apierrors.IsNotFound(err) { + logger.Error(err, "failed to fetch task") // TODO Failure requires reconciliation (also above/below), how to do that? + return + } + + // Create local proxy for remote task + task, err := r.taskForJob(job, key.PlayerID) + if err != nil { + logger.Error(err, "failed to define proxy task") + return + } + err = r.Create(ctx, task) + if err != nil { + logger.Error(err, "failed to create proxy task") + return + } } case mvccpb.DELETE: // Delete task for job iff exists diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 06cde40..a361190 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -9,12 +9,16 @@ package controllers import ( "context" + "encoding/json" "fmt" + "github.com/google/uuid" clientv3 "go.etcd.io/etcd/client/v3" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/apimachinery/pkg/types" + "strconv" "strings" "k8s.io/apimachinery/pkg/runtime" @@ -51,13 +55,48 @@ func isLocalTaskKey(ctx context.Context, client *client.Client, key RosterEntryK if err != nil { return false, fmt.Errorf("can't read local player ID: %w", err) } - return uint(playerID) == key.PlayerID, nil + return playerID == key.PlayerID, nil +} + +func (r *TupleGenerationTaskReconciler) getStatus(ctx context.Context, taskKey RosterEntryKey) (*klyshkov1alpha1.TupleGenerationTaskStatus, error) { + resp, err := r.EtcdClient.Get(ctx, taskKey.ToEtcdKey()) + if err != nil { + return nil, err + } + if len(resp.Kvs) != 1 { + return nil, fmt.Errorf("no status available for roster entry: %v", taskKey) + } + status, err := klyshkov1alpha1.ParseFromJSON(resp.Kvs[0].Value) + if err != nil { + return nil, err + } + if !status.State.IsValid() { + return nil, fmt.Errorf("status contains invalid state: %s", status.State) + } + return status, nil +} + +func (r *TupleGenerationTaskReconciler) setStatus(ctx context.Context, taskKey RosterEntryKey, status *klyshkov1alpha1.TupleGenerationTaskStatus) error { + encoded, err := json.Marshal(status) + if err != nil { + return err + } + _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), string(encoded)) + return err +} + +func (r *TupleGenerationTaskReconciler) setState(ctx context.Context, taskKey RosterEntryKey, status *klyshkov1alpha1.TupleGenerationTaskStatus, state klyshkov1alpha1.TupleGenerationTaskState) error { + logger := log.FromContext(ctx).WithValues("Task.Key", taskKey) + logger.Info("task transitioning into new state", "from", status.State, "to", state) + status.State = state + return r.setStatus(ctx, taskKey, status) } //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/status,verbs=get;update;patch //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues("Task.Name", req.Name) @@ -105,7 +144,8 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, err } if resp.Count == 0 { - _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), "") + status, err := json.Marshal(&klyshkov1alpha1.TupleGenerationTaskStatus{State: klyshkov1alpha1.Launching}) + _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), string(status)) if err != nil { logger.Error(err, "failed to create roster entry") return ctrl.Result{}, err @@ -114,6 +154,11 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. } else { logger.Info("roster entry exists already") } + status, err := r.getStatus(ctx, *taskKey) + if err != nil { + logger.Error(err, "task status not available") + return ctrl.Result{}, err + } // Lookup job job := &klyshkov1alpha1.TupleGenerationJob{} @@ -123,12 +168,86 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, err } - // Create Pod - err = r.createCRGPod(ctx, *taskKey, job, task) + // Update the status according to state in etcd + taskStatus, err := r.getStatus(ctx, *taskKey) if err != nil { return ctrl.Result{}, err } + task.Status = *taskStatus + if err := r.Status().Update(ctx, task); err != nil { + logger.Error(err, "unable to update task status") + return ctrl.Result{}, err + } + if status.State == klyshkov1alpha1.Launching { + // Create persistent volume claim used to store generated tuples, if not existing + err = r.createPVC(ctx, *taskKey) + if err != nil { + return ctrl.Result{}, err + } + + // Create generator pod if not existing + _, err := r.createGeneratorPod(ctx, *taskKey, job, task) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{ + Requeue: true, + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Generating) + } + + if status.State == klyshkov1alpha1.Generating { + genPod, err := r.getGeneratorPod(ctx, task) + if err != nil { + return ctrl.Result{}, err + } + switch genPod.Status.Phase { + case v1.PodSucceeded: + // Generation successful, create provisioner pod to upload tuple shares to VCP-local castor + _, err := r.createProvisionerPod(ctx, *taskKey, job, task) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{ + Requeue: true, + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Provisioning) + case v1.PodFailed: + return ctrl.Result{ + Requeue: true, + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Failed) + } + } + + if status.State == klyshkov1alpha1.Provisioning { + provPod, err := r.getProvisionerPod(ctx, *taskKey) + if err != nil { + return ctrl.Result{}, err + } + switch provPod.Status.Phase { + case v1.PodSucceeded: + // Provisioning successful, activate tuples TODO Move to Job Controller who knows when all parties successfully provisioned + tupleChunkId, err := uuid.Parse(job.Spec.ID) + if err != nil { + logger.Error(err, "invalid job id encountered") + return ctrl.Result{ + Requeue: true, + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Failed) + } + err = activateTupleChunk(ctx, tupleChunkId) + if err != nil { + return ctrl.Result{}, err // TODO Fail here? + } + return ctrl.Result{ + Requeue: true, + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Completed) + case v1.PodFailed: + return ctrl.Result{ + Requeue: true, + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Failed) + } + } + + logger.Info("desired state reached") return ctrl.Result{}, nil } @@ -139,18 +258,144 @@ func (r *TupleGenerationTaskReconciler) SetupWithManager(mgr ctrl.Manager) error Complete(r) } -func (r *TupleGenerationTaskReconciler) createCRGPod(ctx context.Context, key RosterEntryKey, job *klyshkov1alpha1.TupleGenerationJob, task *klyshkov1alpha1.TupleGenerationTask) error { +func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key RosterEntryKey) error { logger := log.FromContext(ctx).WithValues("Task.Key", key) + name := types.NamespacedName{ + Name: key.Name + "-" + strconv.Itoa(int(key.PlayerID)), // TODO Error-prone, + Namespace: key.Namespace, + } + found := &v1.PersistentVolumeClaim{} + err := r.Get(ctx, name, found) + if err == nil { + logger.Info("persistent volume claim already exists") + return nil + } + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name.Name, + Namespace: name.Namespace, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "storage": resource.MustParse("100Mi"), + }, + }, + }, + } + logger.Info("creating persistent volume claim", "PVC", pvc) + err = r.Create(ctx, pvc) + if err != nil { + logger.Error(err, "persistent volume claim creation failed") + return err + } + return nil +} + +func (r *TupleGenerationTaskReconciler) getProvisionerPod(ctx context.Context, key RosterEntryKey) (*v1.Pod, error) { + name := types.NamespacedName{ + Name: key.Name + "-provisioner", + Namespace: key.Namespace, + } + found := &v1.Pod{} + err := r.Get(ctx, name, found) + return found, err +} + +func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context, key RosterEntryKey, job *klyshkov1alpha1.TupleGenerationJob, task *klyshkov1alpha1.TupleGenerationTask) (*v1.Pod, error) { + logger := log.FromContext(ctx).WithValues("Task.Key", key) + name := types.NamespacedName{ + Name: key.Name + "-provisioner", + Namespace: key.Namespace, + } + found, err := r.getProvisionerPod(ctx, key) + if err == nil { + logger.Info("provisioner pod already exists") + return found, nil + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name.Name, + Namespace: name.Namespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "generator", + Image: "carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT", + Env: []v1.EnvVar{ + { + Name: "KII_JOB_ID", + Value: job.Spec.ID, + }, + { + Name: "KII_TUPLE_TYPE", + Value: job.Spec.Type, + }, + { + Name: "KII_TUPLE_FILE", + Value: "/kii/tuples", + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kii", + MountPath: "/kii", + }, + }, + }}, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "kii", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: key.Name + "-" + strconv.Itoa(int(key.PlayerID)), // TODO Error-prone + }, + }, + }, + }, + }, + } + logger.Info("creating provisioner pod", "Pod", pod) + err = ctrl.SetControllerReference(task, pod, r.Scheme) + if err != nil { + logger.Error(err, "setting owner reference failed") + return nil, err + } + err = r.Create(ctx, pod) + if err != nil { + logger.Error(err, "provisioner pod creation failed") + return nil, err + } + return pod, nil +} + +func (r *TupleGenerationTaskReconciler) getGeneratorPod(ctx context.Context, task *klyshkov1alpha1.TupleGenerationTask) (*v1.Pod, error) { found := &v1.Pod{} - err := r.Get(ctx, key.NamespacedName, found) + err := r.Get(ctx, types.NamespacedName{ + Name: task.Name, + Namespace: task.Namespace, + }, found) + return found, err +} + +func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, key RosterEntryKey, job *klyshkov1alpha1.TupleGenerationJob, task *klyshkov1alpha1.TupleGenerationTask) (*v1.Pod, error) { + logger := log.FromContext(ctx).WithValues("Task.Key", key) + found, err := r.getGeneratorPod(ctx, task) if err == nil { logger.Info("pod already exists") - return nil + return found, nil } pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: task.Name, Namespace: task.Namespace, + //Finalizers: []string{ + // "klyshko.carbnyestack.io/provision-tuples", + //}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ @@ -217,13 +462,14 @@ func (r *TupleGenerationTaskReconciler) createCRGPod(ctx context.Context, key Ro }, }, }, - RestartPolicy: v1.RestartPolicyNever, - ShareProcessNamespace: pointer.Bool(true), + RestartPolicy: v1.RestartPolicyNever, Volumes: []v1.Volume{ { Name: "kii", VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: task.Name, + }, }, }, { @@ -261,12 +507,12 @@ func (r *TupleGenerationTaskReconciler) createCRGPod(ctx context.Context, key Ro err = ctrl.SetControllerReference(task, pod, r.Scheme) if err != nil { logger.Error(err, "setting owner reference failed") - return err + return nil, err } err = r.Create(ctx, pod) if err != nil { logger.Error(err, "pod creation failed") - return err + return nil, err } - return nil + return pod, nil } diff --git a/klyshko-operator/deploy.sh b/klyshko-operator/deploy.sh index f0f0e4b..8f74a56 100755 --- a/klyshko-operator/deploy.sh +++ b/klyshko-operator/deploy.sh @@ -53,6 +53,8 @@ do make deploy IMG="carbynestack/klyshko-operator:v0.0.1" if [ "$c" == "apollo" ]; then + JOB_ID=$(uuidgen) + sed -e "s/JOB_ID/${JOB_ID}/" config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template > config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml kubectl apply -f config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml fi done \ No newline at end of file diff --git a/klyshko-operator/go.mod b/klyshko-operator/go.mod index e1626eb..932b985 100644 --- a/klyshko-operator/go.mod +++ b/klyshko-operator/go.mod @@ -3,6 +3,7 @@ module github.com/carbynestack/klyshko go 1.16 require ( + github.com/google/uuid v1.1.2 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.13.0 go.etcd.io/etcd/api/v3 v3.5.2 @@ -10,6 +11,6 @@ require ( k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect sigs.k8s.io/controller-runtime v0.9.2 ) diff --git a/klyshko-provisioner/Dockerfile b/klyshko-provisioner/Dockerfile new file mode 100644 index 0000000..af09a4a --- /dev/null +++ b/klyshko-provisioner/Dockerfile @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +FROM adoptopenjdk/openjdk8:jdk8u292-b10-debian + +ARG CLI_VERSION=0.1-SNAPSHOT-1576571202-7-cf3db5b + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gettext-base \ + && rm -rf /var/lib/apt/lists/* + +# Download Carbyne Stack CLI Jar at specified version +# RUN curl -o cs.jar -L https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-$CLI_VERSION-jar-with-dependencies.jar +COPY cs.jar . + +# Copy resources +COPY kii-provisioner-run.sh . + +CMD ["/bin/bash", "-c", "./kii-provisioner-run.sh"] \ No newline at end of file diff --git a/klyshko-provisioner/deploy.sh b/klyshko-provisioner/deploy.sh new file mode 100755 index 0000000..305b4f3 --- /dev/null +++ b/klyshko-provisioner/deploy.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +declare -a CLUSTERS=("starbuck" "apollo") + +echo -e "${GREEN}Building code and image${NC}" +docker build -t carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT . -f Dockerfile + +echo -e "${GREEN}Loading docker images into cluster registries${NC}" +for c in "${CLUSTERS[@]}" +do + echo -e "${GREEN}Loading docker image into $c${NC}" + kind load docker-image carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT --name "$c" +done \ No newline at end of file diff --git a/klyshko-provisioner/kii-provisioner-run.sh b/klyshko-provisioner/kii-provisioner-run.sh new file mode 100755 index 0000000..96c6cf4 --- /dev/null +++ b/klyshko-provisioner/kii-provisioner-run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Fail, if any command fails +set -e + +declare -A typeByType=( + ["bit_gfp"]="BIT_GFP" + ["bit_gf2n"]="BIT_GF2N" + ["inputmask_gfp"]="INPUT_MASK_GFP" + ["inputmask_gf2n"]="INPUT_MASK_GF2N" + ["inversetuple_gfp"]="INVERSE_TUPLE_GFP" + ["inversetuple_gf2n"]="INVERSE_TUPLE_GF2N" + ["squaretuple_gfp"]="SQUARE_TUPLE_GFP" + ["squaretuple_gf2n"]="SQUARE_TUPLE_GF2N" + ["multiplicationtriple_gfp"]="MULTIPLICATION_TRIPLE_GFP" + ["multiplicationtriple_gf2n"]="MULTIPLICATION_TRIPLE_GF2N" +) + +mkdir -p ~/.cs +cat < cs-config +{ + "prime" : 0, + "r" : 0, + "noSslValidation" : true, + "trustedCertificates" : [ ], + "providers" : [ { + "amphoraServiceUrl" : "http://ignore", + "castorServiceUrl" : "http://cs-castor:10100", + "ephemeralServiceUrl" : "http://ignore", + "id" : 1, + "baseUrl" : "http://ignore" + }, { + "amphoraServiceUrl" : "http://ignore", + "castorServiceUrl" : "http://ignore", + "ephemeralServiceUrl" : "http://ignore", + "id" : 2, + "baseUrl" : "http://ignore" + } ], + "rinv" : 0 +} +EOF + +java -jar cs.jar --config-file cs-config castor upload-tuple -f "${KII_TUPLE_FILE}" -t "${typeByType[${KII_TUPLE_TYPE}]}" -i "${KII_JOB_ID}" 1 From 7fad059781aa5d1c858fd46ab7f4dbe625ed44d7 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 8 Apr 2022 15:31:50 +0200 Subject: [PATCH 04/69] Rename task states to avoid name clashes Signed-off-by: Sven Trieflinger --- .../api/v1alpha1/tuplegenerationtask_types.go | 12 +++++----- .../tuplegenerationtask_controller.go | 22 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go index 5715287..6d5cbcc 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go @@ -15,16 +15,16 @@ import ( type TupleGenerationTaskState string const ( - Launching TupleGenerationTaskState = "Launching" - Generating = "Generating" - Provisioning = "Provisioning" - Completed = "Completed" - Failed = "Failed" + TaskLaunching TupleGenerationTaskState = "Launching" + TaskGenerating = "Generating" + TaskProvisioning = "Provisioning" + TaskCompleted = "Completed" + TaskFailed = "Failed" ) func (s TupleGenerationTaskState) IsValid() bool { switch s { - case Launching, Generating, Provisioning, Completed, Failed: + case TaskLaunching, TaskGenerating, TaskProvisioning, TaskCompleted, TaskFailed: return true default: return false diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index a361190..1872037 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -144,7 +144,7 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, err } if resp.Count == 0 { - status, err := json.Marshal(&klyshkov1alpha1.TupleGenerationTaskStatus{State: klyshkov1alpha1.Launching}) + status, err := json.Marshal(&klyshkov1alpha1.TupleGenerationTaskStatus{State: klyshkov1alpha1.TaskLaunching}) _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), string(status)) if err != nil { logger.Error(err, "failed to create roster entry") @@ -179,7 +179,7 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. return ctrl.Result{}, err } - if status.State == klyshkov1alpha1.Launching { + if status.State == klyshkov1alpha1.TaskLaunching { // Create persistent volume claim used to store generated tuples, if not existing err = r.createPVC(ctx, *taskKey) if err != nil { @@ -193,10 +193,10 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. } return ctrl.Result{ Requeue: true, - }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Generating) + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskGenerating) } - if status.State == klyshkov1alpha1.Generating { + if status.State == klyshkov1alpha1.TaskGenerating { genPod, err := r.getGeneratorPod(ctx, task) if err != nil { return ctrl.Result{}, err @@ -210,28 +210,28 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. } return ctrl.Result{ Requeue: true, - }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Provisioning) + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskProvisioning) case v1.PodFailed: return ctrl.Result{ Requeue: true, - }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Failed) + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskFailed) } } - if status.State == klyshkov1alpha1.Provisioning { + if status.State == klyshkov1alpha1.TaskProvisioning { provPod, err := r.getProvisionerPod(ctx, *taskKey) if err != nil { return ctrl.Result{}, err } switch provPod.Status.Phase { case v1.PodSucceeded: - // Provisioning successful, activate tuples TODO Move to Job Controller who knows when all parties successfully provisioned + // TaskProvisioning successful, activate tuples TODO Move to Job Controller who knows when all parties successfully provisioned tupleChunkId, err := uuid.Parse(job.Spec.ID) if err != nil { logger.Error(err, "invalid job id encountered") return ctrl.Result{ Requeue: true, - }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Failed) + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskFailed) } err = activateTupleChunk(ctx, tupleChunkId) if err != nil { @@ -239,11 +239,11 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. } return ctrl.Result{ Requeue: true, - }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Completed) + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskCompleted) case v1.PodFailed: return ctrl.Result{ Requeue: true, - }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.Failed) + }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskFailed) } } From 73b2b96f6fa847a9c33f6beb2173ef66af2e8c96 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 8 Apr 2022 16:55:29 +0200 Subject: [PATCH 05/69] Track job state based on task states Signed-off-by: Sven Trieflinger --- .../api/v1alpha1/tuplegenerationjob_types.go | 27 +++++++++ ...o.carbnyestack.io_tuplegenerationjobs.yaml | 8 +++ .../klyshko_v1alpha1_tuplegenerationjob.yaml | 2 +- .../tuplegenerationjob_controller.go | 57 +++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go index 1bc9745..c72e85a 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -11,6 +11,31 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type TupleGenerationJobState string + +const ( + // JobPending means that not all tasks of the job have been spawned yet. + JobPending TupleGenerationJobState = "Pending" + + // JobRunning means all tasks for the job have been spawned but have not terminated yet. + JobRunning = "Running" + + // JobCompleted means all tasks have completed successfully. + JobCompleted = "Completed" + + // JobFailed means that all tasks for the job have terminated but at least on failed. + JobFailed = "Failed" +) + +func (s TupleGenerationJobState) IsValid() bool { + switch s { + case JobPending, JobRunning, JobCompleted, JobFailed: + return true + default: + return false + } +} + type TupleGenerationJobSpec struct { ID string `json:"id"` @@ -23,6 +48,7 @@ type TupleGenerationJobSpec struct { } type TupleGenerationJobStatus struct { + State TupleGenerationJobState `json:"state"` } //+kubebuilder:object:root=true @@ -30,6 +56,7 @@ type TupleGenerationJobStatus struct { //+kubebuilder:subresource:status //+kubebuilder:printcolumn:name="Tuple Type",type=string,JSONPath=`.spec.type` //+kubebuilder:printcolumn:name="Tuple Count",type=string,JSONPath=`.spec.count` +//+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state` //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" type TupleGenerationJob struct { diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml index ec499e2..4eaba67 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -26,6 +26,9 @@ spec: - jsonPath: .spec.count name: Tuple Count type: string + - jsonPath: .status.state + name: Status + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -64,6 +67,11 @@ spec: - type type: object status: + properties: + state: + type: string + required: + - state type: object type: object served: true diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index 8d71a7a..120b4b4 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -3,6 +3,6 @@ kind: TupleGenerationJob metadata: name: tuplegenerationjob-sample spec: - id: fa992993-080a-4328-8416-3ec089adfdfe + id: 2351d9aa-352a-4730-babc-538095744538 type: multiplicationtriple_gfp count: 100000 diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index f261adb..8687e04 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -295,6 +295,63 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R logger.Info("local task exists already") } + // Update job status based on owned task statuses + tasks := &klyshkov1alpha1.TupleGenerationTaskList{} + err = r.List(ctx, tasks) // TODO Use ListOption filter based on owner reference, if possible + if err != nil { + return ctrl.Result{}, err + } + + // Collecting tasks owned by job; TODO Move this to task class + isOwnedBy := func(task klyshkov1alpha1.TupleGenerationTask) bool { + for _, or := range task.OwnerReferences { + if or.Name == job.Name { + return true + } + } + return false + } + var ownedBy []klyshkov1alpha1.TupleGenerationTask + for _, t := range tasks.Items { + if isOwnedBy(t) { + ownedBy = append(ownedBy, t) + } + } + logger.Info("Collected statues of owned tasks", "States", ownedBy) + + // Helper functions; TODO Move this to state class? + allTerminated := func(tasks []klyshkov1alpha1.TupleGenerationTask) bool { + for _, t := range tasks { + if t.Status.State != klyshkov1alpha1.TaskCompleted && t.Status.State != klyshkov1alpha1.TaskFailed { + return false + } + } + return true + } + anyFailed := func(tasks []klyshkov1alpha1.TupleGenerationTask) bool { + for _, t := range tasks { + if t.Status.State == klyshkov1alpha1.TaskFailed { + return true + } + } + return false + } + var state klyshkov1alpha1.TupleGenerationJobState + if len(ownedBy) < 2 { + state = klyshkov1alpha1.JobPending + } else if !allTerminated(ownedBy) { + state = klyshkov1alpha1.JobRunning + } else if anyFailed(ownedBy) { + state = klyshkov1alpha1.JobFailed + } else { + state = klyshkov1alpha1.JobCompleted + } + job.Status.State = state + err = r.Status().Update(ctx, job) + if err != nil { + return ctrl.Result{}, err + } + logger.Info("desired state reached") return ctrl.Result{}, nil } From 2f7a4c0e94dd6908a063d0c63058745fc673665f Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 8 Apr 2022 17:05:59 +0200 Subject: [PATCH 06/69] Move activation to job conroller Signed-off-by: Sven Trieflinger --- .../klyshko_v1alpha1_tuplegenerationjob.yaml | 2 +- .../controllers/tuplegenerationjob_controller.go | 13 +++++++++++++ .../controllers/tuplegenerationtask_controller.go | 13 ------------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index 120b4b4..91c7574 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -3,6 +3,6 @@ kind: TupleGenerationJob metadata: name: tuplegenerationjob-sample spec: - id: 2351d9aa-352a-4730-babc-538095744538 + id: b8caa9c3-e0a9-4567-ae15-553a71b9464e type: multiplicationtriple_gfp count: 100000 diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 8687e04..a3ad1e7 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/google/uuid" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -345,6 +346,18 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R state = klyshkov1alpha1.JobFailed } else { state = klyshkov1alpha1.JobCompleted + + // Activate tuples; TODO How to deal with failures here? + tupleChunkId, err := uuid.Parse(job.Spec.ID) + if err != nil { + logger.Error(err, "invalid job id encountered") + return ctrl.Result{}, nil + } + err = activateTupleChunk(ctx, tupleChunkId) + if err != nil { + logger.Error(err, "tuple activation failed") + return ctrl.Result{}, nil + } } job.Status.State = state err = r.Status().Update(ctx, job) diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 1872037..73be267 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -11,7 +11,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/google/uuid" clientv3 "go.etcd.io/etcd/client/v3" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -225,18 +224,6 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. } switch provPod.Status.Phase { case v1.PodSucceeded: - // TaskProvisioning successful, activate tuples TODO Move to Job Controller who knows when all parties successfully provisioned - tupleChunkId, err := uuid.Parse(job.Spec.ID) - if err != nil { - logger.Error(err, "invalid job id encountered") - return ctrl.Result{ - Requeue: true, - }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskFailed) - } - err = activateTupleChunk(ctx, tupleChunkId) - if err != nil { - return ctrl.Result{}, err // TODO Fail here? - } return ctrl.Result{ Requeue: true, }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskCompleted) From 0fdda5a494ac2e511540a758b2a9fc0932ec84c5 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 6 May 2022 17:33:48 +0200 Subject: [PATCH 07/69] Use compliant base images from carbynestack/base-images Signed-off-by: Sven Trieflinger --- klyshko-operator/Dockerfile | 5 ++--- klyshko-provisioner/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/klyshko-operator/Dockerfile b/klyshko-operator/Dockerfile index 4152680..dde286f 100644 --- a/klyshko-operator/Dockerfile +++ b/klyshko-operator/Dockerfile @@ -17,9 +17,8 @@ COPY controllers/ controllers/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +# Use compliant base image from carbynestack/base-images +FROM ghcr.io/carbynestack/ubuntu:20.04-20210827-nonroot WORKDIR / COPY --from=builder /workspace/manager . USER 65532:65532 diff --git a/klyshko-provisioner/Dockerfile b/klyshko-provisioner/Dockerfile index af09a4a..d9c3aaa 100644 --- a/klyshko-provisioner/Dockerfile +++ b/klyshko-provisioner/Dockerfile @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: Apache-2.0 # -FROM adoptopenjdk/openjdk8:jdk8u292-b10-debian +FROM ghcr.io/carbynestack/openjdk:jre8-20210827 ARG CLI_VERSION=0.1-SNAPSHOT-1576571202-7-cf3db5b From d4e05b33cb690f3fd70bc5cfae41db6bb4933e83 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 6 May 2022 17:35:36 +0200 Subject: [PATCH 08/69] Fix build issues resulting from deprecation of insecure schemes by GitHub Signed-off-by: Sven Trieflinger --- klyshko-mp-spdz/Dockerfile.fake-offline | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/klyshko-mp-spdz/Dockerfile.fake-offline b/klyshko-mp-spdz/Dockerfile.fake-offline index ff97a84..b966156 100644 --- a/klyshko-mp-spdz/Dockerfile.fake-offline +++ b/klyshko-mp-spdz/Dockerfile.fake-offline @@ -32,9 +32,10 @@ ARG MP_SPDZ_REPO="https://github.com/carbynestack/MP-SPDZ" ARG MP_SPDZ_VERSION="add-prng-seed-option" # Checkout MP-SPDZ repository at required version -RUN git clone --recurse-submodules ${MP_SPDZ_REPO} mp-spdz-${MP_SPDZ_VERSION} +RUN git clone ${MP_SPDZ_REPO} mp-spdz-${MP_SPDZ_VERSION} WORKDIR mp-spdz-${MP_SPDZ_VERSION} -RUN git checkout ${MP_SPDZ_VERSION} +RUN git checkout ${MP_SPDZ_VERSION} && \ + git submodule set-url -- mpir https://github.com/wbhart/mpir # Build fake offline phase executable RUN echo "MY_CFLAGS += -DINSECURE" >> CONFIG.mine From 8a0bc5df7cfd5ece84d5cb52ba5c808ed4e14456 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 6 May 2022 17:36:49 +0200 Subject: [PATCH 09/69] Compute prng seed from md5sum of job id Signed-off-by: Sven Trieflinger --- klyshko-mp-spdz/kii-run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/klyshko-mp-spdz/kii-run.sh b/klyshko-mp-spdz/kii-run.sh index 86af279..68d369d 100755 --- a/klyshko-mp-spdz/kii-run.sh +++ b/klyshko-mp-spdz/kii-run.sh @@ -64,7 +64,8 @@ do done # Execute offline phase -cmd="./Fake-Offline.x -d 0 --prime ${prime} --prngseed ${KII_JOB_ID} ${argsByType[${KII_TUPLE_TYPE}]} ${KII_PLAYER_COUNT}" +seed=$(echo "${KII_JOB_ID}" | md5sum) +cmd="./Fake-Offline.x -d 0 --prime ${prime} --prngseed ${seed:0:16} ${argsByType[${KII_TUPLE_TYPE}]} ${KII_PLAYER_COUNT}" eval "$cmd" # Copy generated tuples to path expected by KII From 64b7580ee697168b9c06a48b794092bbda92b797 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 6 May 2022 17:50:21 +0200 Subject: [PATCH 10/69] Add pre-commit configuration Signed-off-by: Sven Trieflinger --- .pre-commit-config.yaml | 2 +- README.md | 54 +++++++++++++-------------- klyshko-mp-spdz/tuple_type_mapping.md | 24 ++++++------ 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a480e7..25f9b60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,4 +24,4 @@ repos: rev: v0.31.1 hooks: - id: markdownlint - exclude: ^3RD-PARTY-LICENSES/.*$ \ No newline at end of file + exclude: ^3RD-PARTY-LICENSES/.*|klyshko-mp-spdz/tuple_type_mapping.md$ diff --git a/README.md b/README.md index d5a7fac..19eb151 100644 --- a/README.md +++ b/README.md @@ -4,41 +4,41 @@ Klyshko is a kubernetes-native open source correlated randomness generator service for Secure Multiparty Computation in the offline/online model and part of [Carbyne Stack](https://github.com/carbynestack). -> **DISCLAIMER**: Carbyne Stack Klyshko is in *proof-of-concept* stage. The -> software is not ready for production use. It has neither been developed nor +> **DISCLAIMER**: Carbyne Stack Klyshko is in *proof-of-concept* stage. The +> software is not ready for production use. It has neither been developed nor > tested for a specific use case. ## Namesake -_Klyshko_ is one of the inventors of _spontaneous parametric down-conversion_ -(SPDC). SPDC is an important process in quantum optics, used especially as a -source of entangled photon pairs, and of single photons -(see [Wikipedia](https://en.wikipedia.org/wiki/Spontaneous_parametric_down-conversion)). -The analogy to the _Klyshko_ service is that secret shared tuples are -correlated and thus kind of "entangled" and that the microservice is the -implementation of the process that creates the tuples. +*Klyshko* is one of the inventors of *spontaneous parametric down-conversion* +(SPDC). SPDC is an important process in quantum optics, used especially as a +source of entangled photon pairs, and of single photons (see +[Wikipedia](https://en.wikipedia.org/wiki/Spontaneous_parametric_down-conversion)). +The analogy to the *Klyshko* service is that secret shared tuples are correlated +and thus kind of "entangled" and that the microservice is the implementation of +the process that creates the tuples. ## Klyshko Integration Interface -> **IMPORTANT**: This is an initial incomplete version of the KII that is -> subject to change without notice. For the time being it is very much -> influenced by the CRGs provided as part of the +> **IMPORTANT**: This is an initial incomplete version of the KII that is +> subject to change without notice. For the time being it is very much +> influenced by the CRGs provided as part of the > [MP-SPDZ](https://github.com/data61/MP-SPDZ) project. -_Klyshko_ has been designed to allow for easy integration of different +*Klyshko* has been designed to allow for easy integration of different correlated randomness generators (CRGs). Integration is done by means of -providing a docker image containing the CRG that implements the -_Klyshko Integration Interface_ (KII). +providing a docker image containing the CRG that implements the *Klyshko +Integration Interface* (KII). -> **TIP**: For an example of how to integrate the -> [MP-SPDZ](https://github.com/data61/MP-SPDZ) CRG producing _fake_ tuples see +> **TIP**: For an example of how to integrate the +> [MP-SPDZ](https://github.com/data61/MP-SPDZ) CRG producing *fake* tuples see > the [klyshko-mp-spdz](klyshko-mp-spdz) module. ### Entrypoint -The CRG docker image must contain a `kii-run.sh` script in the working -directory that performs the tuple generation process. The script must terminate -with a non-zero exit code in case the tuples can not be generated. +The CRG docker image must contain a `kii-run.sh` script in the working directory +that performs the tuple generation process. The script must terminate with a +non-zero exit code in case the tuples can not be generated. ### Environment Variables @@ -47,16 +47,16 @@ the tuple generation and provisioning process. #### Input -- `KII_JOB_ID`: The Type 4 UUID used as a job identifier. This is the same -among all VCPs in the VC. -- `KII_TUPLES_PER_JOB`: The number of tuples to be generated. The CRG should -try to match the requested number but is not required to do so. +- `KII_JOB_ID`: The Type 4 UUID used as a job identifier. This is the same among + all VCPs in the VC. +- `KII_TUPLES_PER_JOB`: The number of tuples to be generated. The CRG should try + to match the requested number but is not required to do so. - `KII_PLAYER_NUMBER`: The 0-based number of the local VCP. - `KII_PLAYER_COUNT`: The overall number of VCPs in the VC. -- `KII_TUPLE_TYPE`: The tuple type to generate. Must be one of +- `KII_TUPLE_TYPE`: The tuple type to generate. Must be one of - `bit_gfp`, `bit_gf2n` - `inputmask_gfp`, `inputmask_gf2n` - - `inversetuple_gfp`, `inversetuple_gf2n` + - `inversetuple_gfp`, `inversetuple_gf2n` - `squaretuple_gfp`, `squaretuple_gf2n` - `multiplicationtriple_gfp`, `multiplicationtriple_gf2n` @@ -71,7 +71,7 @@ The prime to be used for generating prime field tuples is provided in the file ### MAC Key Shares -The MAC key shares for prime and binary fields are made available as files +The MAC key shares for prime and binary fields are made available as files `mac_key_share_p` and `mac_key_share_2` in folder `/etc/kii/secret-params`. ## License diff --git a/klyshko-mp-spdz/tuple_type_mapping.md b/klyshko-mp-spdz/tuple_type_mapping.md index 3d2540f..e874d1b 100644 --- a/klyshko-mp-spdz/tuple_type_mapping.md +++ b/klyshko-mp-spdz/tuple_type_mapping.md @@ -1,12 +1,12 @@ -| Type | Flag | Folder | Header Length | Verified | -|-----------------------------|-----------------|-----------------------|---------------|----------| -| bit_gfp | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | yes | -| bit_gf2n | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | yes | -| inputmask_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | -| inputmask_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | -| inversetuple_gfp | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | yes | -| inversetuple_gf2n | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | yes | -| squaretuple_gfp | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | yes | -| squaretuple_gf2n | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | yes | -| multiplicationtriple_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | -| multiplicationtriple_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | \ No newline at end of file +| Type | Flag | Folder | Header Length | Verified | +| ------------------------- | -------------- | --------------------- | ------------- | -------- | +| bit_gfp | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | yes | +| bit_gf2n | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | yes | +| inputmask_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | +| inputmask_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | +| inversetuple_gfp | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | yes | +| inversetuple_gf2n | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | yes | +| squaretuple_gfp | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | yes | +| squaretuple_gf2n | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | yes | +| multiplicationtriple_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | +| multiplicationtriple_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | From cd20989dd6ee25627ecf4dd8c46076d2c53cb641 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 13 May 2022 14:50:38 +0200 Subject: [PATCH 11/69] Add basic scheduler Signed-off-by: Sven Trieflinger --- klyshko-mp-spdz/kii-run.sh | 40 ++--- klyshko-mp-spdz/tuple_type_mapping.md | 24 +-- klyshko-operator/PROJECT | 9 + .../api/v1alpha1/tuplegenerationjob_types.go | 8 +- .../tuplegenerationscheduler_types.go | 48 +++++ .../api/v1alpha1/zz_generated.deepcopy.go | 89 ++++++++++ ...o.carbnyestack.io_tuplegenerationjobs.yaml | 11 +- ...nyestack.io_tuplegenerationschedulers.yaml | 60 +++++++ .../config/crd/kustomization.yaml | 3 + ...njection_in_tuplegenerationschedulers.yaml | 7 + .../webhook_in_tuplegenerationschedulers.yaml | 16 ++ klyshko-operator/config/rbac/role.yaml | 26 +++ .../tuplegenerationscheduler_editor_role.yaml | 24 +++ .../tuplegenerationscheduler_viewer_role.yaml | 20 +++ .../klyshko_v1alpha1_tuplegenerationjob.yaml | 6 +- ...hko_v1alpha1_tuplegenerationscheduler.yaml | 7 + .../config/samples/kustomization.yaml | 1 + klyshko-operator/controllers/castor.go | 44 +++++ klyshko-operator/controllers/suite_test.go | 3 + .../tuplegenerationjob_controller.go | 2 +- .../tuplegenerationscheduler_controller.go | 168 ++++++++++++++++++ klyshko-operator/deploy.sh | 4 +- klyshko-operator/main.go | 8 + klyshko-provisioner/kii-provisioner-run.sh | 15 +- 24 files changed, 587 insertions(+), 56 deletions(-) create mode 100644 klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go create mode 100644 klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml create mode 100644 klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationschedulers.yaml create mode 100644 klyshko-operator/config/crd/patches/webhook_in_tuplegenerationschedulers.yaml create mode 100644 klyshko-operator/config/rbac/tuplegenerationscheduler_editor_role.yaml create mode 100644 klyshko-operator/config/rbac/tuplegenerationscheduler_viewer_role.yaml create mode 100644 klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml create mode 100644 klyshko-operator/controllers/tuplegenerationscheduler_controller.go diff --git a/klyshko-mp-spdz/kii-run.sh b/klyshko-mp-spdz/kii-run.sh index 68d369d..b7472e9 100755 --- a/klyshko-mp-spdz/kii-run.sh +++ b/klyshko-mp-spdz/kii-run.sh @@ -13,28 +13,28 @@ set -e n=${KII_TUPLES_PER_JOB} pn=${KII_PLAYER_NUMBER} declare -A argsByType=( - ["bit_gfp"]="--nbits 0,${n}" - ["bit_gf2n"]="--nbits ${n},0" - ["inputmask_gfp"]="--ntriples 0,${n}" - ["inputmask_gf2n"]="--ntriples ${n},0" - ["inversetuple_gfp"]="--ninverses ${n}" - ["inversetuple_gf2n"]="--ninverses ${n}" - ["squaretuple_gfp"]="--nsquares 0,${n}" - ["squaretuple_gf2n"]="--nsquares ${n},0" - ["multiplicationtriple_gfp"]="--ntriples 0,${n}" - ["multiplicationtriple_gf2n"]="--ntriples ${n},0" + ["BIT_GFP"]="--nbits 0,${n}" + ["BIT_GF2N"]="--nbits ${n},0" + ["INPUT_MASK_GFP"]="--ntriples 0,${n}" + ["INPUT_MASK_GF2N"]="--ntriples ${n},0" + ["INVERSE_TUPLE_GFP"]="--ninverses ${n}" + ["INVERSE_TUPLE_GF2N"]="--ninverses ${n}" + ["SQUARE_TUPLE_GFP"]="--nsquares 0,${n}" + ["SQUARE_TUPLE_GF2N"]="--nsquares ${n},0" + ["MULTIPLICATION_TRIPLE_GFP"]="--ntriples 0,${n}" + ["MULTIPLICATION_TRIPLE_GF2N"]="--ntriples ${n},0" ) declare -A folderByType=( - ["bit_gfp"]="2-p-128/Bits-p-P${pn}" - ["bit_gf2n"]="2-2-128/Bits-2-P${pn}" - ["inputmask_gfp"]="2-p-128/Triples-p-P${pn}" - ["inputmask_gf2n"]="2-2-128/Triples-2-P${pn}" - ["inversetuple_gfp"]="2-p-128/Inverses-p-P${pn}" - ["inversetuple_gf2n"]="2-2-128/Inverses-2-P${pn}" - ["squaretuple_gfp"]="2-p-128/Squares-p-P${pn}" - ["squaretuple_gf2n"]="2-2-128/Squares-2-P${pn}" - ["multiplicationtriple_gfp"]="2-p-128/Triples-p-P${pn}" - ["multiplicationtriple_gf2n"]="2-2-128/Triples-2-P${pn}" + ["BIT_GFP"]="2-p-128/Bits-p-P${pn}" + ["BIT_GF2N"]="2-2-128/Bits-2-P${pn}" + ["INPUT_MASK_GFP"]="2-p-128/Triples-p-P${pn}" + ["INPUT_MASK_GF2N"]="2-2-128/Triples-2-P${pn}" + ["INVERSE_TUPLE_GFP"]="2-p-128/Inverses-p-P${pn}" + ["INVERSE_TUPLE_GF2N"]="2-2-128/Inverses-2-P${pn}" + ["SQUARE_TUPLE_GFP"]="2-p-128/Squares-p-P${pn}" + ["SQUARE_TUPLE_GF2N"]="2-2-128/Squares-2-P${pn}" + ["MULTIPLICATION_TRIPLE_GFP"]="2-p-128/Triples-p-P${pn}" + ["MULTIPLICATION_TRIPLE_GF2N"]="2-2-128/Triples-2-P${pn}" ) # Provide required parameters in MP-SPDZ "Player-Data" folder diff --git a/klyshko-mp-spdz/tuple_type_mapping.md b/klyshko-mp-spdz/tuple_type_mapping.md index e874d1b..8b3590e 100644 --- a/klyshko-mp-spdz/tuple_type_mapping.md +++ b/klyshko-mp-spdz/tuple_type_mapping.md @@ -1,12 +1,12 @@ -| Type | Flag | Folder | Header Length | Verified | -| ------------------------- | -------------- | --------------------- | ------------- | -------- | -| bit_gfp | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | yes | -| bit_gf2n | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | yes | -| inputmask_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | -| inputmask_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | -| inversetuple_gfp | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | yes | -| inversetuple_gf2n | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | yes | -| squaretuple_gfp | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | yes | -| squaretuple_gf2n | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | yes | -| multiplicationtriple_gfp | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | -| multiplicationtriple_gf2n | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | +| Type | Flag | Folder | Header Length | Verified | +| -------------------------- | -------------- | --------------------- | ------------- | -------- | +| BIT_GFP | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | yes | +| BIT_GF2N | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | yes | +| INPUT_MASK_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | +| INPUT_MASK_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | +| INVERSE_TUPLE_GFP | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | yes | +| INVERSE_TUPLE_GF2N | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | yes | +| SQUARE_TUPLE_GFP | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | yes | +| SQUARE_TUPLE_GF2N | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | yes | +| MULTIPLICATION_TRIPLE_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | +| MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | diff --git a/klyshko-operator/PROJECT b/klyshko-operator/PROJECT index fdf0129..5faaab7 100644 --- a/klyshko-operator/PROJECT +++ b/klyshko-operator/PROJECT @@ -34,4 +34,13 @@ resources: kind: TupleGenerationTask path: github.com/carbynestack/klyshko/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: carbnyestack.io + group: klyshko + kind: TupleGenerationScheduler + path: github.com/carbynestack/klyshko/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go index c72e85a..484d40c 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -36,15 +36,19 @@ func (s TupleGenerationJobState) IsValid() bool { } } +func (s TupleGenerationJobState) IsDone() bool { + return s == JobCompleted || s == JobFailed +} + type TupleGenerationJobSpec struct { ID string `json:"id"` - // +kubebuilder:validation:Enum=multiplicationtriple_gfp + // +kubebuilder:validation:Enum=BIT_GFP;BIT_GF2N;INPUT_MASK_GFP;INPUT_MASK_GF2N;INVERSE_TUPLE_GFP;INVERSE_TUPLE_GF2N;SQUARE_TUPLE_GFP;SQUARE_TUPLE_GF2N;MULTIPLICATION_TRIPLE_GFP;MULTIPLICATION_TRIPLE_GF2N Type string `json:"type"` //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:ExclusiveMinimum=true - Count int32 `json:"count"` + Count int `json:"count"` } type TupleGenerationJobStatus struct { diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go new file mode 100644 index 0000000..a2e2e8f --- /dev/null +++ b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go @@ -0,0 +1,48 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type TupleGenerationSchedulerSpec struct { + //+kubebuilder:validation:Minimum=0 + //+kubebuilder:validation:ExclusiveMinimum=false + Concurrency int `json:"concurrency"` + + //+kubebuilder:validation:Minimum=0 + //+kubebuilder:validation:ExclusiveMinimum=true + Threshold int `json:"threshold"` +} + +type TupleGenerationSchedulerStatus struct { +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +type TupleGenerationScheduler struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TupleGenerationSchedulerSpec `json:"spec,omitempty"` + Status TupleGenerationSchedulerStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +type TupleGenerationSchedulerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TupleGenerationScheduler `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TupleGenerationScheduler{}, &TupleGenerationSchedulerList{}) +} diff --git a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go index 4b51bd6..71149bf 100644 --- a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -113,6 +113,95 @@ func (in *TupleGenerationJobStatus) DeepCopy() *TupleGenerationJobStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationScheduler) DeepCopyInto(out *TupleGenerationScheduler) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationScheduler. +func (in *TupleGenerationScheduler) DeepCopy() *TupleGenerationScheduler { + if in == nil { + return nil + } + out := new(TupleGenerationScheduler) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TupleGenerationScheduler) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationSchedulerList) DeepCopyInto(out *TupleGenerationSchedulerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TupleGenerationScheduler, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationSchedulerList. +func (in *TupleGenerationSchedulerList) DeepCopy() *TupleGenerationSchedulerList { + if in == nil { + return nil + } + out := new(TupleGenerationSchedulerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TupleGenerationSchedulerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationSchedulerSpec) DeepCopyInto(out *TupleGenerationSchedulerSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationSchedulerSpec. +func (in *TupleGenerationSchedulerSpec) DeepCopy() *TupleGenerationSchedulerSpec { + if in == nil { + return nil + } + out := new(TupleGenerationSchedulerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TupleGenerationSchedulerStatus) DeepCopyInto(out *TupleGenerationSchedulerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationSchedulerStatus. +func (in *TupleGenerationSchedulerStatus) DeepCopy() *TupleGenerationSchedulerStatus { + if in == nil { + return nil + } + out := new(TupleGenerationSchedulerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TupleGenerationTask) DeepCopyInto(out *TupleGenerationTask) { *out = *in diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml index 4eaba67..9b62255 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -59,7 +59,16 @@ spec: type: string type: enum: - - multiplicationtriple_gfp + - BIT_GFP + - BIT_GF2N + - INPUT_MASK_GFP + - INPUT_MASK_GF2N + - INVERSE_TUPLE_GFP + - INVERSE_TUPLE_GF2N + - SQUARE_TUPLE_GFP + - SQUARE_TUPLE_GF2N + - MULTIPLICATION_TRIPLE_GFP + - MULTIPLICATION_TRIPLE_GF2N type: string required: - count diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml new file mode 100644 index 0000000..44ce6f6 --- /dev/null +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml @@ -0,0 +1,60 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null + name: tuplegenerationschedulers.klyshko.carbnyestack.io +spec: + group: klyshko.carbnyestack.io + names: + kind: TupleGenerationScheduler + listKind: TupleGenerationSchedulerList + plural: tuplegenerationschedulers + singular: tuplegenerationscheduler + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + concurrency: + minimum: 0 + type: integer + threshold: + exclusiveMinimum: true + minimum: 0 + type: integer + required: + - concurrency + - threshold + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/klyshko-operator/config/crd/kustomization.yaml b/klyshko-operator/config/crd/kustomization.yaml index f35a6da..c11c17b 100644 --- a/klyshko-operator/config/crd/kustomization.yaml +++ b/klyshko-operator/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml - bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml +- bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -11,12 +12,14 @@ patchesStrategicMerge: # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_tuplegenerationjobs.yaml #- patches/webhook_in_tuplegenerationtasks.yaml +#- patches/webhook_in_tuplegenerationschedulers.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_tuplegenerationjobs.yaml #- patches/cainjection_in_tuplegenerationtasks.yaml +#- patches/cainjection_in_tuplegenerationschedulers.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationschedulers.yaml b/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationschedulers.yaml new file mode 100644 index 0000000..1de411a --- /dev/null +++ b/klyshko-operator/config/crd/patches/cainjection_in_tuplegenerationschedulers.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: tuplegenerationschedulers.klyshko.carbnyestack.io diff --git a/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationschedulers.yaml b/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationschedulers.yaml new file mode 100644 index 0000000..8f80cb0 --- /dev/null +++ b/klyshko-operator/config/crd/patches/webhook_in_tuplegenerationschedulers.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: tuplegenerationschedulers.klyshko.carbnyestack.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/klyshko-operator/config/rbac/role.yaml b/klyshko-operator/config/rbac/role.yaml index 8c44b9b..3384634 100644 --- a/klyshko-operator/config/rbac/role.yaml +++ b/klyshko-operator/config/rbac/role.yaml @@ -64,6 +64,32 @@ rules: - get - patch - update +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers/finalizers + verbs: + - update +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers/status + verbs: + - get + - patch + - update - apiGroups: - klyshko.carbnyestack.io resources: diff --git a/klyshko-operator/config/rbac/tuplegenerationscheduler_editor_role.yaml b/klyshko-operator/config/rbac/tuplegenerationscheduler_editor_role.yaml new file mode 100644 index 0000000..23ebaa5 --- /dev/null +++ b/klyshko-operator/config/rbac/tuplegenerationscheduler_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit tuplegenerationschedulers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tuplegenerationscheduler-editor-role +rules: +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers/status + verbs: + - get diff --git a/klyshko-operator/config/rbac/tuplegenerationscheduler_viewer_role.yaml b/klyshko-operator/config/rbac/tuplegenerationscheduler_viewer_role.yaml new file mode 100644 index 0000000..746455f --- /dev/null +++ b/klyshko-operator/config/rbac/tuplegenerationscheduler_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view tuplegenerationschedulers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tuplegenerationscheduler-viewer-role +rules: +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers + verbs: + - get + - list + - watch +- apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers/status + verbs: + - get diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index 91c7574..53c0362 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -3,6 +3,6 @@ kind: TupleGenerationJob metadata: name: tuplegenerationjob-sample spec: - id: b8caa9c3-e0a9-4567-ae15-553a71b9464e - type: multiplicationtriple_gfp - count: 100000 + id: 9fc7f5b7-09d9-4c62-97c7-2d7eeddfb858 + type: MULTIPLICATION_TRIPLE_GFP + count: 10000 diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml new file mode 100644 index 0000000..f448ea8 --- /dev/null +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml @@ -0,0 +1,7 @@ +apiVersion: klyshko.carbnyestack.io/v1alpha1 +kind: TupleGenerationScheduler +metadata: + name: tuplegenerationscheduler-sample +spec: + concurrency: 3 + threshold: 400000 diff --git a/klyshko-operator/config/samples/kustomization.yaml b/klyshko-operator/config/samples/kustomization.yaml index 0b61aa2..f572074 100644 --- a/klyshko-operator/config/samples/kustomization.yaml +++ b/klyshko-operator/config/samples/kustomization.yaml @@ -2,4 +2,5 @@ resources: - klyshko_v1alpha1_tuplegenerationjob.yaml - klyshko_v1alpha1_tuplegenerationtask.yaml +- klyshko_v1alpha1_tuplegenerationscheduler.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/klyshko-operator/controllers/castor.go b/klyshko-operator/controllers/castor.go index 3bc3fd4..da1e1c0 100644 --- a/klyshko-operator/controllers/castor.go +++ b/klyshko-operator/controllers/castor.go @@ -9,6 +9,7 @@ package controllers import ( "context" + "encoding/json" "fmt" "github.com/google/uuid" "io/ioutil" @@ -34,3 +35,46 @@ func activateTupleChunk(ctx context.Context, chunkId uuid.UUID) error { logger.Info("response from castor", "Status", resp.Status, "Body", body) // TODO Body is Base64 encoded :-( return err } + +type TupleMetrics struct { + Available int `json:"available"` + ConsumptionRate int `json:"consumptionRate"` + TupleType string `json:"type"` +} + +type Telemetry struct { + TupleMetrics []TupleMetrics `json:"metrics"` +} + +func getTelemetry(ctx context.Context) (Telemetry, error) { + logger := log.FromContext(ctx) + client := &http.Client{} + + // Building the request + req, err := http.NewRequest("GET", "http://cs-castor.default.svc.cluster.local:10100/intra-vcp/telemetry", nil) // TODO Make servername configurable / use discovery + if err != nil { + logger.Error(err, "failed to build request for castor telemetry data") + return Telemetry{}, err + } + req.Header.Add("Accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + // Doing the request + resp, err := client.Do(req) + if err != nil { + logger.Error(err, "failed to fetch castor telemetry data") + return Telemetry{}, err + } + + // Read and parse response + defer resp.Body.Close() + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger.Error(err, "failed to read response body") + return Telemetry{}, err + } + var response Telemetry + json.Unmarshal(bodyBytes, &response) + + return response, nil +} diff --git a/klyshko-operator/controllers/suite_test.go b/klyshko-operator/controllers/suite_test.go index 9f2b3a8..212644f 100644 --- a/klyshko-operator/controllers/suite_test.go +++ b/klyshko-operator/controllers/suite_test.go @@ -62,6 +62,9 @@ var _ = BeforeSuite(func() { err = klyshkov1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = klyshkov1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index a3ad1e7..eaa88e0 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -347,7 +347,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R } else { state = klyshkov1alpha1.JobCompleted - // Activate tuples; TODO How to deal with failures here? + // Activate tuples; TODO How to deal with failures here? Introduce JobActivating state, how to sync between VCPs? Is activation still used in new Castor implementation? tupleChunkId, err := uuid.Parse(job.Spec.ID) if err != nil { logger.Error(err, "invalid job id encountered") diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go new file mode 100644 index 0000000..3cad8e6 --- /dev/null +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -0,0 +1,168 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "github.com/google/uuid" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sort" + "time" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" +) + +type TupleGenerationSchedulerReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationschedulers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationschedulers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationschedulers/finalizers,verbs=update + +func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + // Fetch scheduler resource + scheduler := &klyshkov1alpha1.TupleGenerationScheduler{} + err := r.Get(ctx, req.NamespacedName, scheduler) + if err != nil { + if apierrors.IsNotFound(err) { + // Scheduler resource not available -> has been deleted + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + logger.Error(err, "failed to read scheduler resource") + return ctrl.Result{}, err + } + + activeJobs, err := r.getMatchingJobs(ctx, func(job klyshkov1alpha1.TupleGenerationJob) bool { + return !job.Status.State.IsDone() + }) + if err != nil { + logger.Error(err, "failed to fetch active jobs") + return ctrl.Result{}, err + } + + // Stop if already at maximum concurrency level + activeJobCount := len(activeJobs) + if scheduler.Spec.Concurrency <= activeJobCount { + logger.Info("at maximum concurrency level", "Jobs.Active", activeJobCount, "Scheduler.Concurrency", scheduler.Spec.Concurrency) + return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + } + + // Least available first strategy for selecting tuples to create a job for + // 1. Compute available and in generation number of tuples per type + // 2. Filter out all above threshold + // 3. Sort ascending wrt to sum from step 1 + telemetry, err := getTelemetry(ctx) + if err != nil { + return ctrl.Result{RequeueAfter: 60 * time.Second}, err + } + logger.Info("tuple telemetry data fetched", "Metrics.Available", telemetry.TupleMetrics) + for _, j := range activeJobs { + for idx := range telemetry.TupleMetrics { + if j.Spec.Type == telemetry.TupleMetrics[idx].TupleType { + telemetry.TupleMetrics[idx].Available = telemetry.TupleMetrics[idx].Available + int(j.Spec.Count) + break + } + } + } + logger.Info("with in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) + belowThreshold := make([]TupleMetrics, 0) + for _, m := range telemetry.TupleMetrics { + if m.Available < scheduler.Spec.Threshold { + belowThreshold = append(belowThreshold, m) + } + } + logger.Info("filtered for eligible types", "Metrics.Eligible", belowThreshold) + if len(belowThreshold) == 0 { + logger.Info("above threshold for all tuple types - do nothing") + return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + } + sort.Slice(belowThreshold, func(i, j int) bool { + return belowThreshold[i].Available < belowThreshold[j].Available + }) + logger.Info("sorted by priority", "Metrics.Sorted", belowThreshold) + + // Create job for first below threshold + jobId := uuid.New().String() + job := &klyshkov1alpha1.TupleGenerationJob{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "klyshko.carbnyestack.io/v1alpha1", + Kind: "TupleGenerationJob", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: scheduler.Name + "-" + jobId, + Namespace: req.Namespace, + }, + Spec: klyshkov1alpha1.TupleGenerationJobSpec{ + ID: jobId, + Type: belowThreshold[0].TupleType, + Count: 10000, + }, + } + err = ctrl.SetControllerReference(scheduler, job, r.Scheme) + if err != nil { + logger.Error(err, "could not set owner reference on job", "Job", job) + return ctrl.Result{}, err + } + err = r.Create(ctx, job) + if err != nil { + logger.Error(err, "job creation failed", "Job", job) + return ctrl.Result{}, err + } + logger.Info("job created", "Job", job) + + // Delete all finished jobs + finishedJobs, err := r.getMatchingJobs(ctx, func(job klyshkov1alpha1.TupleGenerationJob) bool { + return job.Status.State.IsDone() + }) + if err != nil { + logger.Error(err, "failed to fetch finished jobs") + return ctrl.Result{}, err + } + for _, j := range finishedJobs { + err := r.Delete(ctx, &j) + if err != nil { + logger.Error(err, "failed to delete finished job", "Job", j) + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func (r *TupleGenerationSchedulerReconciler) getMatchingJobs(ctx context.Context, pred func(klyshkov1alpha1.TupleGenerationJob) bool) ([]klyshkov1alpha1.TupleGenerationJob, error) { + allJobs := &klyshkov1alpha1.TupleGenerationJobList{} + err := r.List(ctx, allJobs) + if err != nil { + return nil, err + } + var matchingJobs []klyshkov1alpha1.TupleGenerationJob + for _, j := range allJobs.Items { + if pred(j) { + matchingJobs = append(matchingJobs, j) + } + } + return matchingJobs, nil +} + +func (r *TupleGenerationSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&klyshkov1alpha1.TupleGenerationScheduler{}). + Owns(&klyshkov1alpha1.TupleGenerationJob{}). + Complete(r) +} diff --git a/klyshko-operator/deploy.sh b/klyshko-operator/deploy.sh index 8f74a56..6b07f6b 100755 --- a/klyshko-operator/deploy.sh +++ b/klyshko-operator/deploy.sh @@ -53,8 +53,6 @@ do make deploy IMG="carbynestack/klyshko-operator:v0.0.1" if [ "$c" == "apollo" ]; then - JOB_ID=$(uuidgen) - sed -e "s/JOB_ID/${JOB_ID}/" config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template > config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml - kubectl apply -f config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml + kubectl apply -f config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml fi done \ No newline at end of file diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index 4e940f8..803c6a5 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -94,6 +94,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationTask") os.Exit(1) } + + if err = (&controllers.TupleGenerationSchedulerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationScheduler") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/klyshko-provisioner/kii-provisioner-run.sh b/klyshko-provisioner/kii-provisioner-run.sh index 96c6cf4..8a328a0 100755 --- a/klyshko-provisioner/kii-provisioner-run.sh +++ b/klyshko-provisioner/kii-provisioner-run.sh @@ -9,19 +9,6 @@ # Fail, if any command fails set -e -declare -A typeByType=( - ["bit_gfp"]="BIT_GFP" - ["bit_gf2n"]="BIT_GF2N" - ["inputmask_gfp"]="INPUT_MASK_GFP" - ["inputmask_gf2n"]="INPUT_MASK_GF2N" - ["inversetuple_gfp"]="INVERSE_TUPLE_GFP" - ["inversetuple_gf2n"]="INVERSE_TUPLE_GF2N" - ["squaretuple_gfp"]="SQUARE_TUPLE_GFP" - ["squaretuple_gf2n"]="SQUARE_TUPLE_GF2N" - ["multiplicationtriple_gfp"]="MULTIPLICATION_TRIPLE_GFP" - ["multiplicationtriple_gf2n"]="MULTIPLICATION_TRIPLE_GF2N" -) - mkdir -p ~/.cs cat < cs-config { @@ -46,4 +33,4 @@ cat < cs-config } EOF -java -jar cs.jar --config-file cs-config castor upload-tuple -f "${KII_TUPLE_FILE}" -t "${typeByType[${KII_TUPLE_TYPE}]}" -i "${KII_JOB_ID}" 1 +java -jar cs.jar --config-file cs-config castor upload-tuple -f "${KII_TUPLE_FILE}" -t "${KII_TUPLE_TYPE}" -i "${KII_JOB_ID}" 1 From b1f9eb2c380b3105dd346581390780f01d3bd3d2 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 07:59:14 +0200 Subject: [PATCH 12/69] Add SBOM creation script and 3rd party licenses Signed-off-by: Sven Trieflinger --- .gitignore | 5 +- .../cloud.google.com/go/LICENSE | 202 ++++++++++ .../github.com/Azure/go-autorest/LICENSE | 191 +++++++++ .../Azure/go-autorest/autorest/LICENSE | 191 +++++++++ .../Azure/go-autorest/autorest/adal/LICENSE | 191 +++++++++ .../Azure/go-autorest/autorest/date/LICENSE | 191 +++++++++ .../Azure/go-autorest/logger/LICENSE | 191 +++++++++ .../Azure/go-autorest/tracing/LICENSE | 191 +++++++++ .../github.com/beorn7/perks/LICENSE | 20 + .../github.com/cespare/xxhash/v2/LICENSE.txt | 22 ++ .../github.com/coreos/go-semver/LICENSE | 202 ++++++++++ .../github.com/coreos/go-semver/NOTICE | 5 + .../github.com/coreos/go-systemd/v22/LICENSE | 191 +++++++++ .../github.com/coreos/go-systemd/v22/NOTICE | 5 + .../github.com/davecgh/go-spew/LICENSE | 15 + .../github.com/evanphx/json-patch/LICENSE | 25 ++ .../github.com/form3tech-oss/jwt-go/LICENSE | 8 + .../github.com/fsnotify/fsnotify/LICENSE | 28 ++ .../github.com/go-logr/logr/LICENSE | 201 ++++++++++ .../github.com/go-logr/zapr/LICENSE | 201 ++++++++++ .../github.com/golang/groupcache/LICENSE | 191 +++++++++ .../github.com/golang/protobuf/LICENSE | 28 ++ .../github.com/google/go-cmp/LICENSE | 27 ++ .../github.com/google/gofuzz/LICENSE | 202 ++++++++++ .../github.com/google/uuid/LICENSE | 27 ++ .../github.com/googleapis/gnostic/LICENSE | 203 ++++++++++ .../github.com/hashicorp/golang-lru/LICENSE | 362 ++++++++++++++++++ .../github.com/imdario/mergo/LICENSE | 28 ++ .../github.com/json-iterator/go/LICENSE | 21 + .../golang_protobuf_extensions/LICENSE | 201 ++++++++++ .../golang_protobuf_extensions/NOTICE | 1 + .../github.com/modern-go/concurrent/LICENSE | 201 ++++++++++ .../github.com/modern-go/reflect2/LICENSE | 201 ++++++++++ .../github.com/nxadm/tail/LICENSE | 21 + .../github.com/nxadm/tail/ratelimiter/Licence | 7 + .../github.com/onsi/ginkgo/LICENSE | 20 + .../stenographer/support/go-colorable/LICENSE | 21 + .../stenographer/support/go-isatty/LICENSE | 9 + .../github.com/onsi/gomega/LICENSE | 20 + .../prometheus/client_golang/LICENSE | 201 ++++++++++ .../prometheus/client_golang/NOTICE | 23 ++ .../prometheus/client_model/LICENSE | 201 ++++++++++ .../github.com/prometheus/client_model/NOTICE | 5 + .../github.com/prometheus/common/LICENSE | 201 ++++++++++ .../github.com/prometheus/common/NOTICE | 5 + .../github.com/prometheus/procfs/LICENSE | 201 ++++++++++ .../github.com/prometheus/procfs/NOTICE | 7 + .../github.com/spf13/pflag/LICENSE | 28 ++ .../go.etcd.io/etcd/api/v3/LICENSE | 202 ++++++++++ .../go.etcd.io/etcd/client/pkg/v3/LICENSE | 202 ++++++++++ .../go.etcd.io/etcd/client/v3/LICENSE | 202 ++++++++++ .../go.uber.org/atomic/LICENSE.txt | 19 + .../go.uber.org/multierr/LICENSE.txt | 19 + .../go.uber.org/zap/LICENSE.txt | 19 + .../golang.org/x/crypto/LICENSE | 27 ++ .../golang.org/x/net/LICENSE | 27 ++ .../golang.org/x/oauth2/LICENSE | 27 ++ .../golang.org/x/sys/LICENSE | 27 ++ .../golang.org/x/term/LICENSE | 27 ++ .../golang.org/x/text/LICENSE | 27 ++ .../golang.org/x/time/LICENSE | 27 ++ .../gomodules.xyz/jsonpatch/v2/LICENSE | 202 ++++++++++ .../google.golang.org/appengine/LICENSE | 202 ++++++++++ .../google.golang.org/genproto/LICENSE | 202 ++++++++++ .../google.golang.org/grpc/LICENSE | 202 ++++++++++ .../google.golang.org/protobuf/LICENSE | 27 ++ .../gopkg.in/inf.v0/LICENSE | 28 ++ .../gopkg.in/tomb.v1/LICENSE | 29 ++ .../gopkg.in/yaml.v2/LICENSE | 201 ++++++++++ .../gopkg.in/yaml.v2/LICENSE.libyaml | 31 ++ .../gopkg.in/yaml.v2/NOTICE | 13 + .../gopkg.in/yaml.v3/LICENSE | 50 +++ .../gopkg.in/yaml.v3/NOTICE | 13 + .../3RD-PARTY-LICENSES/k8s.io/api/LICENSE | 202 ++++++++++ .../k8s.io/apiextensions-apiserver/LICENSE | 202 ++++++++++ .../k8s.io/apimachinery/LICENSE | 202 ++++++++++ .../k8s.io/client-go/LICENSE | 202 ++++++++++ .../k8s.io/component-base/LICENSE | 202 ++++++++++ .../3RD-PARTY-LICENSES/k8s.io/klog/v2/LICENSE | 191 +++++++++ .../k8s.io/kube-openapi/LICENSE | 202 ++++++++++ .../3RD-PARTY-LICENSES/k8s.io/utils/LICENSE | 202 ++++++++++ klyshko-operator/3RD-PARTY-LICENSES/sbom.json | 294 ++++++++++++++ .../sigs.k8s.io/controller-runtime/LICENSE | 201 ++++++++++ .../structured-merge-diff/v4/LICENSE | 201 ++++++++++ klyshko-operator/create-sbom.sh | 52 +++ 85 files changed, 9343 insertions(+), 1 deletion(-) create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/cloud.google.com/go/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/adal/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/date/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/logger/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/tracing/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/beorn7/perks/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/cespare/xxhash/v2/LICENSE.txt create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/davecgh/go-spew/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/logr/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/zapr/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/groupcache/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/google/go-cmp/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/google/gofuzz/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/google/uuid/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/imdario/mergo/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/json-iterator/go/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/concurrent/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/reflect2/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/gomega/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/spf13/pflag/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/api/v3/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/pkg/v3/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/v3/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/atomic/LICENSE.txt create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/multierr/LICENSE.txt create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/zap/LICENSE.txt create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/appengine/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/genproto/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/inf.v0/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE.libyaml create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/NOTICE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/api/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apiextensions-apiserver/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apimachinery/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/client-go/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/component-base/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/klog/v2/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/kube-openapi/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/k8s.io/utils/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/sbom.json create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/controller-runtime/LICENSE create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/structured-merge-diff/v4/LICENSE create mode 100755 klyshko-operator/create-sbom.sh diff --git a/.gitignore b/.gitignore index 2fa2b55..c055966 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ .idea *.ipr *.iml -*.iws \ No newline at end of file +*.iws + +# Go +/klyshko-operator/vendor/ diff --git a/klyshko-operator/3RD-PARTY-LICENSES/cloud.google.com/go/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/cloud.google.com/go/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/cloud.google.com/go/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/LICENSE new file mode 100644 index 0000000..b9d6a27 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015 Microsoft Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/LICENSE new file mode 100644 index 0000000..b9d6a27 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015 Microsoft Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/adal/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/adal/LICENSE new file mode 100644 index 0000000..b9d6a27 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/adal/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015 Microsoft Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/date/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/date/LICENSE new file mode 100644 index 0000000..b9d6a27 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/autorest/date/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015 Microsoft Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/logger/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/logger/LICENSE new file mode 100644 index 0000000..b9d6a27 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/logger/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015 Microsoft Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/tracing/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/tracing/LICENSE new file mode 100644 index 0000000..b9d6a27 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/Azure/go-autorest/tracing/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015 Microsoft Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/beorn7/perks/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/beorn7/perks/LICENSE new file mode 100644 index 0000000..339177b --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/beorn7/perks/LICENSE @@ -0,0 +1,20 @@ +Copyright (C) 2013 Blake Mizerany + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/cespare/xxhash/v2/LICENSE.txt b/klyshko-operator/3RD-PARTY-LICENSES/github.com/cespare/xxhash/v2/LICENSE.txt new file mode 100644 index 0000000..24b5306 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/cespare/xxhash/v2/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/NOTICE new file mode 100644 index 0000000..23a0ada --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-semver/NOTICE @@ -0,0 +1,5 @@ +CoreOS Project +Copyright 2018 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/LICENSE new file mode 100644 index 0000000..37ec93a --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/NOTICE new file mode 100644 index 0000000..23a0ada --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/coreos/go-systemd/v22/NOTICE @@ -0,0 +1,5 @@ +CoreOS Project +Copyright 2018 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/davecgh/go-spew/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/davecgh/go-spew/LICENSE new file mode 100644 index 0000000..bc52e96 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/davecgh/go-spew/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE new file mode 100644 index 0000000..df76d7d --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2014, Evan Phoenix +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the Evan Phoenix nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE new file mode 100644 index 0000000..df83a9c --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE @@ -0,0 +1,8 @@ +Copyright (c) 2012 Dave Grijalva + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/LICENSE new file mode 100644 index 0000000..e180c8f --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2012-2019 fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/logr/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/logr/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/logr/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/zapr/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/zapr/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/go-logr/zapr/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/groupcache/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/groupcache/LICENSE new file mode 100644 index 0000000..37ec93a --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/groupcache/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE new file mode 100644 index 0000000..0f64693 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE @@ -0,0 +1,28 @@ +Copyright 2010 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/go-cmp/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/go-cmp/LICENSE new file mode 100644 index 0000000..32017f8 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/go-cmp/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2017 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/gofuzz/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/gofuzz/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/gofuzz/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/uuid/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/uuid/LICENSE new file mode 100644 index 0000000..5dc6826 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/google/uuid/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE new file mode 100644 index 0000000..6b0b127 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/LICENSE new file mode 100644 index 0000000..be2cc4d --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/LICENSE @@ -0,0 +1,362 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/imdario/mergo/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/imdario/mergo/LICENSE new file mode 100644 index 0000000..6866802 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/imdario/mergo/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/json-iterator/go/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/json-iterator/go/LICENSE new file mode 100644 index 0000000..2cf4f5a --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/json-iterator/go/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 json-iterator + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/NOTICE new file mode 100644 index 0000000..5d8cb5b --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/matttproud/golang_protobuf_extensions/NOTICE @@ -0,0 +1 @@ +Copyright 2012 Matt T. Proud (matt.proud@gmail.com) diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/concurrent/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/concurrent/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/concurrent/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/reflect2/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/reflect2/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/modern-go/reflect2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE new file mode 100644 index 0000000..818d802 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE @@ -0,0 +1,21 @@ +# The MIT License (MIT) + +# © Copyright 2015 Hewlett Packard Enterprise Development LP +Copyright (c) 2014 ActiveState + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence b/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence new file mode 100644 index 0000000..434aab1 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence @@ -0,0 +1,7 @@ +Copyright (C) 2013 99designs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/LICENSE new file mode 100644 index 0000000..9415ee7 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2013-2014 Onsi Fakhouri + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE new file mode 100644 index 0000000..91b5cef --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE new file mode 100644 index 0000000..65dc692 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) Yasuhiro MATSUMOTO + +MIT License (Expat) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/gomega/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/gomega/LICENSE new file mode 100644 index 0000000..9415ee7 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/gomega/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2013-2014 Onsi Fakhouri + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/NOTICE new file mode 100644 index 0000000..dd878a3 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_golang/NOTICE @@ -0,0 +1,23 @@ +Prometheus instrumentation library for Go applications +Copyright 2012-2015 The Prometheus Authors + +This product includes software developed at +SoundCloud Ltd. (http://soundcloud.com/). + + +The following components are included in this product: + +perks - a fork of https://github.com/bmizerany/perks +https://github.com/beorn7/perks +Copyright 2013-2015 Blake Mizerany, Björn Rabenstein +See https://github.com/beorn7/perks/blob/master/README.md for license details. + +Go support for Protocol Buffers - Google's data interchange format +http://github.com/golang/protobuf/ +Copyright 2010 The Go Authors +See source code for license details. + +Support for streaming Protocol Buffer messages for the Go language (golang). +https://github.com/matttproud/golang_protobuf_extensions +Copyright 2013 Matt T. Proud +Licensed under the Apache License, Version 2.0 diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/NOTICE new file mode 100644 index 0000000..20110e4 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/client_model/NOTICE @@ -0,0 +1,5 @@ +Data model artifacts for Prometheus. +Copyright 2012-2015 The Prometheus Authors + +This product includes software developed at +SoundCloud Ltd. (http://soundcloud.com/). diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/NOTICE new file mode 100644 index 0000000..636a2c1 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/NOTICE @@ -0,0 +1,5 @@ +Common libraries shared by Prometheus Go components. +Copyright 2015 The Prometheus Authors + +This product includes software developed at +SoundCloud Ltd. (http://soundcloud.com/). diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/NOTICE new file mode 100644 index 0000000..53c5e9a --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/procfs/NOTICE @@ -0,0 +1,7 @@ +procfs provides functions to retrieve system, kernel and process +metrics from the pseudo-filesystem proc. + +Copyright 2014-2015 The Prometheus Authors + +This product includes software developed at +SoundCloud Ltd. (http://soundcloud.com/). diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/spf13/pflag/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/spf13/pflag/LICENSE new file mode 100644 index 0000000..63ed1cf --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/spf13/pflag/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 Alex Ogier. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/api/v3/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/api/v3/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/api/v3/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/pkg/v3/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/pkg/v3/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/pkg/v3/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/v3/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/v3/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/go.etcd.io/etcd/client/v3/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/atomic/LICENSE.txt b/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/atomic/LICENSE.txt new file mode 100644 index 0000000..8765c9f --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/atomic/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/multierr/LICENSE.txt b/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/multierr/LICENSE.txt new file mode 100644 index 0000000..858e024 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/multierr/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2017 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/zap/LICENSE.txt b/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/zap/LICENSE.txt new file mode 100644 index 0000000..6652bed --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/go.uber.org/zap/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2016-2017 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE new file mode 100644 index 0000000..8f71f43 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/appengine/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/appengine/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/appengine/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/genproto/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/genproto/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/genproto/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/LICENSE new file mode 100644 index 0000000..49ea0f9 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/inf.v0/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/inf.v0/LICENSE new file mode 100644 index 0000000..87a5ced --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/inf.v0/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 Péter Surányi. Portions Copyright (c) 2009 The Go +Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE new file mode 100644 index 0000000..a4249bb --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE @@ -0,0 +1,29 @@ +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE.libyaml b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE.libyaml new file mode 100644 index 0000000..8da58fb --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/LICENSE.libyaml @@ -0,0 +1,31 @@ +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original copyright and license: + + apic.go + emitterc.go + parserc.go + readerc.go + scannerc.go + writerc.go + yamlh.go + yamlprivateh.go + +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/NOTICE new file mode 100644 index 0000000..866d74a --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v2/NOTICE @@ -0,0 +1,13 @@ +Copyright 2011-2016 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/LICENSE new file mode 100644 index 0000000..2683e4b --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/LICENSE @@ -0,0 +1,50 @@ + +This project is covered by two different licenses: MIT and Apache. + +#### MIT License #### + +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original MIT license, with the additional +copyright staring in 2011 when the project was ported over: + + apic.go emitterc.go parserc.go readerc.go scannerc.go + writerc.go yamlh.go yamlprivateh.go + +Copyright (c) 2006-2010 Kirill Simonov +Copyright (c) 2006-2011 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +### Apache License ### + +All the remaining project files are covered by the Apache license: + +Copyright (c) 2011-2019 Canonical Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/NOTICE b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/NOTICE new file mode 100644 index 0000000..866d74a --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/yaml.v3/NOTICE @@ -0,0 +1,13 @@ +Copyright 2011-2016 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/api/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/api/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/api/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apiextensions-apiserver/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apiextensions-apiserver/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apiextensions-apiserver/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apimachinery/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apimachinery/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/apimachinery/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/client-go/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/client-go/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/client-go/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/component-base/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/component-base/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/component-base/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/klog/v2/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/klog/v2/LICENSE new file mode 100644 index 0000000..37ec93a --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/klog/v2/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/kube-openapi/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/kube-openapi/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/kube-openapi/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/utils/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/utils/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/k8s.io/utils/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/sbom.json b/klyshko-operator/3RD-PARTY-LICENSES/sbom.json new file mode 100644 index 0000000..f98a25f --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/sbom.json @@ -0,0 +1,294 @@ +[ + { + "project": "vendor/github.com/davecgh/go-spew", + "license": "0BSD" + }, + { + "project": "vendor/github.com/Azure/go-autorest", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/Azure/go-autorest/logger", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/Azure/go-autorest/autorest", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/Azure/go-autorest/autorest/adal", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/Azure/go-autorest/autorest/date", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/Azure/go-autorest/tracing", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/go-logr/logr", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/go-logr/zapr", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/onsi/ginkgo", + "license": "MIT" + }, + { + "project": "vendor/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty", + "license": "MIT" + }, + { + "project": "vendor/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable", + "license": "MIT" + }, + { + "project": "vendor/github.com/onsi/gomega", + "license": "MIT" + }, + { + "project": "vendor/github.com/matttproud/golang_protobuf_extensions", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/spf13/pflag", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/github.com/beorn7/perks", + "license": "MIT" + }, + { + "project": "vendor/github.com/cespare/xxhash/v2", + "license": "MIT" + }, + { + "project": "vendor/github.com/coreos/go-systemd/v22", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/coreos/go-semver", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/nxadm/tail", + "license": "MIT" + }, + { + "project": "vendor/github.com/nxadm/tail/ratelimiter", + "license": "MIT" + }, + { + "project": "vendor/github.com/json-iterator/go", + "license": "MIT" + }, + { + "project": "vendor/github.com/evanphx/json-patch", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/github.com/google/go-cmp", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/github.com/google/gofuzz", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/google/uuid", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/github.com/form3tech-oss/jwt-go", + "license": "MIT" + }, + { + "project": "vendor/github.com/imdario/mergo", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/github.com/golang/groupcache", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/golang/protobuf", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/github.com/prometheus/client_model", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/prometheus/common", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg", + "license": "Unlicense" + }, + { + "project": "vendor/github.com/prometheus/procfs", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/prometheus/client_golang", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/modern-go/concurrent", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/modern-go/reflect2", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/googleapis/gnostic", + "license": "Apache-2.0" + }, + { + "project": "vendor/github.com/fsnotify/fsnotify", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/github.com/hashicorp/golang-lru", + "license": "MPL-2.0-no-copyleft-exception" + }, + { + "project": "vendor/go.uber.org/atomic", + "license": "MIT" + }, + { + "project": "vendor/go.uber.org/multierr", + "license": "MIT" + }, + { + "project": "vendor/go.uber.org/zap", + "license": "MIT" + }, + { + "project": "vendor/sigs.k8s.io/structured-merge-diff/v4", + "license": "Apache-2.0" + }, + { + "project": "vendor/sigs.k8s.io/controller-runtime", + "license": "Apache-2.0" + }, + { + "project": "vendor/golang.org/x/net", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/golang.org/x/crypto", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/golang.org/x/oauth2", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/golang.org/x/sys", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/golang.org/x/time", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/golang.org/x/term", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/golang.org/x/text", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/gomodules.xyz/jsonpatch/v2", + "license": "Apache-2.0" + }, + { + "project": "vendor/go.etcd.io/etcd/api/v3", + "license": "Apache-2.0" + }, + { + "project": "vendor/go.etcd.io/etcd/client/pkg/v3", + "license": "Apache-2.0" + }, + { + "project": "vendor/go.etcd.io/etcd/client/v3", + "license": "Apache-2.0" + }, + { + "project": "vendor/google.golang.org/grpc", + "license": "Apache-2.0" + }, + { + "project": "vendor/google.golang.org/appengine", + "license": "Apache-2.0" + }, + { + "project": "vendor/google.golang.org/protobuf", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/google.golang.org/genproto", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/apiextensions-apiserver", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/api", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/utils", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/kube-openapi", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/klog/v2", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/component-base", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/apimachinery", + "license": "Apache-2.0" + }, + { + "project": "vendor/k8s.io/client-go", + "license": "Apache-2.0" + }, + { + "project": "vendor/cloud.google.com/go", + "license": "Apache-2.0" + }, + { + "project": "vendor/gopkg.in/yaml.v2", + "license": "Apache-2.0" + }, + { + "project": "vendor/gopkg.in/yaml.v3", + "license": "Apache-2.0" + }, + { + "project": "vendor/gopkg.in/tomb.v1", + "license": "BSD-3-Clause" + }, + { + "project": "vendor/gopkg.in/inf.v0", + "license": "BSD-3-Clause" + } +] diff --git a/klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/controller-runtime/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/controller-runtime/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/controller-runtime/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/structured-merge-diff/v4/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/structured-merge-diff/v4/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/sigs.k8s.io/structured-merge-diff/v4/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-operator/create-sbom.sh b/klyshko-operator/create-sbom.sh new file mode 100755 index 0000000..4d38215 --- /dev/null +++ b/klyshko-operator/create-sbom.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# check that required tools are installed +# +if ! command -v jq &> /dev/null +then + echo "jq could not be found! See https://stedolan.github.io/jq/download/" + exit 1 +fi +if ! command -v license-detector &> /dev/null +then + echo "license-detector could not be found! See https://github.com/go-enry/go-license-detector" + exit 1 +fi +if ! command -v sponge &> /dev/null +then + echo "sponge could not be found! See https://command-not-found.com/sponge" + exit 1 +fi + +# download go dependencies into vendor directory - so we have all required license files +go mod vendor + +# Traverses the vendor folder and collects license information in the 3RD-PARTY-LICENSES/sbom.json file. License and +# notice files are copied to the respective subfolder in the 3RD-PARTY-LICENSES folder. +SBOM_FILE="3RD-PARTY-LICENSES/sbom.json" +echo "[]" > "${SBOM_FILE}" +COUNT=$(find vendor -type d | wc -l) +POS=0 +FOUND=0 +RES_FILE=$(mktemp /tmp/result.XXXXXX) +echo "Traversing ${COUNT} directories" +find vendor -type d | while IFS= read -r d; do + echo -ne "\r${POS}/${COUNT}: ${FOUND} licenses found" + license-detector "$d" -f json > "${RES_FILE}" + if ! grep -q "error" "${RES_FILE}"; then + jq -s ".[0] + [.[1][] | { project: .project, license: .matches[0].license }]" "${SBOM_FILE}" "${RES_FILE}" | sponge "${SBOM_FILE}" + ARTIFACT_FOLDER="3RD-PARTY-LICENSES/${d#*/}" + mkdir -p "${ARTIFACT_FOLDER}" + cp "${d}"/LICENSE* "${d}"/LICENCE* "${d}"/Licence* "${d}"/NOTICE* "${ARTIFACT_FOLDER}" 2>/dev/null || true + ((FOUND++)) + fi + ((POS++)) +done +echo -e "\nDONE" \ No newline at end of file From 31c5cde88473e31f50b48d162b25a39fe47b164f Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 08:08:53 +0200 Subject: [PATCH 13/69] Use latest CLI version in provisioner Signed-off-by: Sven Trieflinger --- klyshko-provisioner/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/klyshko-provisioner/Dockerfile b/klyshko-provisioner/Dockerfile index d9c3aaa..29773d4 100644 --- a/klyshko-provisioner/Dockerfile +++ b/klyshko-provisioner/Dockerfile @@ -6,15 +6,15 @@ # FROM ghcr.io/carbynestack/openjdk:jre8-20210827 -ARG CLI_VERSION=0.1-SNAPSHOT-1576571202-7-cf3db5b +ARG CLI_VERSION=0.2-SNAPSHOT-2336890983-14-a4260ab RUN apt-get update && apt-get install -y --no-install-recommends \ gettext-base \ + curl \ && rm -rf /var/lib/apt/lists/* # Download Carbyne Stack CLI Jar at specified version -# RUN curl -o cs.jar -L https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-$CLI_VERSION-jar-with-dependencies.jar -COPY cs.jar . +RUN curl -o cs.jar -L https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-$CLI_VERSION-jar-with-dependencies.jar # Copy resources COPY kii-provisioner-run.sh . From acb459326030fcdf048b4edbe7e37c09be477118 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 11:55:33 +0200 Subject: [PATCH 14/69] Add readme for fake-offline crg Signed-off-by: Sven Trieflinger --- .pre-commit-config.yaml | 5 ++- .../{tuple_type_mapping.md => README.md} | 34 ++++++++++++------- 2 files changed, 26 insertions(+), 13 deletions(-) rename klyshko-mp-spdz/{tuple_type_mapping.md => README.md} (56%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25f9b60..d0ce786 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,4 +24,7 @@ repos: rev: v0.31.1 hooks: - id: markdownlint - exclude: ^3RD-PARTY-LICENSES/.*|klyshko-mp-spdz/tuple_type_mapping.md$ + args: + - --disable + - MD013 + exclude: ^3RD-PARTY-LICENSES/.*$ diff --git a/klyshko-mp-spdz/tuple_type_mapping.md b/klyshko-mp-spdz/README.md similarity index 56% rename from klyshko-mp-spdz/tuple_type_mapping.md rename to klyshko-mp-spdz/README.md index 8b3590e..f2fe775 100644 --- a/klyshko-mp-spdz/tuple_type_mapping.md +++ b/klyshko-mp-spdz/README.md @@ -1,12 +1,22 @@ -| Type | Flag | Folder | Header Length | Verified | -| -------------------------- | -------------- | --------------------- | ------------- | -------- | -| BIT_GFP | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | yes | -| BIT_GF2N | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | yes | -| INPUT_MASK_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | -| INPUT_MASK_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | -| INVERSE_TUPLE_GFP | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | yes | -| INVERSE_TUPLE_GF2N | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | yes | -| SQUARE_TUPLE_GFP | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | yes | -| SQUARE_TUPLE_GF2N | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | yes | -| MULTIPLICATION_TRIPLE_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | yes | -| MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | yes | +# Klyshko Fake Correlated Randomness Generator + +Provides a Klyshko Correlated Randomness Generator that uses the MP-SPDZ ability +to generate fake tuples. + +## KII Tuple Type Mapping + +The mapping from KII tuple types to the flags required for invoking the +`Fake-Offline.x` executable is as follows: + +| Type | Flag | Folder | Header Length | +| -------------------------- | -------------- | --------------------- | ------------- | +| BIT_GFP | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | +| BIT_GF2N | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | +| INPUT_MASK_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | +| INPUT_MASK_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | +| INVERSE_TUPLE_GFP | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | +| INVERSE_TUPLE_GF2N | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | +| SQUARE_TUPLE_GFP | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | +| SQUARE_TUPLE_GF2N | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | +| MULTIPLICATION_TRIPLE_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | +| MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | From 1b03e71037e358b794325d90b8543491ad319dc7 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 12:03:52 +0200 Subject: [PATCH 15/69] Move deploy scripts to hack folder Signed-off-by: Sven Trieflinger --- deploy.sh => hack/deploy.sh | 6 +++--- klyshko-mp-spdz/{ => hack}/deploy.sh | 0 klyshko-operator/{ => hack}/deploy.sh | 0 klyshko-provisioner/{ => hack}/deploy.sh | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename deploy.sh => hack/deploy.sh (91%) rename klyshko-mp-spdz/{ => hack}/deploy.sh (100%) rename klyshko-operator/{ => hack}/deploy.sh (100%) rename klyshko-provisioner/{ => hack}/deploy.sh (100%) diff --git a/deploy.sh b/hack/deploy.sh similarity index 91% rename from deploy.sh rename to hack/deploy.sh index 66a1a6e..8a209f5 100755 --- a/deploy.sh +++ b/hack/deploy.sh @@ -11,13 +11,13 @@ helm install test-etcd --set auth.rbac.enabled=false --set service.type=LoadBala ( cd klyshko-mp-spdz -./deploy.sh +hack/deploy.sh ) ( cd klyshko-provisioner -./deploy.sh +hack/deploy.sh ) ( cd klyshko-operator -./deploy.sh +hack/deploy.sh ) \ No newline at end of file diff --git a/klyshko-mp-spdz/deploy.sh b/klyshko-mp-spdz/hack/deploy.sh similarity index 100% rename from klyshko-mp-spdz/deploy.sh rename to klyshko-mp-spdz/hack/deploy.sh diff --git a/klyshko-operator/deploy.sh b/klyshko-operator/hack/deploy.sh similarity index 100% rename from klyshko-operator/deploy.sh rename to klyshko-operator/hack/deploy.sh diff --git a/klyshko-provisioner/deploy.sh b/klyshko-provisioner/hack/deploy.sh similarity index 100% rename from klyshko-provisioner/deploy.sh rename to klyshko-provisioner/hack/deploy.sh From 825aa4cac385f22ed58ce92f59c6f0ec1e2dbe4c Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 13:19:41 +0200 Subject: [PATCH 16/69] Add architecture and usage instructions to README Signed-off-by: Sven Trieflinger --- README.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 19eb151..8236d11 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Carbyne Stack Klyshko Correlated Randomness Generation -Klyshko is a kubernetes-native open source correlated randomness generator +[![stability-wip](https://img.shields.io/badge/stability-wip-lightgrey.svg)](https://github.com/mkenney/software-guides/blob/master/STABILITY-BADGES.md#work-in-progress) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) + +Klyshko is a kubernetes-native open source correlated randomness generator (CRG) service for Secure Multiparty Computation in the offline/online model and part of [Carbyne Stack](https://github.com/carbynestack). @@ -18,7 +22,62 @@ The analogy to the *Klyshko* service is that secret shared tuples are correlated and thus kind of "entangled" and that the microservice is the implementation of the process that creates the tuples. -## Klyshko Integration Interface +## Architecture + +Klyshko consists of three main components: + +- *Correlated Randomness Generators (CRGs)* are the workhorses within Klyshko. + They are actually generating correlated randomness. CRGs are packaged as + Docker images and have to implement the + [Klyshko Integration Interface (KII)](#klyshko-integration-interface-kii). +- The *Klyshko Operator* coordinates the invocation of CRGs across the VCPs in a + VC. It consists of a number of components implemented as a Kubernetes API + called `klyshko.carbnyestack.io/v1alpha1` providing the following + [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/): + - A *Scheduler* (kind: `TupleGenerationScheduler`) monitors the availability + of correlated randomness within the VC using the + [Castor](https://github.com/carbynestack/castor) *Telemetry API* and + schedules CRG invocations accordingly. + - A *Job* (kind: `TupleGenerationJob`) abstracts a CRG invocation across the + VCPs of a VC. The job holds the specification of the correlated randomness + to be generated including tuple type and the number of tuples to be + generated. + - A *Task* (kind: `TupleGenerationTask`) represents a local or remote + execution of a CRG. A task exposes the state of the invocation on a single + VCP. On the job level task states are aggregated into a job state. Remote + tasks are proxied locally to make their state available to the job + controller. The task controller makes use of the + [Klyshko Integration Interface (KII)](#klyshko-integration-interface-kii) to + interact with different CRG implementations in an implementation-independent + way. +- The *Klyshko Provisioner* is used to upload the generated correlated + randomness to [Castor](https://github.com/carbynestack/castor). + +Klyshko uses an [etcd](https://etcd.io/) cluster to manage distributed state and +to orchestrate actions across VCPs. + +## Usage + +To deploy Klyshko to your VC you have to install the operator on all VCPs. You +then create a scheduler on **one** of the clusters by applying the respective +manifest, e.g., + +```yaml +apiVersion: klyshko.carbnyestack.io/v1alpha1 +kind: TupleGenerationScheduler +metadata: + name: sample-crg-scheduler +spec: + concurrency: 3 + threshold: 500000 +``` + +Klyshko will start producing correlated randomness by creating respective jobs +whenever the number of tuples for a specific type drops below the given +`threshold`. `concurrency` specifies the maximum number of jobs are allowed to +run concurrently. This is the upper limit across all tuple types together. + +## Klyshko Integration Interface (KII) > **IMPORTANT**: This is an initial incomplete version of the KII that is > subject to change without notice. For the time being it is very much @@ -26,9 +85,11 @@ the process that creates the tuples. > [MP-SPDZ](https://github.com/data61/MP-SPDZ) project. *Klyshko* has been designed to allow for easy integration of different -correlated randomness generators (CRGs). Integration is done by means of +*Correlated Randomness Generators* (CRGs). Integration is done by means of providing a docker image containing the CRG that implements the *Klyshko -Integration Interface* (KII). +Integration Interface* (KII). The parameters required by the CRG are provided +using a mix of environment variables and files made available to the container +during execution. See below for a detailed description. > **TIP**: For an example of how to integrate the > [MP-SPDZ](https://github.com/data61/MP-SPDZ) CRG producing *fake* tuples see @@ -36,9 +97,10 @@ Integration Interface* (KII). ### Entrypoint -The CRG docker image must contain a `kii-run.sh` script in the working directory -that performs the tuple generation process. The script must terminate with a -non-zero exit code in case the tuples can not be generated. +The CRG docker image must contain a script called `kii-run.sh` in the working +directory that spawns the tuple generation process. The script must terminate +with a non-zero exit code if and only if the tuples can not be generated for +some reason. ### Environment Variables @@ -49,16 +111,17 @@ the tuple generation and provisioning process. - `KII_JOB_ID`: The Type 4 UUID used as a job identifier. This is the same among all VCPs in the VC. -- `KII_TUPLES_PER_JOB`: The number of tuples to be generated. The CRG should try - to match the requested number but is not required to do so. +- `KII_TUPLES_PER_JOB`: The number of tuples to be generated. The CRG should + make its best effort to match the requested number but is not required to do + so in case optimizations like batching mandate it. - `KII_PLAYER_NUMBER`: The 0-based number of the local VCP. - `KII_PLAYER_COUNT`: The overall number of VCPs in the VC. - `KII_TUPLE_TYPE`: The tuple type to generate. Must be one of - - `bit_gfp`, `bit_gf2n` - - `inputmask_gfp`, `inputmask_gf2n` - - `inversetuple_gfp`, `inversetuple_gf2n` - - `squaretuple_gfp`, `squaretuple_gf2n` - - `multiplicationtriple_gfp`, `multiplicationtriple_gf2n` + - `BIT_GFP`, `BIT_GF2N` + - `INPUT_MASK_GFP`, `INPUT_MASK_GF2N` + - `INVERSE_TUPLE_GFP`, `INVERSE_TUPLE_GF2N` + - `SQUARE_TUPLE_GFP`, `SQUARE_TUPLE_GF2N` + - `MULTIPLICATION_TRIPLE_GFP`, `MULTIPLICATION_TRIPLE_GF2N` #### Output @@ -74,6 +137,13 @@ The prime to be used for generating prime field tuples is provided in the file The MAC key shares for prime and binary fields are made available as files `mac_key_share_p` and `mac_key_share_2` in folder `/etc/kii/secret-params`. +## Development + +The `deploy.sh` scripts in the `hack` folders (top-level and within modules) can +be used to (re-)deploy Klyshko to a 2-party Carbyne Stack VC setup as described +in the [tutorials](https://carbynestack.io/getting-started) on the Carbyne Stack +website. + ## License Carbyne Stack *Klyshko Correlated Randomness Generation Subsystem* is From b4bfeef00e5328a87791fa11ce567d5420899821 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 13:22:35 +0200 Subject: [PATCH 17/69] Update copyright headers Signed-off-by: Sven Trieflinger --- .../config/samples/engine-params-extra.yaml.template | 2 +- .../config/samples/engine-params-secret.yaml.template | 2 +- klyshko-operator/config/samples/engine-params.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/klyshko-operator/config/samples/engine-params-extra.yaml.template b/klyshko-operator/config/samples/engine-params-extra.yaml.template index 9f5215f..9261e25 100644 --- a/klyshko-operator/config/samples/engine-params-extra.yaml.template +++ b/klyshko-operator/config/samples/engine-params-extra.yaml.template @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 - for information on the respective copyright owner +# Copyright (c) 2022 - for information on the respective copyright owner # see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. # # SPDX-License-Identifier: Apache-2.0 diff --git a/klyshko-operator/config/samples/engine-params-secret.yaml.template b/klyshko-operator/config/samples/engine-params-secret.yaml.template index b2ee084..1eaeba2 100644 --- a/klyshko-operator/config/samples/engine-params-secret.yaml.template +++ b/klyshko-operator/config/samples/engine-params-secret.yaml.template @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 - for information on the respective copyright owner +# Copyright (c) 2022 - for information on the respective copyright owner # see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. # # SPDX-License-Identifier: Apache-2.0 diff --git a/klyshko-operator/config/samples/engine-params.yaml b/klyshko-operator/config/samples/engine-params.yaml index cbf2847..c34336d 100644 --- a/klyshko-operator/config/samples/engine-params.yaml +++ b/klyshko-operator/config/samples/engine-params.yaml @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 - for information on the respective copyright owner +# Copyright (c) 2022 - for information on the respective copyright owner # see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. # # SPDX-License-Identifier: Apache-2.0 From e140000e8c4d1ae73f38942d559a4822422a6f68 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 13:25:42 +0200 Subject: [PATCH 18/69] Add copyright headers to sample manifests Signed-off-by: Sven Trieflinger --- klyshko-operator/config/samples/apollo-vcp.yaml | 6 ++++++ .../samples/klyshko_v1alpha1_tuplegenerationjob.yaml | 6 ++++++ .../klyshko_v1alpha1_tuplegenerationjob.yaml.template | 6 ++++++ .../klyshko_v1alpha1_tuplegenerationscheduler.yaml | 10 ++++++++-- .../samples/klyshko_v1alpha1_tuplegenerationtask.yaml | 8 ++++++-- klyshko-operator/config/samples/starbuck-vcp.yaml | 6 ++++++ 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/klyshko-operator/config/samples/apollo-vcp.yaml b/klyshko-operator/config/samples/apollo-vcp.yaml index 5e060b8..d520578 100644 --- a/klyshko-operator/config/samples/apollo-vcp.yaml +++ b/klyshko-operator/config/samples/apollo-vcp.yaml @@ -1,3 +1,9 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# apiVersion: v1 kind: ConfigMap metadata: diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index 53c0362..fef196b 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -1,3 +1,9 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# apiVersion: klyshko.carbnyestack.io/v1alpha1 kind: TupleGenerationJob metadata: diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template index abc6b67..964e8f1 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template @@ -1,3 +1,9 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# apiVersion: klyshko.carbnyestack.io/v1alpha1 kind: TupleGenerationJob metadata: diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml index f448ea8..69a9ecf 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml @@ -1,7 +1,13 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# apiVersion: klyshko.carbnyestack.io/v1alpha1 kind: TupleGenerationScheduler metadata: name: tuplegenerationscheduler-sample spec: - concurrency: 3 - threshold: 400000 + concurrency: 2 + threshold: 50000 diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml index a1c9324..ec571b8 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml @@ -1,7 +1,11 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# apiVersion: klyshko.carbnyestack.io/v1alpha1 kind: TupleGenerationTask metadata: name: tuplegenerationtask-sample spec: - # Add fields here - foo: bar diff --git a/klyshko-operator/config/samples/starbuck-vcp.yaml b/klyshko-operator/config/samples/starbuck-vcp.yaml index 1761940..c3d8b2c 100644 --- a/klyshko-operator/config/samples/starbuck-vcp.yaml +++ b/klyshko-operator/config/samples/starbuck-vcp.yaml @@ -1,3 +1,9 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# apiVersion: v1 kind: ConfigMap metadata: From e5594d9f589d69ace486916079e9bb710f1870d3 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 13:27:08 +0200 Subject: [PATCH 19/69] Change type for count property of job Signed-off-by: Sven Trieflinger --- .../crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml index 9b62255..936495f 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -52,7 +52,6 @@ spec: properties: count: exclusiveMinimum: true - format: int32 minimum: 0 type: integer id: From 01e7d444b2f92ff7c01635e852cf1d36e89d62da Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 13:43:56 +0200 Subject: [PATCH 20/69] Add README to provisioner Signed-off-by: Sven Trieflinger --- klyshko-provisioner/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 klyshko-provisioner/README.md diff --git a/klyshko-provisioner/README.md b/klyshko-provisioner/README.md new file mode 100644 index 0000000..0500e95 --- /dev/null +++ b/klyshko-provisioner/README.md @@ -0,0 +1,11 @@ +# Klyshko Provisioner + +Uploads tuples generated by a CRG to [Castor] using the Carbyne Stack [CLI]. +According to the [Klyshko Integration Interface (KII)][kii] tuples are read from +the path made available in the environment variable `KII_TUPLE_FILE`, uploaded +as tuples of type given in `KII_TUPLE_TYPE` with the chunk identifier specified +in `KII_JOB_ID`. + +[castor]: https://github.com/carbynestack/castor +[cli]: https://github.com/carbynestack/cli +[kii]: ../klyshko-operator/README.md#klyshko-integration-interface-kii From 1074963f46c047f22e7c56e424fb8262b1a9af74 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 14:29:13 +0200 Subject: [PATCH 21/69] Add / revise READMEs in submodules Signed-off-by: Sven Trieflinger --- klyshko-mp-spdz/README.md | 14 ++++++++++---- klyshko-operator/README.md | 13 +++++++++++++ klyshko-provisioner/README.md | 15 ++++++++++----- 3 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 klyshko-operator/README.md diff --git a/klyshko-mp-spdz/README.md b/klyshko-mp-spdz/README.md index f2fe775..2eda353 100644 --- a/klyshko-mp-spdz/README.md +++ b/klyshko-mp-spdz/README.md @@ -1,14 +1,17 @@ # Klyshko Fake Correlated Randomness Generator -Provides a Klyshko Correlated Randomness Generator that uses the MP-SPDZ ability -to generate fake tuples. +Provides a Klyshko *Correlated Randomness Generator* (CRG) that uses the MP-SPDZ +ability to generate fake tuples. + +For a high-level description of the Klyshko subsystem, its components, and how +these interact, please see the [README] at the root of this repository. ## KII Tuple Type Mapping -The mapping from KII tuple types to the flags required for invoking the +The mapping from [KII] tuple types to the flags required for invoking the `Fake-Offline.x` executable is as follows: -| Type | Flag | Folder | Header Length | +| KII Tuple Type | Flag | Folder | Header Length | | -------------------------- | -------------- | --------------------- | ------------- | | BIT_GFP | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | | BIT_GF2N | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | @@ -20,3 +23,6 @@ The mapping from KII tuple types to the flags required for invoking the | SQUARE_TUPLE_GF2N | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | | MULTIPLICATION_TRIPLE_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | | MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | + +[kii]: ../klyshko-operator/README.md#klyshko-integration-interface-kii +[readme]: ../README.md diff --git a/klyshko-operator/README.md b/klyshko-operator/README.md new file mode 100644 index 0000000..956addd --- /dev/null +++ b/klyshko-operator/README.md @@ -0,0 +1,13 @@ +# Klyshko Operator + +The *Klyshko Operator* is the brain of the Klyshko subsystem. A set of +Kubernetes [Controllers] is used to drive the lifecycle of jobs (VC-level) and +tasks (VCP-level) and to decide when jobs are to be scheduled based on observing +tuple availability in [Castor]. + +For a high-level description of the Klyshko subsystem, its components, and how +these interact, please see the [README] at the root of this repository. + +[castor]: https://github.com/carbynestack/castor +[controllers]: https://kubernetes.io/docs/concepts/architecture/controller/ +[readme]: ../README.md diff --git a/klyshko-provisioner/README.md b/klyshko-provisioner/README.md index 0500e95..733a5c2 100644 --- a/klyshko-provisioner/README.md +++ b/klyshko-provisioner/README.md @@ -1,11 +1,16 @@ # Klyshko Provisioner -Uploads tuples generated by a CRG to [Castor] using the Carbyne Stack [CLI]. -According to the [Klyshko Integration Interface (KII)][kii] tuples are read from -the path made available in the environment variable `KII_TUPLE_FILE`, uploaded -as tuples of type given in `KII_TUPLE_TYPE` with the chunk identifier specified -in `KII_JOB_ID`. +Uploads tuples generated by a *Correlated Randomness Generator* (CRG) to +[Castor] using the Carbyne Stack [CLI]. In accordance with the +[Klyshko Integration Interface (KII)][kii], tuples are read from the path made +available in the environment variable `KII_TUPLE_FILE`, uploaded as tuples of +the type given in `KII_TUPLE_TYPE`, and with the chunk identifier specified in +`KII_JOB_ID`. + +For a high-level description of the Klyshko subsystem, its components, and how +these interact, please see the [README] at the root of this repository. [castor]: https://github.com/carbynestack/castor [cli]: https://github.com/carbynestack/cli [kii]: ../klyshko-operator/README.md#klyshko-integration-interface-kii +[readme]: ../README.md From fc98c5ca2569f259b95313ab1d04174bd3cdc0de Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 15:16:49 +0200 Subject: [PATCH 22/69] Use compliant base image in fake CRG, fix bugs in KII implementation Signed-off-by: Sven Trieflinger --- klyshko-mp-spdz/Dockerfile.fake-offline | 38 +------------------------ klyshko-mp-spdz/README.md | 10 +++---- klyshko-mp-spdz/kii-run.sh | 15 +++++----- 3 files changed, 14 insertions(+), 49 deletions(-) diff --git a/klyshko-mp-spdz/Dockerfile.fake-offline b/klyshko-mp-spdz/Dockerfile.fake-offline index b966156..50ea9e4 100644 --- a/klyshko-mp-spdz/Dockerfile.fake-offline +++ b/klyshko-mp-spdz/Dockerfile.fake-offline @@ -4,43 +4,7 @@ # # SPDX-License-Identifier: Apache-2.0 # -FROM ubuntu:20.04 - -# Install prerequisites -ENV TZ=Europe/Berlin -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN apt-get update && apt-get install -y --no-install-recommends \ - automake \ - build-essential \ - git \ - libboost-dev \ - libboost-thread-dev \ - libntl-dev \ - libsodium-dev \ - libssl-dev \ - libtool \ - m4 \ - python3 \ - texinfo \ - yasm \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# https://github.com/data61/MP-SPDZ -ARG MP_SPDZ_REPO="https://github.com/carbynestack/MP-SPDZ" -# v0.2.8 -ARG MP_SPDZ_VERSION="add-prng-seed-option" - -# Checkout MP-SPDZ repository at required version -RUN git clone ${MP_SPDZ_REPO} mp-spdz-${MP_SPDZ_VERSION} -WORKDIR mp-spdz-${MP_SPDZ_VERSION} -RUN git checkout ${MP_SPDZ_VERSION} && \ - git submodule set-url -- mpir https://github.com/wbhart/mpir - -# Build fake offline phase executable -RUN echo "MY_CFLAGS += -DINSECURE" >> CONFIG.mine -RUN make -j 8 tldr -RUN make -j 8 Fake-Offline.x +FROM ghcr.io/carbynestack/spdz:5350e66 # Copy resources COPY kii-run.sh . diff --git a/klyshko-mp-spdz/README.md b/klyshko-mp-spdz/README.md index 2eda353..755c126 100644 --- a/klyshko-mp-spdz/README.md +++ b/klyshko-mp-spdz/README.md @@ -14,15 +14,15 @@ The mapping from [KII] tuple types to the flags required for invoking the | KII Tuple Type | Flag | Folder | Header Length | | -------------------------- | -------------- | --------------------- | ------------- | | BIT_GFP | --nbits 0,n | 2-p-128/Bits-p-P0 | 37 | -| BIT_GF2N | --nbits n,0 | 2-2-128/Bits-2-P0 | 34 | +| BIT_GF2N | --nbits n,0 | 2-2-40/Bits-2-P0 | 34 | | INPUT_MASK_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | -| INPUT_MASK_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | +| INPUT_MASK_GF2N | --ntriples n,0 | 2-2-40/Triples-2-P0 | 34 | | INVERSE_TUPLE_GFP | --ninverses n | 2-p-128/Inverses-p-P0 | 37 | -| INVERSE_TUPLE_GF2N | --ninverses n | 2-2-128/Inverses-2-P0 | 34 | +| INVERSE_TUPLE_GF2N | --ninverses n | 2-2-40/Inverses-2-P0 | 34 | | SQUARE_TUPLE_GFP | --nsquares 0,n | 2-p-128/Squares-p-P0 | 37 | -| SQUARE_TUPLE_GF2N | --nsquares n,0 | 2-2-128/Squares-2-P0 | 34 | +| SQUARE_TUPLE_GF2N | --nsquares n,0 | 2-2-40/Squares-2-P0 | 34 | | MULTIPLICATION_TRIPLE_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | -| MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-128/Triples-2-P0 | 34 | +| MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-40/Triples-2-P0 | 34 | [kii]: ../klyshko-operator/README.md#klyshko-integration-interface-kii [readme]: ../README.md diff --git a/klyshko-mp-spdz/kii-run.sh b/klyshko-mp-spdz/kii-run.sh index b7472e9..0e53b08 100755 --- a/klyshko-mp-spdz/kii-run.sh +++ b/klyshko-mp-spdz/kii-run.sh @@ -26,15 +26,15 @@ declare -A argsByType=( ) declare -A folderByType=( ["BIT_GFP"]="2-p-128/Bits-p-P${pn}" - ["BIT_GF2N"]="2-2-128/Bits-2-P${pn}" + ["BIT_GF2N"]="2-2-40/Bits-2-P${pn}" ["INPUT_MASK_GFP"]="2-p-128/Triples-p-P${pn}" - ["INPUT_MASK_GF2N"]="2-2-128/Triples-2-P${pn}" + ["INPUT_MASK_GF2N"]="2-2-40/Triples-2-P${pn}" ["INVERSE_TUPLE_GFP"]="2-p-128/Inverses-p-P${pn}" - ["INVERSE_TUPLE_GF2N"]="2-2-128/Inverses-2-P${pn}" + ["INVERSE_TUPLE_GF2N"]="2-2-40/Inverses-2-P${pn}" ["SQUARE_TUPLE_GFP"]="2-p-128/Squares-p-P${pn}" - ["SQUARE_TUPLE_GF2N"]="2-2-128/Squares-2-P${pn}" + ["SQUARE_TUPLE_GF2N"]="2-2-40/Squares-2-P${pn}" ["MULTIPLICATION_TRIPLE_GFP"]="2-p-128/Triples-p-P${pn}" - ["MULTIPLICATION_TRIPLE_GF2N"]="2-2-128/Triples-2-P${pn}" + ["MULTIPLICATION_TRIPLE_GF2N"]="2-2-40/Triples-2-P${pn}" ) # Provide required parameters in MP-SPDZ "Player-Data" folder @@ -43,9 +43,10 @@ declare fields=("p" "2") for f in "${fields[@]}" do - folder="Player-Data/${KII_PLAYER_COUNT}-${f}-128" + [[ "$f" = "p" ]] && bit_width="128" || bit_width="40" + folder="Player-Data/${KII_PLAYER_COUNT}-${f}-${bit_width}" mkdir -p "${folder}" - echo "Providing parameters for field ${f}-128 in folder ${folder}" + echo "Providing parameters for field ${f}-${bit_width} in folder ${folder}" # Write MAC key shares for pn in $(seq 0 $((KII_PLAYER_COUNT-1))) From f577e69732dd7b2b05468c057ea3b03b9066045a Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 15:57:38 +0200 Subject: [PATCH 23/69] Include AUTHORS files when collecting 3rd party license information Signed-off-by: Sven Trieflinger --- .../github.com/fsnotify/fsnotify/AUTHORS | 52 ++++++++++++++ .../github.com/golang/protobuf/AUTHORS | 3 + .../bitbucket.org/ww/goautoneg/README.txt | 67 +++++++++++++++++++ .../golang.org/x/crypto/AUTHORS | 3 + .../golang.org/x/net/AUTHORS | 3 + .../golang.org/x/oauth2/AUTHORS | 3 + .../golang.org/x/sys/AUTHORS | 3 + .../golang.org/x/term/AUTHORS | 3 + .../golang.org/x/text/AUTHORS | 3 + .../golang.org/x/time/AUTHORS | 3 + .../google.golang.org/grpc/AUTHORS | 1 + .../google.golang.org/protobuf/AUTHORS | 3 + klyshko-operator/create-sbom.sh | 2 +- 13 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/README.txt create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/AUTHORS create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/AUTHORS diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/AUTHORS new file mode 100644 index 0000000..5ab5d41 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/fsnotify/fsnotify/AUTHORS @@ -0,0 +1,52 @@ +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# You can update this list using the following command: +# +# $ git shortlog -se | awk '{print $2 " " $3 " " $4}' + +# Please keep the list sorted. + +Aaron L +Adrien Bustany +Amit Krishnan +Anmol Sethi +Bjørn Erik Pedersen +Bruno Bigras +Caleb Spare +Case Nelson +Chris Howey +Christoffer Buchholz +Daniel Wagner-Hall +Dave Cheney +Evan Phoenix +Francisco Souza +Hari haran +John C Barstow +Kelvin Fo +Ken-ichirou MATSUZAWA +Matt Layher +Nathan Youngman +Nickolai Zeldovich +Patrick +Paul Hammond +Pawel Knap +Pieter Droogendijk +Pursuit92 +Riku Voipio +Rob Figueiredo +Rodrigo Chiossi +Slawek Ligus +Soge Zhang +Tiffany Jernigan +Tilak Sharma +Tom Payne +Travis Cline +Tudor Golubenco +Vahe Khachikyan +Yukang +bronze1man +debrando +henrikedwards +铁哥 diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/AUTHORS new file mode 100644 index 0000000..15167cd --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/README.txt b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/README.txt new file mode 100644 index 0000000..7723656 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/README.txt @@ -0,0 +1,67 @@ +PACKAGE + +package goautoneg +import "bitbucket.org/ww/goautoneg" + +HTTP Content-Type Autonegotiation. + +The functions in this package implement the behaviour specified in +http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + +Copyright (c) 2011, Open Knowledge Foundation Ltd. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + Neither the name of the Open Knowledge Foundation Ltd. nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +FUNCTIONS + +func Negotiate(header string, alternatives []string) (content_type string) +Negotiate the most appropriate content_type given the accept header +and a list of alternatives. + +func ParseAccept(header string) (accept []Accept) +Parse an Accept Header string returning a sorted list +of clauses + + +TYPES + +type Accept struct { + Type, SubType string + Q float32 + Params map[string]string +} +Structure to represent a clause in an HTTP Accept Header + + +SUBDIRECTORIES + + .hg diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/AUTHORS new file mode 100644 index 0000000..2b00ddb --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/crypto/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at https://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/AUTHORS new file mode 100644 index 0000000..15167cd --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/net/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/AUTHORS new file mode 100644 index 0000000..15167cd --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/oauth2/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/AUTHORS new file mode 100644 index 0000000..15167cd --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/sys/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/AUTHORS new file mode 100644 index 0000000..15167cd --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/term/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/AUTHORS new file mode 100644 index 0000000..15167cd --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/text/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/AUTHORS new file mode 100644 index 0000000..15167cd --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/golang.org/x/time/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/AUTHORS new file mode 100644 index 0000000..e491a9e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/grpc/AUTHORS @@ -0,0 +1 @@ +Google Inc. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/AUTHORS b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/AUTHORS new file mode 100644 index 0000000..2b00ddb --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/google.golang.org/protobuf/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at https://tip.golang.org/AUTHORS. diff --git a/klyshko-operator/create-sbom.sh b/klyshko-operator/create-sbom.sh index 4d38215..a485b02 100755 --- a/klyshko-operator/create-sbom.sh +++ b/klyshko-operator/create-sbom.sh @@ -44,7 +44,7 @@ find vendor -type d | while IFS= read -r d; do jq -s ".[0] + [.[1][] | { project: .project, license: .matches[0].license }]" "${SBOM_FILE}" "${RES_FILE}" | sponge "${SBOM_FILE}" ARTIFACT_FOLDER="3RD-PARTY-LICENSES/${d#*/}" mkdir -p "${ARTIFACT_FOLDER}" - cp "${d}"/LICENSE* "${d}"/LICENCE* "${d}"/Licence* "${d}"/NOTICE* "${ARTIFACT_FOLDER}" 2>/dev/null || true + cp "${d}"/LICENSE* "${d}"/LICENCE* "${d}"/Licence* "${d}"/NOTICE* "${d}"/AUTHORS* "${d}"/Authors* "${ARTIFACT_FOLDER}" 2>/dev/null || true ((FOUND++)) fi ((POS++)) From c270633dc34fb2511a72533be82992b098613a5f Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 16:20:29 +0200 Subject: [PATCH 24/69] Add license header to .gitignore Signed-off-by: Sven Trieflinger --- .gitignore | 7 +++++++ klyshko-operator/README.md | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/.gitignore b/.gitignore index c055966..579edad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# + # IntelliJ .idea *.ipr diff --git a/klyshko-operator/README.md b/klyshko-operator/README.md index 956addd..bb9707a 100644 --- a/klyshko-operator/README.md +++ b/klyshko-operator/README.md @@ -8,6 +8,14 @@ tuple availability in [Castor]. For a high-level description of the Klyshko subsystem, its components, and how these interact, please see the [README] at the root of this repository. +## Additional License Information + +Most of the files in the `config` folder are scaffolded or auto-generated by the +[Operator SDK][opsdk]. These files inherit the license from the Operator SDK, +i.e., are distributed under the Apache 2 license. Some files are handcrafted by +the authors of Carbyne Stack and as such carry an explicit license header. + [castor]: https://github.com/carbynestack/castor [controllers]: https://kubernetes.io/docs/concepts/architecture/controller/ +[opsdk]: https://sdk.operatorframework.io/ [readme]: ../README.md From 49f813c5a9f3308dd67953a71c92df2f5c062119 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 20 May 2022 17:32:01 +0200 Subject: [PATCH 25/69] Add Codacy badge Signed-off-by: Sven Trieflinger --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8236d11..75e3dc3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Carbyne Stack Klyshko Correlated Randomness Generation [![stability-wip](https://img.shields.io/badge/stability-wip-lightgrey.svg)](https://github.com/mkenney/software-guides/blob/master/STABILITY-BADGES.md#work-in-progress) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/3a07fd83b67647138b8ea660d16cdc35)](https://www.codacy.com/gh/carbynestack/klyshko/dashboard?utm_source=github.com&utm_medium=referral&utm_content=carbynestack/klyshko&utm_campaign=Badge_Grade) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) From ad228af55755b98f2cd39546d549b8e80afeca78 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Mon, 23 May 2022 17:24:06 +0200 Subject: [PATCH 26/69] Fix first batch of SCA findings Signed-off-by: Sven Trieflinger --- klyshko-mp-spdz/README.md | 2 +- klyshko-operator/README.md | 8 ++++---- klyshko-operator/controllers/castor.go | 6 +++--- klyshko-operator/controllers/config_utils.go | 4 ++-- klyshko-operator/controllers/etcd_keys.go | 7 +++---- .../controllers/tuplegenerationjob_controller.go | 7 +++---- .../controllers/tuplegenerationscheduler_controller.go | 6 +++--- .../controllers/tuplegenerationtask_controller.go | 3 +-- klyshko-operator/hack/deploy.sh | 4 ++-- klyshko-provisioner/README.md | 2 +- 10 files changed, 23 insertions(+), 26 deletions(-) diff --git a/klyshko-mp-spdz/README.md b/klyshko-mp-spdz/README.md index 755c126..5cc8bb0 100644 --- a/klyshko-mp-spdz/README.md +++ b/klyshko-mp-spdz/README.md @@ -24,5 +24,5 @@ The mapping from [KII] tuple types to the flags required for invoking the | MULTIPLICATION_TRIPLE_GFP | --ntriples 0,n | 2-p-128/Triples-p-P0 | 37 | | MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-40/Triples-2-P0 | 34 | -[kii]: ../klyshko-operator/README.md#klyshko-integration-interface-kii +[kii]: ../README.md#klyshko-integration-interface-kii [readme]: ../README.md diff --git a/klyshko-operator/README.md b/klyshko-operator/README.md index bb9707a..c61db03 100644 --- a/klyshko-operator/README.md +++ b/klyshko-operator/README.md @@ -1,9 +1,9 @@ # Klyshko Operator The *Klyshko Operator* is the brain of the Klyshko subsystem. A set of -Kubernetes [Controllers] is used to drive the lifecycle of jobs (VC-level) and -tasks (VCP-level) and to decide when jobs are to be scheduled based on observing -tuple availability in [Castor]. +Kubernetes [Controllers][k8s-controllers] is used to drive the lifecycle of jobs +(VC-level) and tasks (VCP-level) and to decide when jobs are to be scheduled +based on observing tuple availability in [Castor]. For a high-level description of the Klyshko subsystem, its components, and how these interact, please see the [README] at the root of this repository. @@ -16,6 +16,6 @@ i.e., are distributed under the Apache 2 license. Some files are handcrafted by the authors of Carbyne Stack and as such carry an explicit license header. [castor]: https://github.com/carbynestack/castor -[controllers]: https://kubernetes.io/docs/concepts/architecture/controller/ +[k8s-controllers]: https://kubernetes.io/docs/concepts/architecture/controller/ [opsdk]: https://sdk.operatorframework.io/ [readme]: ../README.md diff --git a/klyshko-operator/controllers/castor.go b/klyshko-operator/controllers/castor.go index da1e1c0..e53a093 100644 --- a/klyshko-operator/controllers/castor.go +++ b/klyshko-operator/controllers/castor.go @@ -17,10 +17,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -func activateTupleChunk(ctx context.Context, chunkId uuid.UUID) error { - logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkId) +func activateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { + logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkID) client := &http.Client{} - url := fmt.Sprintf("http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", chunkId) // TODO Make servername configurable / use discovery + url := fmt.Sprintf("http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", chunkID) // TODO Make servername configurable / use discovery logger.Info("activating tuple chunk with castor URL", "URL", url) req, err := http.NewRequest(http.MethodPut, url, nil) if err != nil { diff --git a/klyshko-operator/controllers/config_utils.go b/klyshko-operator/controllers/config_utils.go index e6573b7..c4c1950 100644 --- a/klyshko-operator/controllers/config_utils.go +++ b/klyshko-operator/controllers/config_utils.go @@ -32,8 +32,8 @@ func localPlayerID(ctx context.Context, client *client.Client, namespace string) } // Extract playerId - if playerId, ok := cfm.Data["playerId"]; ok { - pid, err := strconv.Atoi(playerId) + if playerID, ok := cfm.Data["playerId"]; ok { + pid, err := strconv.Atoi(playerID) if err != nil { return 0, err } diff --git a/klyshko-operator/controllers/etcd_keys.go b/klyshko-operator/controllers/etcd_keys.go index dd74eb0..c2cb512 100644 --- a/klyshko-operator/controllers/etcd_keys.go +++ b/klyshko-operator/controllers/etcd_keys.go @@ -82,9 +82,8 @@ func ParseKey(s string) (Key, error) { }, PlayerID: uint(pid), }, nil - } else { - return RosterKey{ - name, - }, nil } + return RosterKey{ + name, + }, nil } diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index eaa88e0..429e9f4 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -236,9 +236,8 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R // Error reading the object - requeue the request. logger.Error(err, "failed to read job resource") return ctrl.Result{}, err - } else { - logger.Info("job exists already") } + logger.Info("job exists already") // Create roster if not existing (no etcd transaction needed as remote job creation is triggered by roster creation) resp, err := r.EtcdClient.Get(ctx, jobKey.ToEtcdKey()) @@ -348,12 +347,12 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R state = klyshkov1alpha1.JobCompleted // Activate tuples; TODO How to deal with failures here? Introduce JobActivating state, how to sync between VCPs? Is activation still used in new Castor implementation? - tupleChunkId, err := uuid.Parse(job.Spec.ID) + tupleChunkID, err := uuid.Parse(job.Spec.ID) if err != nil { logger.Error(err, "invalid job id encountered") return ctrl.Result{}, nil } - err = activateTupleChunk(ctx, tupleChunkId) + err = activateTupleChunk(ctx, tupleChunkID) if err != nil { logger.Error(err, "tuple activation failed") return ctrl.Result{}, nil diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 3cad8e6..0787d0c 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -98,18 +98,18 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req logger.Info("sorted by priority", "Metrics.Sorted", belowThreshold) // Create job for first below threshold - jobId := uuid.New().String() + jobID := uuid.New().String() job := &klyshkov1alpha1.TupleGenerationJob{ TypeMeta: metav1.TypeMeta{ APIVersion: "klyshko.carbnyestack.io/v1alpha1", Kind: "TupleGenerationJob", }, ObjectMeta: metav1.ObjectMeta{ - Name: scheduler.Name + "-" + jobId, + Name: scheduler.Name + "-" + jobID, Namespace: req.Namespace, }, Spec: klyshkov1alpha1.TupleGenerationJobSpec{ - ID: jobId, + ID: jobID, Type: belowThreshold[0].TupleType, Count: 10000, }, diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 73be267..d0fd89e 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -132,9 +132,8 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. // Error reading the object - requeue the request. logger.Error(err, "failed to read task resource") return ctrl.Result{}, err - } else { - logger.Info("task exists already") } + logger.Info("task exists already") // Create roster entry if not existing resp, err := r.EtcdClient.Get(ctx, taskKey.ToEtcdKey()) diff --git a/klyshko-operator/hack/deploy.sh b/klyshko-operator/hack/deploy.sh index 6b07f6b..87a54ff 100755 --- a/klyshko-operator/hack/deploy.sh +++ b/klyshko-operator/hack/deploy.sh @@ -47,8 +47,8 @@ do EXTRA_MAC_KEY_SHARE_P=$([ "$c" == "starbuck" ] && echo "-88222337191559387830816715872691188861" || echo "1113507028231509545156335486838233835") EXTRA_MAC_KEY_SHARE_2=$([ "$c" == "starbuck" ] && echo "f0cf6099e629fd0bda2de3f9515ab72b" || echo "c347ce3d9e165e4e85221f9da7591d98") sed -e "s/MAC_KEY_SHARE_P/${EXTRA_MAC_KEY_SHARE_P}/" -e "s/MAC_KEY_SHARE_2/${EXTRA_MAC_KEY_SHARE_2}/" config/samples/engine-params-extra.yaml.template > "/tmp/$c-engine-params-extra.yaml" - kubectl apply -f /tmp/$c-engine-params-secret.yaml - kubectl apply -f /tmp/$c-engine-params-extra.yaml + kubectl apply -f "/tmp/$c-engine-params-secret.yaml" + kubectl apply -f "/tmp/$c-engine-params-extra.yaml" kubectl apply -f config/samples/engine-params.yaml make deploy IMG="carbynestack/klyshko-operator:v0.0.1" diff --git a/klyshko-provisioner/README.md b/klyshko-provisioner/README.md index 733a5c2..c672032 100644 --- a/klyshko-provisioner/README.md +++ b/klyshko-provisioner/README.md @@ -12,5 +12,5 @@ these interact, please see the [README] at the root of this repository. [castor]: https://github.com/carbynestack/castor [cli]: https://github.com/carbynestack/cli -[kii]: ../klyshko-operator/README.md#klyshko-integration-interface-kii +[kii]: ../README.md#klyshko-integration-interface-kii [readme]: ../README.md From 543054a43dbae09a09c35bf169c0eaee528141af Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Mon, 23 May 2022 23:33:28 +0200 Subject: [PATCH 27/69] Add EOL pre-commit hook Signed-off-by: Sven Trieflinger --- .markdownlint.json | 2 +- .pre-commit-config.yaml | 4 ++++ LICENSE | 2 +- hack/deploy.sh | 2 +- klyshko-mp-spdz/hack/deploy.sh | 2 +- .../github.com/form3tech-oss/jwt-go/LICENSE | 1 - .../3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE | 1 - .../3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE | 1 - .../3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE | 1 - klyshko-operator/config/samples/apollo-vcp.yaml | 2 +- .../config/samples/engine-params-extra.yaml.template | 2 +- .../config/samples/engine-params-secret.yaml.template | 2 +- klyshko-operator/config/samples/engine-params.yaml | 2 +- klyshko-operator/config/samples/starbuck-vcp.yaml | 2 +- klyshko-operator/create-sbom.sh | 2 +- klyshko-operator/hack/boilerplate.go.txt | 2 +- klyshko-operator/hack/deploy.sh | 2 +- klyshko-provisioner/Dockerfile | 2 +- klyshko-provisioner/hack/deploy.sh | 2 +- 19 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.markdownlint.json b/.markdownlint.json index 78d8ee9..ad72af7 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -6,4 +6,4 @@ "MD043": { "headings": null } -} \ No newline at end of file +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d0ce786..ae7bafc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,10 @@ # SPDX-License-Identifier: Apache-2.0 # repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: end-of-file-fixer - repo: https://github.com/tekwizely/pre-commit-golang rev: v1.0.0-beta.5 hooks: diff --git a/LICENSE b/LICENSE index 7a4a3ea..d645695 100644 --- a/LICENSE +++ b/LICENSE @@ -199,4 +199,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/hack/deploy.sh b/hack/deploy.sh index 8a209f5..700af33 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -20,4 +20,4 @@ hack/deploy.sh ( cd klyshko-operator hack/deploy.sh -) \ No newline at end of file +) diff --git a/klyshko-mp-spdz/hack/deploy.sh b/klyshko-mp-spdz/hack/deploy.sh index fecedd4..e5619a8 100755 --- a/klyshko-mp-spdz/hack/deploy.sh +++ b/klyshko-mp-spdz/hack/deploy.sh @@ -18,4 +18,4 @@ for c in "${CLUSTERS[@]}" do echo -e "${GREEN}Loading docker image into $c${NC}" kind load docker-image carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT --name "$c" -done \ No newline at end of file +done diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE index df83a9c..b15dc8d 100644 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/form3tech-oss/jwt-go/LICENSE @@ -5,4 +5,3 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE index 0f64693..8e63345 100644 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/golang/protobuf/LICENSE @@ -25,4 +25,3 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE index 6b0b127..d645695 100644 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/googleapis/gnostic/LICENSE @@ -200,4 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE index 8f71f43..8dada3e 100644 --- a/klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE +++ b/klyshko-operator/3RD-PARTY-LICENSES/gomodules.xyz/jsonpatch/v2/LICENSE @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/klyshko-operator/config/samples/apollo-vcp.yaml b/klyshko-operator/config/samples/apollo-vcp.yaml index d520578..f3ed972 100644 --- a/klyshko-operator/config/samples/apollo-vcp.yaml +++ b/klyshko-operator/config/samples/apollo-vcp.yaml @@ -9,4 +9,4 @@ kind: ConfigMap metadata: name: vcp-config data: - playerId: "0" \ No newline at end of file + playerId: "0" diff --git a/klyshko-operator/config/samples/engine-params-extra.yaml.template b/klyshko-operator/config/samples/engine-params-extra.yaml.template index 9261e25..a1b7f60 100644 --- a/klyshko-operator/config/samples/engine-params-extra.yaml.template +++ b/klyshko-operator/config/samples/engine-params-extra.yaml.template @@ -12,4 +12,4 @@ data: mac_key_share_p: | MAC_KEY_SHARE_P mac_key_share_2: | - MAC_KEY_SHARE_2 \ No newline at end of file + MAC_KEY_SHARE_2 diff --git a/klyshko-operator/config/samples/engine-params-secret.yaml.template b/klyshko-operator/config/samples/engine-params-secret.yaml.template index 1eaeba2..9b1cb97 100644 --- a/klyshko-operator/config/samples/engine-params-secret.yaml.template +++ b/klyshko-operator/config/samples/engine-params-secret.yaml.template @@ -13,4 +13,4 @@ data: mac_key_share_p: | MAC_KEY_SHARE_P mac_key_share_2: | - MAC_KEY_SHARE_2 \ No newline at end of file + MAC_KEY_SHARE_2 diff --git a/klyshko-operator/config/samples/engine-params.yaml b/klyshko-operator/config/samples/engine-params.yaml index c34336d..a2f5c8e 100644 --- a/klyshko-operator/config/samples/engine-params.yaml +++ b/klyshko-operator/config/samples/engine-params.yaml @@ -10,4 +10,4 @@ metadata: name: io.carbynestack.engine.params data: prime: | - 198766463529478683931867765928436695041 \ No newline at end of file + 198766463529478683931867765928436695041 diff --git a/klyshko-operator/config/samples/starbuck-vcp.yaml b/klyshko-operator/config/samples/starbuck-vcp.yaml index c3d8b2c..6a734ca 100644 --- a/klyshko-operator/config/samples/starbuck-vcp.yaml +++ b/klyshko-operator/config/samples/starbuck-vcp.yaml @@ -9,4 +9,4 @@ kind: ConfigMap metadata: name: vcp-config data: - playerId: "1" \ No newline at end of file + playerId: "1" diff --git a/klyshko-operator/create-sbom.sh b/klyshko-operator/create-sbom.sh index a485b02..0c8acd7 100755 --- a/klyshko-operator/create-sbom.sh +++ b/klyshko-operator/create-sbom.sh @@ -49,4 +49,4 @@ find vendor -type d | while IFS= read -r d; do fi ((POS++)) done -echo -e "\nDONE" \ No newline at end of file +echo -e "\nDONE" diff --git a/klyshko-operator/hack/boilerplate.go.txt b/klyshko-operator/hack/boilerplate.go.txt index 29c55ec..b7950f8 100644 --- a/klyshko-operator/hack/boilerplate.go.txt +++ b/klyshko-operator/hack/boilerplate.go.txt @@ -12,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ \ No newline at end of file +*/ diff --git a/klyshko-operator/hack/deploy.sh b/klyshko-operator/hack/deploy.sh index 87a54ff..59f4427 100755 --- a/klyshko-operator/hack/deploy.sh +++ b/klyshko-operator/hack/deploy.sh @@ -55,4 +55,4 @@ do if [ "$c" == "apollo" ]; then kubectl apply -f config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml fi -done \ No newline at end of file +done diff --git a/klyshko-provisioner/Dockerfile b/klyshko-provisioner/Dockerfile index 29773d4..0f4d6e6 100644 --- a/klyshko-provisioner/Dockerfile +++ b/klyshko-provisioner/Dockerfile @@ -19,4 +19,4 @@ RUN curl -o cs.jar -L https://github.com/carbynestack/cli/releases/download/$CLI # Copy resources COPY kii-provisioner-run.sh . -CMD ["/bin/bash", "-c", "./kii-provisioner-run.sh"] \ No newline at end of file +CMD ["/bin/bash", "-c", "./kii-provisioner-run.sh"] diff --git a/klyshko-provisioner/hack/deploy.sh b/klyshko-provisioner/hack/deploy.sh index 305b4f3..58a431b 100755 --- a/klyshko-provisioner/hack/deploy.sh +++ b/klyshko-provisioner/hack/deploy.sh @@ -18,4 +18,4 @@ for c in "${CLUSTERS[@]}" do echo -e "${GREEN}Loading docker image into $c${NC}" kind load docker-image carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT --name "$c" -done \ No newline at end of file +done From 52ccfbf030f618acbc7f225b6aa76058ac0305bb Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 24 Jun 2022 22:46:59 +0200 Subject: [PATCH 28/69] Implement improvements from review Signed-off-by: Sven Trieflinger --- .pre-commit-config.yaml | 4 + README.md | 80 +++++++++++++++++-- hack/deploy.sh | 19 ++++- klyshko-mp-spdz/README.md | 28 +++++++ klyshko-mp-spdz/hack/deploy.sh | 4 + klyshko-mp-spdz/kii-run.sh | 37 +++++---- klyshko-operator/Dockerfile | 2 + klyshko-operator/Makefile | 6 +- .../api/v1alpha1/tuplegenerationjob_types.go | 2 + .../manager/controller_manager_config.yaml | 2 +- .../config/samples/apollo-vcp.yaml | 3 +- .../samples/engine-params-extra.yaml.template | 4 +- .../config/samples/engine-params.yaml | 3 +- .../klyshko_v1alpha1_tuplegenerationjob.yaml | 4 +- ..._v1alpha1_tuplegenerationjob.yaml.template | 14 ---- .../klyshko_v1alpha1_tuplegenerationtask.yaml | 2 +- .../config/samples/starbuck-vcp.yaml | 3 +- klyshko-operator/controllers/castor.go | 20 +++-- klyshko-operator/controllers/config_utils.go | 61 ++++++++++---- klyshko-operator/controllers/etcd_keys.go | 39 ++++----- klyshko-operator/controllers/suite_test.go | 9 --- .../tuplegenerationjob_controller.go | 24 +++--- .../tuplegenerationscheduler_controller.go | 6 +- .../tuplegenerationtask_controller.go | 25 +++--- klyshko-operator/hack/deploy.sh | 54 ++++++++++--- klyshko-operator/main.go | 2 +- klyshko-provisioner/Dockerfile | 10 +-- klyshko-provisioner/hack/deploy.sh | 4 + klyshko-provisioner/kii-provisioner-run.sh | 4 +- 29 files changed, 333 insertions(+), 142 deletions(-) delete mode 100644 klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae7bafc..c3523ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,10 @@ repos: hooks: - id: go-fmt args: [-w] # Update source files directly + - repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.7.2 + hooks: + - id: shellcheck - repo: https://github.com/executablebooks/mdformat rev: 0.7.14 hooks: diff --git a/README.md b/README.md index 75e3dc3..1c4a8c5 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,71 @@ to orchestrate actions across VCPs. ## Usage -To deploy Klyshko to your VC you have to install the operator on all VCPs. You -then create a scheduler on **one** of the clusters by applying the respective -manifest, e.g., +To deploy Klyshko to your VC you have to perform the following steps: + +### Install the operator + +You can use the `make` tool to deploy the operator using + +```shell +make deploy IMG="carbynestack/klyshko-operator:v0.0.1" +``` + +Remember to do this on all VCPs of your VC. + +### Provide CRG Configuration + +CRGs require some configuration that has to be provided using K8s config maps +and secrets. + +Public parameters are provided in a config map with name +`io.carbynestack.engine.params` as follows: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params +data: + prime: <> +``` + +Sensitive parameters are provided using a K8s secret with name +`io.carbynestack.engine.params.secret` as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: io.carbynestack.engine.params.secret +type: Opaque +data: + mac_key_share_p: | + <> + mac_key_share_2: | + <> +``` + +Additional parameters _may_ be provided using a K8s config map with name +`io.carbynestack.engine.params.extra` as follows: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params.extra +data: + <>: <> + <>: <> +``` + +Please consult the documentation of the CRG you want to use to understand +whether any extra arguments are expected. + +### Instantiating a Scheduler + +After configuration is done, you create a scheduler on **one** of the clusters +by applying the respective manifest, e.g., ```yaml apiVersion: klyshko.carbnyestack.io/v1alpha1 @@ -75,8 +137,8 @@ spec: Klyshko will start producing correlated randomness by creating respective jobs whenever the number of tuples for a specific type drops below the given -`threshold`. `concurrency` specifies the maximum number of jobs are allowed to -run concurrently. This is the upper limit across all tuple types together. +`threshold`. `concurrency` specifies the maximum number of jobs that are allowed +to run concurrently. This is the upper limit across jobs for all tuple types. ## Klyshko Integration Interface (KII) @@ -138,6 +200,12 @@ The prime to be used for generating prime field tuples is provided in the file The MAC key shares for prime and binary fields are made available as files `mac_key_share_p` and `mac_key_share_2` in folder `/etc/kii/secret-params`. +### Additional Parameters + +Some CRGs might require additional _non-standard_ parameters. These are made +available by the Klyshko runtime in folder `/etc/kii/extra-params`. For an +example of how this is used see the [MP-SPDZ fake tuple CRG][mp-spdz-fake]. + ## Development The `deploy.sh` scripts in the `hack` folders (top-level and within modules) can @@ -162,3 +230,5 @@ the Carbyne Stack repository. Please see the Carbyne Stack [Contributor's Guide](https://github.com/carbynestack/carbynestack/blob/master/CONTRIBUTING.md) . + +[mp-spdz-fake]: klyshko-mp-spdz/README.md#foreign-mac-key-shares diff --git a/hack/deploy.sh b/hack/deploy.sh index 700af33..d314f3a 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -5,9 +5,24 @@ # # SPDX-License-Identifier: Apache-2.0 # + +# Fail, if any command fails +set -e + +# Make sure that the following versions "match", i.e., the charts application +# version matches ETCD_VERSION (see https://artifacthub.io/packages/helm/bitnami/etcd). +# Upgrading the Bitnami etcd chart does not work at the moment (see https://github.com/bitnami/bitnami-docker-etcd/pull/47). +# Hence, you have to delete the old chart manually when changing the chart +# version using +# +# helm uninstall test-etcd +# +export ETCD_VERSION=v3.5.4 +export ETCD_CHART_VERSION=8.3.1 + kubectl config use-context "kind-apollo" -helm repo add bitnami https://charts.bitnami.com/bitnami -helm install test-etcd --set auth.rbac.enabled=false --set service.type=LoadBalancer bitnami/etcd +helm repo add bitnami https://charts.bitnami.com/bitnami --force-update +helm install test-etcd --set auth.rbac.enabled=false --set service.type=LoadBalancer bitnami/etcd --version ${ETCD_CHART_VERSION} || true ( cd klyshko-mp-spdz diff --git a/klyshko-mp-spdz/README.md b/klyshko-mp-spdz/README.md index 5cc8bb0..2dac7f3 100644 --- a/klyshko-mp-spdz/README.md +++ b/klyshko-mp-spdz/README.md @@ -6,6 +6,33 @@ ability to generate fake tuples. For a high-level description of the Klyshko subsystem, its components, and how these interact, please see the [README] at the root of this repository. +## Foreign MAC Key Shares + +The MP-SPDZ fake tuple generator requires all MAC key shares to be available to +all parties. They are expected to be made available using the +[KII mechanism for providing extra parameters][kii-extra] using a config map +like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params.extra +data: + 1_mac_key_share_p: | + ... + 1_mac_key_share_2: | + ... + 2_mac_key_share_p: | + ... + 2_mac_key_share_2: | + ... +``` + +There must be entries for all VCPs except the local one (although it doesn't +hurt if that is also provided). The numeric prefix of the key specifies the +number (zero-based) of the VCP this key is for. + ## KII Tuple Type Mapping The mapping from [KII] tuple types to the flags required for invoking the @@ -25,4 +52,5 @@ The mapping from [KII] tuple types to the flags required for invoking the | MULTIPLICATION_TRIPLE_GF2N | --ntriples n,0 | 2-2-40/Triples-2-P0 | 34 | [kii]: ../README.md#klyshko-integration-interface-kii +[kii-extra]: ../README.md#additional-parameters [readme]: ../README.md diff --git a/klyshko-mp-spdz/hack/deploy.sh b/klyshko-mp-spdz/hack/deploy.sh index e5619a8..8c0b204 100755 --- a/klyshko-mp-spdz/hack/deploy.sh +++ b/klyshko-mp-spdz/hack/deploy.sh @@ -5,6 +5,10 @@ # # SPDX-License-Identifier: Apache-2.0 # + +# Fail, if any command fails +set -e + GREEN='\033[0;32m' NC='\033[0m' # No Color diff --git a/klyshko-mp-spdz/kii-run.sh b/klyshko-mp-spdz/kii-run.sh index 0e53b08..1b947d8 100755 --- a/klyshko-mp-spdz/kii-run.sh +++ b/klyshko-mp-spdz/kii-run.sh @@ -12,6 +12,7 @@ set -e # Setup offline executable command line arguments dictionary n=${KII_TUPLES_PER_JOB} pn=${KII_PLAYER_NUMBER} +pc=${KII_PLAYER_COUNT} declare -A argsByType=( ["BIT_GFP"]="--nbits 0,${n}" ["BIT_GF2N"]="--nbits ${n},0" @@ -24,20 +25,22 @@ declare -A argsByType=( ["MULTIPLICATION_TRIPLE_GFP"]="--ntriples 0,${n}" ["MULTIPLICATION_TRIPLE_GF2N"]="--ntriples ${n},0" ) -declare -A folderByType=( - ["BIT_GFP"]="2-p-128/Bits-p-P${pn}" - ["BIT_GF2N"]="2-2-40/Bits-2-P${pn}" - ["INPUT_MASK_GFP"]="2-p-128/Triples-p-P${pn}" - ["INPUT_MASK_GF2N"]="2-2-40/Triples-2-P${pn}" - ["INVERSE_TUPLE_GFP"]="2-p-128/Inverses-p-P${pn}" - ["INVERSE_TUPLE_GF2N"]="2-2-40/Inverses-2-P${pn}" - ["SQUARE_TUPLE_GFP"]="2-p-128/Squares-p-P${pn}" - ["SQUARE_TUPLE_GF2N"]="2-2-40/Squares-2-P${pn}" - ["MULTIPLICATION_TRIPLE_GFP"]="2-p-128/Triples-p-P${pn}" - ["MULTIPLICATION_TRIPLE_GF2N"]="2-2-40/Triples-2-P${pn}" +declare -A tupleFileByType=( + ["BIT_GFP"]="${pc}-p-128/Bits-p-P${pn}" + ["BIT_GF2N"]="${pc}-2-40/Bits-2-P${pn}" + ["INPUT_MASK_GFP"]="${pc}-p-128/Triples-p-P${pn}" + ["INPUT_MASK_GF2N"]="${pc}-2-40/Triples-2-P${pn}" + ["INVERSE_TUPLE_GFP"]="${pc}-p-128/Inverses-p-P${pn}" + ["INVERSE_TUPLE_GF2N"]="${pc}-2-40/Inverses-2-P${pn}" + ["SQUARE_TUPLE_GFP"]="${pc}-p-128/Squares-p-P${pn}" + ["SQUARE_TUPLE_GF2N"]="${pc}-2-40/Squares-2-P${pn}" + ["MULTIPLICATION_TRIPLE_GFP"]="${pc}-p-128/Triples-p-P${pn}" + ["MULTIPLICATION_TRIPLE_GF2N"]="${pc}-2-40/Triples-2-P${pn}" ) -# Provide required parameters in MP-SPDZ "Player-Data" folder +# Provide parameters in MP-SPDZ "Player-Data" folder. +# Note that we always provide parameters for both prime fields and fields of +# characteristic 2 for reasons of simplicity here. prime=$(cat /etc/kii/params/prime) declare fields=("p" "2") for f in "${fields[@]}" @@ -48,16 +51,16 @@ do mkdir -p "${folder}" echo "Providing parameters for field ${f}-${bit_width} in folder ${folder}" - # Write MAC key shares + # Write MAC key shares for all players as this is required by the fake + # offline phase implementation of MP-SPDZ for pn in $(seq 0 $((KII_PLAYER_COUNT-1))) do macKeyShareFile="${folder}/Player-MAC-Keys-${f}-P${pn}" if [[ ${pn} -eq ${KII_PLAYER_NUMBER} ]]; then - src="/etc/kii/secret-params" + macKeyShare=$(cat "/etc/kii/secret-params/mac_key_share_${f}") else - src="/etc/kii/extra-params" + macKeyShare=$(cat "/etc/kii/extra-params/${pn}_mac_key_share_${f}") fi - macKeyShare=$(cat "${src}/mac_key_share_${f}") echo "${KII_PLAYER_COUNT} ${macKeyShare}" > "${macKeyShareFile}" echo "MAC key share for player ${pn} written to ${macKeyShareFile}" done @@ -70,4 +73,4 @@ cmd="./Fake-Offline.x -d 0 --prime ${prime} --prngseed ${seed:0:16} ${argsByType eval "$cmd" # Copy generated tuples to path expected by KII -cp "Player-Data/${folderByType[${KII_TUPLE_TYPE}]}" "${KII_TUPLE_FILE}" +cp "Player-Data/${tupleFileByType[${KII_TUPLE_TYPE}]}" "${KII_TUPLE_FILE}" diff --git a/klyshko-operator/Dockerfile b/klyshko-operator/Dockerfile index dde286f..a0be8f1 100644 --- a/klyshko-operator/Dockerfile +++ b/klyshko-operator/Dockerfile @@ -1,3 +1,5 @@ +# Dockerfile scaffolded by the Operator SDK. + # Build the manager binary FROM golang:1.16 as builder diff --git a/klyshko-operator/Makefile b/klyshko-operator/Makefile index 65eef2c..3736dba 100644 --- a/klyshko-operator/Makefile +++ b/klyshko-operator/Makefile @@ -1,3 +1,5 @@ +# Makefile scaffolded by the Operator SDK. + # VERSION defines the project version for the bundle. # Update this value when you upgrade the version of your project. # To re-generate a bundle for another specific version without changing the standard setup, you can: @@ -110,14 +112,14 @@ install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~ $(KUSTOMIZE) build config/crd | kubectl apply -f - uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl delete -f - + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=true -f - deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/default | kubectl delete -f - + $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=true -f - CONTROLLER_GEN = $(shell pwd)/bin/controller-gen diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go index 484d40c..2dc8221 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -27,6 +27,8 @@ const ( JobFailed = "Failed" ) +// IsValid returns true if this state is among the defined ones and false +// otherwise. func (s TupleGenerationJobState) IsValid() bool { switch s { case JobPending, JobRunning, JobCompleted, JobFailed: diff --git a/klyshko-operator/config/manager/controller_manager_config.yaml b/klyshko-operator/config/manager/controller_manager_config.yaml index 3eb1d02..e65099a 100644 --- a/klyshko-operator/config/manager/controller_manager_config.yaml +++ b/klyshko-operator/config/manager/controller_manager_config.yaml @@ -8,4 +8,4 @@ webhook: port: 9443 leaderElection: leaderElect: true - resourceName: 02e9e224.carbnyestack.io + resourceName: operator.klyshko.carbynestack.io diff --git a/klyshko-operator/config/samples/apollo-vcp.yaml b/klyshko-operator/config/samples/apollo-vcp.yaml index f3ed972..7f6a58e 100644 --- a/klyshko-operator/config/samples/apollo-vcp.yaml +++ b/klyshko-operator/config/samples/apollo-vcp.yaml @@ -7,6 +7,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: vcp-config + name: cs-vcp-config data: + playerCount: "2" playerId: "0" diff --git a/klyshko-operator/config/samples/engine-params-extra.yaml.template b/klyshko-operator/config/samples/engine-params-extra.yaml.template index a1b7f60..7ee8fc5 100644 --- a/klyshko-operator/config/samples/engine-params-extra.yaml.template +++ b/klyshko-operator/config/samples/engine-params-extra.yaml.template @@ -9,7 +9,7 @@ kind: ConfigMap metadata: name: io.carbynestack.engine.params.extra data: - mac_key_share_p: | + PLAYER_ID_mac_key_share_p: | MAC_KEY_SHARE_P - mac_key_share_2: | + PLAYER_ID_mac_key_share_2: | MAC_KEY_SHARE_2 diff --git a/klyshko-operator/config/samples/engine-params.yaml b/klyshko-operator/config/samples/engine-params.yaml index a2f5c8e..e65f717 100644 --- a/klyshko-operator/config/samples/engine-params.yaml +++ b/klyshko-operator/config/samples/engine-params.yaml @@ -9,5 +9,4 @@ kind: ConfigMap metadata: name: io.carbynestack.engine.params data: - prime: | - 198766463529478683931867765928436695041 + prime: "198766463529478683931867765928436695041" diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index fef196b..c022a57 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -7,8 +7,8 @@ apiVersion: klyshko.carbnyestack.io/v1alpha1 kind: TupleGenerationJob metadata: - name: tuplegenerationjob-sample + name: tuplegenerationjob-sample-0f990412-e8a0-4ef3-8e3f-a6a3a9191948 spec: - id: 9fc7f5b7-09d9-4c62-97c7-2d7eeddfb858 + id: 0f990412-e8a0-4ef3-8e3f-a6a3a9191948 type: MULTIPLICATION_TRIPLE_GFP count: 10000 diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template deleted file mode 100644 index 964e8f1..0000000 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml.template +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) 2022 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. -# -# SPDX-License-Identifier: Apache-2.0 -# -apiVersion: klyshko.carbnyestack.io/v1alpha1 -kind: TupleGenerationJob -metadata: - name: tuplegenerationjob-sample -spec: - id: JOB_ID - type: multiplicationtriple_gfp - count: 100000 diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml index ec571b8..13a76ed 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationtask.yaml @@ -7,5 +7,5 @@ apiVersion: klyshko.carbnyestack.io/v1alpha1 kind: TupleGenerationTask metadata: - name: tuplegenerationtask-sample + name: tuplegenerationtask-sample-0f990412-e8a0-4ef3-8e3f-a6a3a9191948-0 spec: diff --git a/klyshko-operator/config/samples/starbuck-vcp.yaml b/klyshko-operator/config/samples/starbuck-vcp.yaml index 6a734ca..c957275 100644 --- a/klyshko-operator/config/samples/starbuck-vcp.yaml +++ b/klyshko-operator/config/samples/starbuck-vcp.yaml @@ -7,6 +7,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: vcp-config + name: cs-vcp-config data: + playerCount: "2" playerId: "1" diff --git a/klyshko-operator/controllers/castor.go b/klyshko-operator/controllers/castor.go index e53a093..38102f3 100644 --- a/klyshko-operator/controllers/castor.go +++ b/klyshko-operator/controllers/castor.go @@ -10,8 +10,10 @@ package controllers import ( "context" "encoding/json" + "errors" "fmt" "github.com/google/uuid" + "io" "io/ioutil" "net/http" "sigs.k8s.io/controller-runtime/pkg/log" @@ -22,7 +24,7 @@ func activateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { client := &http.Client{} url := fmt.Sprintf("http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", chunkID) // TODO Make servername configurable / use discovery logger.Info("activating tuple chunk with castor URL", "URL", url) - req, err := http.NewRequest(http.MethodPut, url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, nil) if err != nil { return err } @@ -30,10 +32,13 @@ func activateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { if err != nil { return err } + if resp.StatusCode != http.StatusOK { + return errors.New(fmt.Sprintf("received response with status code %d", resp.StatusCode)) + } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - logger.Info("response from castor", "Status", resp.Status, "Body", body) // TODO Body is Base64 encoded :-( - return err + defer io.Copy(ioutil.Discard, resp.Body) + logger.Info("response from castor", "Status", resp.Status) + return nil } type TupleMetrics struct { @@ -51,7 +56,7 @@ func getTelemetry(ctx context.Context) (Telemetry, error) { client := &http.Client{} // Building the request - req, err := http.NewRequest("GET", "http://cs-castor.default.svc.cluster.local:10100/intra-vcp/telemetry", nil) // TODO Make servername configurable / use discovery + req, err := http.NewRequestWithContext(ctx, "GET", "http://cs-castor.default.svc.cluster.local:10100/intra-vcp/telemetry", nil) // TODO Make servername configurable / use discovery if err != nil { logger.Error(err, "failed to build request for castor telemetry data") return Telemetry{}, err @@ -65,6 +70,11 @@ func getTelemetry(ctx context.Context) (Telemetry, error) { logger.Error(err, "failed to fetch castor telemetry data") return Telemetry{}, err } + if resp.StatusCode != http.StatusOK { + err := errors.New(fmt.Sprintf("received response with status code %d", resp.StatusCode)) + logger.Error(err, "failed to fetch castor telemetry data") + return Telemetry{}, err + } // Read and parse response defer resp.Body.Close() diff --git a/klyshko-operator/controllers/config_utils.go b/klyshko-operator/controllers/config_utils.go index c4c1950..a1c02e9 100644 --- a/klyshko-operator/controllers/config_utils.go +++ b/klyshko-operator/controllers/config_utils.go @@ -18,30 +18,59 @@ import ( "strconv" ) -func localPlayerID(ctx context.Context, client *client.Client, namespace string) (uint, error) { +const VcpConfigMapName = "cs-vcp-config" - // Get VCP configuration config map +func getVcpConfig(ctx context.Context, client *client.Client, namespace string) (v1.ConfigMap, error) { name := types.NamespacedName{ Namespace: namespace, - Name: "vcp-config", + Name: VcpConfigMapName, } - cfm := &v1.ConfigMap{} - err := (*client).Get(ctx, name, cfm) + cfm := v1.ConfigMap{} + err := (*client).Get(ctx, name, &cfm) if err != nil { - return 0, errors.Unwrap(fmt.Errorf("VCP configuration not found: %w", err)) + return cfm, errors.Unwrap(fmt.Errorf("VCP configuration not found: %w", err)) + } + return cfm, nil +} + +func localPlayerID(ctx context.Context, client *client.Client, namespace string) (uint, error) { + cfm, err := getVcpConfig(ctx, client, namespace) + if err != nil { + return 0, err } // Extract playerId - if playerID, ok := cfm.Data["playerId"]; ok { - pid, err := strconv.Atoi(playerID) - if err != nil { - return 0, err - } - if pid < 0 || pid > math.MaxUint32 { - return 0, fmt.Errorf("invalid playerId '%d'- must be in range [0,%d]", pid, math.MaxUint32) - } - return uint(pid), nil - } else { + playerIDStr, ok := cfm.Data["playerId"] + if !ok { return 0, errors.New("invalid VCP configuration - missing playerId") } + playerID, err := strconv.Atoi(playerIDStr) + if err != nil { + return 0, err + } + if playerID < 0 || playerID > math.MaxUint32 { + return 0, fmt.Errorf("invalid playerId '%d'- must be in range [0,%d]", playerID, math.MaxUint32) + } + return uint(playerID), nil +} + +func numberOfPlayers(ctx context.Context, client *client.Client, namespace string) (uint, error) { + cfm, err := getVcpConfig(ctx, client, namespace) + if err != nil { + return 0, err + } + + // Extract playerCount + playerCountStr, ok := cfm.Data["playerCount"] + if !ok { + return 0, errors.New("invalid VCP configuration - missing playerCount") + } + playerCount, err := strconv.Atoi(playerCountStr) + if err != nil { + return 0, err + } + if playerCount < 0 || playerCount > math.MaxUint32 { + return 0, fmt.Errorf("invalid playerCount '%d'- must be in range [0,%d]", playerCount, math.MaxUint32) + } + return uint(playerCount), nil } diff --git a/klyshko-operator/controllers/etcd_keys.go b/klyshko-operator/controllers/etcd_keys.go index c2cb512..380095a 100644 --- a/klyshko-operator/controllers/etcd_keys.go +++ b/klyshko-operator/controllers/etcd_keys.go @@ -15,6 +15,8 @@ import ( "strconv" ) +const rosterKey = "/klyshko/roster" + type Key interface { ToEtcdKey() string } @@ -24,7 +26,7 @@ type RosterKey struct { } func (k RosterKey) ToEtcdKey() string { - return rosterKey + "/" + k.Namespace + "/" + k.Name + return fmt.Sprintf("%s/%s/%s", rosterKey, k.Namespace, k.Name) } func (k RosterKey) String() string { @@ -37,15 +39,13 @@ type RosterEntryKey struct { } func (k RosterEntryKey) ToEtcdKey() string { - return rosterKey + "/" + k.Namespace + "/" + k.Name + "/" + strconv.Itoa(int(k.PlayerID)) + return fmt.Sprintf("%s/%d", k.RosterKey.ToEtcdKey(), k.PlayerID) } func (k RosterEntryKey) String() string { return k.ToEtcdKey() } -const rosterKey = "/klyshko/roster" - var etcdRosterKeyPattern = regexp.MustCompile("^" + rosterKey + "/(?P(\\w|-)+)/(?P(\\w|-)+)(?:/(?P\\d+))?$") func etcdKeyParts(s string) map[string]string { @@ -68,22 +68,23 @@ func ParseKey(s string) (Key, error) { return nil, fmt.Errorf("not a key: %v", s) } name := types.NamespacedName{Name: parts["jobName"], Namespace: parts["namespace"]} - if playerID, ok := parts["localPlayerID"]; ok { - pid, err := strconv.Atoi(playerID) - if err != nil { - return nil, fmt.Errorf("invalid playerId '%v' - not an integer: %w", playerID, err) - } - if pid < 0 || pid > math.MaxUint32 { - return nil, fmt.Errorf("invalid playerId '%d' - must be in range [0,%d]", pid, math.MaxUint32) - } - return RosterEntryKey{ - RosterKey: RosterKey{ - name, - }, - PlayerID: uint(pid), + playerID, ok := parts["localPlayerID"] + if !ok { + return RosterKey{ + name, }, nil } - return RosterKey{ - name, + pid, err := strconv.Atoi(playerID) + if err != nil { + return nil, fmt.Errorf("invalid playerId '%v' - not an integer: %w", playerID, err) + } + if pid < 0 || pid > math.MaxUint32 { + return nil, fmt.Errorf("invalid playerId '%d' - must be in range [0,%d]", pid, math.MaxUint32) + } + return RosterEntryKey{ + RosterKey: RosterKey{ + name, + }, + PlayerID: uint(pid), }, nil } diff --git a/klyshko-operator/controllers/suite_test.go b/klyshko-operator/controllers/suite_test.go index 212644f..cbf4005 100644 --- a/klyshko-operator/controllers/suite_test.go +++ b/klyshko-operator/controllers/suite_test.go @@ -56,15 +56,6 @@ var _ = BeforeSuite(func() { err = klyshkov1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - err = klyshkov1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - err = klyshkov1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - err = klyshkov1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 429e9f4..734647f 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -34,11 +34,12 @@ type TupleGenerationJobReconciler struct { } func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Scheme, etcdClient *clientv3.Client) *TupleGenerationJobReconciler { - r := new(TupleGenerationJobReconciler) - r.Client = client - r.Scheme = scheme - r.EtcdClient = etcdClient - r.rosterWatcherCh = etcdClient.Watch(context.Background(), rosterKey, clientv3.WithPrefix()) // TODO Close channel? + r := &TupleGenerationJobReconciler{ + Client: client, + Scheme: scheme, + EtcdClient: etcdClient, + rosterWatcherCh: etcdClient.Watch(context.Background(), rosterKey, clientv3.WithPrefix()), // TODO Close channel? + } go r.handleWatchEvents() return r } @@ -89,7 +90,7 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key return } // TODO Create or update depending on whether Job already exists - err = r.createJob(ctx, key.NamespacedName, jobSpec) + err = r.createJobIfNotExists(ctx, key.NamespacedName, jobSpec) if err != nil { logger.Error(err, "failed to create job") return @@ -101,7 +102,6 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key err := r.Client.Get(ctx, key.NamespacedName, found) if err != nil { if apierrors.IsNotFound(err) { - logger.Error(err, "job does not exist - ignoring") return } logger.Error(err, "failed to read job resource") @@ -172,7 +172,6 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex err = r.Get(ctx, taskName, task) if err != nil { if apierrors.IsNotFound(err) { - logger.Error(err, "proxy task does not exist - ignoring") return } logger.Error(err, "failed to read proxy task resource") @@ -187,7 +186,7 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex } } -func (r *TupleGenerationJobReconciler) createJob(ctx context.Context, name types.NamespacedName, jobSpec *klyshkov1alpha1.TupleGenerationJobSpec) error { +func (r *TupleGenerationJobReconciler) createJobIfNotExists(ctx context.Context, name types.NamespacedName, jobSpec *klyshkov1alpha1.TupleGenerationJobSpec) error { logger := log.FromContext(ctx).WithValues("Job.Name", name) found := &klyshkov1alpha1.TupleGenerationJob{} err := r.Client.Get(ctx, name, found) @@ -336,8 +335,13 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R } return false } + nbrOfPlayers, err := numberOfPlayers(ctx, &r.Client, req.Namespace) + if err != nil { + logger.Error(err, "can't read playerCount from VCP configuration") + return ctrl.Result{RequeueAfter: 60 * time.Second}, err + } var state klyshkov1alpha1.TupleGenerationJobState - if len(ownedBy) < 2 { + if uint(len(ownedBy)) < nbrOfPlayers { state = klyshkov1alpha1.JobPending } else if !allTerminated(ownedBy) { state = klyshkov1alpha1.JobRunning diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 0787d0c..eeb844d 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -23,6 +23,8 @@ import ( klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" ) +const MinimumTuplesPerJob = 10000 + type TupleGenerationSchedulerReconciler struct { client.Client Scheme *runtime.Scheme @@ -81,7 +83,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req } } logger.Info("with in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) - belowThreshold := make([]TupleMetrics, 0) + var belowThreshold []TupleMetrics for _, m := range telemetry.TupleMetrics { if m.Available < scheduler.Spec.Threshold { belowThreshold = append(belowThreshold, m) @@ -111,7 +113,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req Spec: klyshkov1alpha1.TupleGenerationJobSpec{ ID: jobID, Type: belowThreshold[0].TupleType, - Count: 10000, + Count: MinimumTuplesPerJob, // TODO Make this configurable }, } err = ctrl.SetControllerReference(scheduler, job, r.Scheme) diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index d0fd89e..20970b9 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -179,7 +179,7 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. if status.State == klyshkov1alpha1.TaskLaunching { // Create persistent volume claim used to store generated tuples, if not existing - err = r.createPVC(ctx, *taskKey) + err = r.createPVC(ctx, taskKey) if err != nil { return ctrl.Result{}, err } @@ -244,10 +244,14 @@ func (r *TupleGenerationTaskReconciler) SetupWithManager(mgr ctrl.Manager) error Complete(r) } -func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key RosterEntryKey) error { +func pvcName(key RosterEntryKey) string { + return key.Name + "-" + strconv.Itoa(int(key.PlayerID)) +} + +func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key *RosterEntryKey) error { logger := log.FromContext(ctx).WithValues("Task.Key", key) name := types.NamespacedName{ - Name: key.Name + "-" + strconv.Itoa(int(key.PlayerID)), // TODO Error-prone, + Name: pvcName(*key), Namespace: key.Namespace, } found := &v1.PersistentVolumeClaim{} @@ -281,9 +285,13 @@ func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key Roste return nil } +func provisionerPodName(key RosterEntryKey) string { + return key.Name + "-provisioner" +} + func (r *TupleGenerationTaskReconciler) getProvisionerPod(ctx context.Context, key RosterEntryKey) (*v1.Pod, error) { name := types.NamespacedName{ - Name: key.Name + "-provisioner", + Name: provisionerPodName(key), Namespace: key.Namespace, } found := &v1.Pod{} @@ -294,7 +302,7 @@ func (r *TupleGenerationTaskReconciler) getProvisionerPod(ctx context.Context, k func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context, key RosterEntryKey, job *klyshkov1alpha1.TupleGenerationJob, task *klyshkov1alpha1.TupleGenerationTask) (*v1.Pod, error) { logger := log.FromContext(ctx).WithValues("Task.Key", key) name := types.NamespacedName{ - Name: key.Name + "-provisioner", + Name: provisionerPodName(key), Namespace: key.Namespace, } found, err := r.getProvisionerPod(ctx, key) @@ -338,7 +346,7 @@ func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context Name: "kii", VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: key.Name + "-" + strconv.Itoa(int(key.PlayerID)), // TODO Error-prone + ClaimName: pvcName(key), }, }, }, @@ -379,9 +387,6 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, ObjectMeta: metav1.ObjectMeta{ Name: task.Name, Namespace: task.Namespace, - //Finalizers: []string{ - // "klyshko.carbnyestack.io/provision-tuples", - //}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ @@ -454,7 +459,7 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, Name: "kii", VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: task.Name, + ClaimName: pvcName(key), }, }, }, diff --git a/klyshko-operator/hack/deploy.sh b/klyshko-operator/hack/deploy.sh index 59f4427..1d8225f 100755 --- a/klyshko-operator/hack/deploy.sh +++ b/klyshko-operator/hack/deploy.sh @@ -5,9 +5,36 @@ # # SPDX-License-Identifier: Apache-2.0 # + +# Fail, if any command fails +set -e + GREEN='\033[0;32m' NC='\033[0m' # No Color +# Makes etcdctl with version specified as environment variable ETCD_VERSION available in ./bin folder +function provide_etcdctl() { + REQUIRED_VERSION=$1 + echo "Checking for etcdctl ${REQUIRED_VERSION} in $(pwd)/bin" + if [[ -f "bin/etcdctl" ]] + then + INSTALLED_VERSION="v$(bin/etcdctl version | head -1 | cut -c 18-)" + if [[ ${INSTALLED_VERSION} != "${REQUIRED_VERSION}" ]] + then + echo "Removing version ${INSTALLED_VERSION}" + rm bin/etcdctl + else + return 0 + fi + fi + echo "Downloading version ${REQUIRED_VERSION}" + DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download + rm -f "/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" + curl -L "${DOWNLOAD_URL}/${REQUIRED_VERSION}/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" -o "/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" + tar --extract --file="/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" -C bin/ "etcd-${REQUIRED_VERSION}-linux-amd64/etcdctl" --strip-components=1 + rm -f "/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" +} + declare -a CLUSTERS=("starbuck" "apollo") echo -e "${GREEN}Undeploying from clusters${NC}" @@ -15,15 +42,13 @@ for c in "${CLUSTERS[@]}" do echo -e "${GREEN}Undeploying from $c${NC}" kubectl config use-context "kind-$c" - if [ "$c" == "apollo" ]; then - kubectl delete -f config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml - fi - kubectl delete --all tgj - kubectl delete --all tgt + kubectl delete --all tuplegenerationjobs + kubectl delete --all tuplegenerationtasks make undeploy IMG="carbynestack/klyshko-operator:v0.0.1" done echo -e "${GREEN}Cleaning up etcd${NC}" +provide_etcdctl "${ETCD_VERSION}" bin/etcdctl --endpoints 172.18.1.129:2379 del "/klyshko/roster" --prefix echo -e "${GREEN}Building code and image${NC}" @@ -41,12 +66,21 @@ do echo -e "${GREEN}Deploying in $c${NC}" kubectl config use-context "kind-$c" kubectl apply -f "config/samples/$c-vcp.yaml" - MAC_KEY_SHARE_P=$([ "$c" == "apollo" ] && echo "-88222337191559387830816715872691188861" | base64 || echo "1113507028231509545156335486838233835" | base64) - MAC_KEY_SHARE_2=$([ "$c" == "apollo" ] && echo "f0cf6099e629fd0bda2de3f9515ab72b" | base64 || echo "c347ce3d9e165e4e85221f9da7591d98" | base64) + if [ "$c" == "apollo" ]; then + MAC_KEY_SHARE_P=$(echo "-88222337191559387830816715872691188861" | base64) + MAC_KEY_SHARE_2=$(echo "f0cf6099e629fd0bda2de3f9515ab72b" | base64) + EXTRA_MAC_KEY_SHARE_P="1113507028231509545156335486838233835" + EXTRA_MAC_KEY_SHARE_2="c347ce3d9e165e4e85221f9da7591d98" + OTHER_PLAYER_ID=1 + else + MAC_KEY_SHARE_P=$(echo "1113507028231509545156335486838233835" | base64) + MAC_KEY_SHARE_2=$(echo "c347ce3d9e165e4e85221f9da7591d98" | base64) + EXTRA_MAC_KEY_SHARE_P="-88222337191559387830816715872691188861" + EXTRA_MAC_KEY_SHARE_2="f0cf6099e629fd0bda2de3f9515ab72b" + OTHER_PLAYER_ID=0 + fi sed -e "s/MAC_KEY_SHARE_P/${MAC_KEY_SHARE_P}/" -e "s/MAC_KEY_SHARE_2/${MAC_KEY_SHARE_2}/" config/samples/engine-params-secret.yaml.template > "/tmp/$c-engine-params-secret.yaml" - EXTRA_MAC_KEY_SHARE_P=$([ "$c" == "starbuck" ] && echo "-88222337191559387830816715872691188861" || echo "1113507028231509545156335486838233835") - EXTRA_MAC_KEY_SHARE_2=$([ "$c" == "starbuck" ] && echo "f0cf6099e629fd0bda2de3f9515ab72b" || echo "c347ce3d9e165e4e85221f9da7591d98") - sed -e "s/MAC_KEY_SHARE_P/${EXTRA_MAC_KEY_SHARE_P}/" -e "s/MAC_KEY_SHARE_2/${EXTRA_MAC_KEY_SHARE_2}/" config/samples/engine-params-extra.yaml.template > "/tmp/$c-engine-params-extra.yaml" + sed -e "s/MAC_KEY_SHARE_P/${EXTRA_MAC_KEY_SHARE_P}/" -e "s/MAC_KEY_SHARE_2/${EXTRA_MAC_KEY_SHARE_2}/" -e "s/PLAYER_ID/${OTHER_PLAYER_ID}/" config/samples/engine-params-extra.yaml.template > "/tmp/$c-engine-params-extra.yaml" kubectl apply -f "/tmp/$c-engine-params-secret.yaml" kubectl apply -f "/tmp/$c-engine-params-extra.yaml" kubectl apply -f config/samples/engine-params.yaml diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index 803c6a5..dc8058c 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -64,7 +64,7 @@ func main() { Port: 9443, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, - LeaderElectionID: "02e9e224.carbnyestack.io", + LeaderElectionID: "operator.klyshko.carbynestack.io", }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/klyshko-provisioner/Dockerfile b/klyshko-provisioner/Dockerfile index 0f4d6e6..e2d70fe 100644 --- a/klyshko-provisioner/Dockerfile +++ b/klyshko-provisioner/Dockerfile @@ -6,15 +6,9 @@ # FROM ghcr.io/carbynestack/openjdk:jre8-20210827 -ARG CLI_VERSION=0.2-SNAPSHOT-2336890983-14-a4260ab - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gettext-base \ - curl \ - && rm -rf /var/lib/apt/lists/* - # Download Carbyne Stack CLI Jar at specified version -RUN curl -o cs.jar -L https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-$CLI_VERSION-jar-with-dependencies.jar +ARG CLI_VERSION=0.2-SNAPSHOT-2336890983-14-a4260ab +ADD https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-$CLI_VERSION-jar-with-dependencies.jar cs.jar # Copy resources COPY kii-provisioner-run.sh . diff --git a/klyshko-provisioner/hack/deploy.sh b/klyshko-provisioner/hack/deploy.sh index 58a431b..ae2b9e9 100755 --- a/klyshko-provisioner/hack/deploy.sh +++ b/klyshko-provisioner/hack/deploy.sh @@ -5,6 +5,10 @@ # # SPDX-License-Identifier: Apache-2.0 # + +# Fail, if any command fails +set -e + GREEN='\033[0;32m' NC='\033[0m' # No Color diff --git a/klyshko-provisioner/kii-provisioner-run.sh b/klyshko-provisioner/kii-provisioner-run.sh index 8a328a0..a1b8e24 100755 --- a/klyshko-provisioner/kii-provisioner-run.sh +++ b/klyshko-provisioner/kii-provisioner-run.sh @@ -14,6 +14,7 @@ cat < cs-config { "prime" : 0, "r" : 0, + "rinv" : 0, "noSslValidation" : true, "trustedCertificates" : [ ], "providers" : [ { @@ -28,8 +29,7 @@ cat < cs-config "ephemeralServiceUrl" : "http://ignore", "id" : 2, "baseUrl" : "http://ignore" - } ], - "rinv" : 0 + } ] } EOF From 885c15eae0b55ffa02ce42a8eefe819c617c5228 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 1 Jul 2022 17:21:30 +0200 Subject: [PATCH 29/69] Add tests Signed-off-by: Sven Trieflinger --- klyshko-operator/Makefile | 2 +- .../{castor.go => castor_client.go} | 58 ++++- .../controllers/castor_client_test.go | 135 ++++++++++ .../controllers/config_utils_test.go | 176 +++++++++++++ .../controllers/controller_test.go | 243 ++++++++++++++++++ .../controllers/etcd_keys_test.go | 63 +++++ klyshko-operator/controllers/suite_test.go | 60 +---- .../tuplegenerationjob_controller.go | 6 +- .../tuplegenerationscheduler_controller.go | 5 +- klyshko-operator/go.mod | 5 +- klyshko-operator/go.sum | 38 ++- klyshko-operator/main.go | 16 +- 12 files changed, 723 insertions(+), 84 deletions(-) rename klyshko-operator/controllers/{castor.go => castor_client.go} (61%) create mode 100644 klyshko-operator/controllers/castor_client_test.go create mode 100644 klyshko-operator/controllers/config_utils_test.go create mode 100644 klyshko-operator/controllers/controller_test.go create mode 100644 klyshko-operator/controllers/etcd_keys_test.go diff --git a/klyshko-operator/Makefile b/klyshko-operator/Makefile index 3736dba..7e1e9ad 100644 --- a/klyshko-operator/Makefile +++ b/klyshko-operator/Makefile @@ -90,7 +90,7 @@ vet: ## Run go vet against code. go vet ./... test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" GINKGO_EDITOR_INTEGRATION=true go test ./... -coverprofile cover.out ##@ Build diff --git a/klyshko-operator/controllers/castor.go b/klyshko-operator/controllers/castor_client.go similarity index 61% rename from klyshko-operator/controllers/castor.go rename to klyshko-operator/controllers/castor_client.go index 38102f3..78c8e61 100644 --- a/klyshko-operator/controllers/castor.go +++ b/klyshko-operator/controllers/castor_client.go @@ -19,24 +19,43 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -func activateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { +type CastorClient struct { + Url string + client *http.Client +} + +func NewCastorClient(url string) *CastorClient { + return &CastorClient{ + Url: url, + client: &http.Client{}, + } +} + +func (castorClient CastorClient) activateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkID) - client := &http.Client{} - url := fmt.Sprintf("http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", chunkID) // TODO Make servername configurable / use discovery + url := fmt.Sprintf("%s/intra-vcp/tuple-chunks/activate/%s", castorClient.Url, chunkID) logger.Info("activating tuple chunk with castor URL", "URL", url) req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, nil) if err != nil { return err } - resp, err := client.Do(req) + resp, err := castorClient.client.Do(req) if err != nil { return err } if resp.StatusCode != http.StatusOK { return errors.New(fmt.Sprintf("received response with status code %d", resp.StatusCode)) } - defer resp.Body.Close() - defer io.Copy(ioutil.Discard, resp.Body) + defer func() { + _, err := io.Copy(ioutil.Discard, resp.Body) + if err != nil { + logger.Error(err, "failed to discard response from castor") + } + err = resp.Body.Close() + if err != nil { + logger.Error(err, "failed to close response from castor") + } + }() logger.Info("response from castor", "Status", resp.Status) return nil } @@ -51,12 +70,16 @@ type Telemetry struct { TupleMetrics []TupleMetrics `json:"metrics"` } -func getTelemetry(ctx context.Context) (Telemetry, error) { +func (castorClient CastorClient) getTelemetry(ctx context.Context) (Telemetry, error) { logger := log.FromContext(ctx) - client := &http.Client{} // Building the request - req, err := http.NewRequestWithContext(ctx, "GET", "http://cs-castor.default.svc.cluster.local:10100/intra-vcp/telemetry", nil) // TODO Make servername configurable / use discovery + req, err := http.NewRequestWithContext( + ctx, + "GET", + fmt.Sprintf("%s/intra-vcp/telemetry", castorClient.Url), + nil, + ) if err != nil { logger.Error(err, "failed to build request for castor telemetry data") return Telemetry{}, err @@ -65,7 +88,7 @@ func getTelemetry(ctx context.Context) (Telemetry, error) { req.Header.Add("Content-Type", "application/json") // Doing the request - resp, err := client.Do(req) + resp, err := castorClient.client.Do(req) if err != nil { logger.Error(err, "failed to fetch castor telemetry data") return Telemetry{}, err @@ -76,15 +99,22 @@ func getTelemetry(ctx context.Context) (Telemetry, error) { return Telemetry{}, err } - // Read and parse response - defer resp.Body.Close() + // Read, parse, and return telemetry response + defer func() { + err := resp.Body.Close() + if err != nil { + logger.Error(err, "failed to close response from castor") + } + }() bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { logger.Error(err, "failed to read response body") return Telemetry{}, err } var response Telemetry - json.Unmarshal(bodyBytes, &response) - + err = json.Unmarshal(bodyBytes, &response) + if err != nil { + return Telemetry{}, err + } return response, nil } diff --git a/klyshko-operator/controllers/castor_client_test.go b/klyshko-operator/controllers/castor_client_test.go new file mode 100644 index 0000000..05c8fe4 --- /dev/null +++ b/klyshko-operator/controllers/castor_client_test.go @@ -0,0 +1,135 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "fmt" + "github.com/google/uuid" + "github.com/jarcoal/httpmock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const validCastorUrl = "http://cs-castor.default.svc.cluster.local:10100" +const invalidCastorUrl = "http://cs-castor.default.svc.cluster.local:10101" + +var _ = Describe("Fetching telemetry", func() { + + ctx := context.TODO() + + AfterEach(func() { + httpmock.DeactivateAndReset() + }) + + When("when Castor services responds with expected JSON data", func() { + + expectedTelemetry := Telemetry{TupleMetrics: []TupleMetrics{ + { + Available: 0, + ConsumptionRate: 0, + TupleType: "MULTIPLICATION_TRIPLE_GFP", + }, + }} + + BeforeEach(func() { + httpmock.Activate() + responder, err := httpmock.NewJsonResponder(200, expectedTelemetry) + Expect(err).NotTo(HaveOccurred()) + httpmock.RegisterResponder( + "GET", + fmt.Sprintf("=~^%s/intra-vcp/telemetry", validCastorUrl), + responder, + ) + }) + It("succeeds", func() { + castorClient := NewCastorClient(validCastorUrl) + telemetry, err := castorClient.getTelemetry(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(telemetry).To(Equal(expectedTelemetry)) + }) + }) + + When("when Castor services responds with non-success status code", func() { + BeforeEach(func() { + httpmock.Activate() + httpmock.RegisterResponder( + "GET", + fmt.Sprintf("=~^%s/intra-vcp/telemetry", validCastorUrl), + httpmock.NewStringResponder(404, ""), + ) + }) + It("fails", func() { + castorClient := NewCastorClient(validCastorUrl) + _, err := castorClient.getTelemetry(ctx) + Expect(err).To(HaveOccurred()) + }) + }) + + When("when Castor service is not available", func() { + It("fails", func() { + castorClient := NewCastorClient(invalidCastorUrl) + _, err := castorClient.getTelemetry(ctx) + Expect(err).To(HaveOccurred()) + }) + }) + +}) + +var _ = Describe("Activating a tuple chunk", func() { + + ctx := context.TODO() + + AfterEach(func() { + httpmock.DeactivateAndReset() + }) + + When("when Castor services responds with success status code", func() { + BeforeEach(func() { + httpmock.Activate() + httpmock.RegisterResponder( + "PUT", + fmt.Sprintf("=~^%s/intra-vcp/tuple-chunks/activate/.*", validCastorUrl), + httpmock.NewStringResponder(200, ""), + ) + }) + It("succeeds", func() { + chunkId := uuid.New() + castorClient := NewCastorClient(validCastorUrl) + err := castorClient.activateTupleChunk(ctx, chunkId) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + When("when Castor services responds with bon-success status code", func() { + BeforeEach(func() { + httpmock.Activate() + httpmock.RegisterResponder( + "PUT", + fmt.Sprintf("=~^%s/intra-vcp/tuple-chunks/activate/.*", validCastorUrl), + httpmock.NewStringResponder(404, ""), + ) + }) + It("fails", func() { + chunkId := uuid.New() + castorClient := NewCastorClient(validCastorUrl) + err := castorClient.activateTupleChunk(ctx, chunkId) + Expect(err).To(HaveOccurred()) + }) + }) + + When("when Castor service is not available", func() { + It("fails", func() { + chunkId := uuid.New() + castorClient := NewCastorClient(invalidCastorUrl) + err := castorClient.activateTupleChunk(ctx, chunkId) + Expect(err).To(HaveOccurred()) + }) + }) + +}) diff --git a/klyshko-operator/controllers/config_utils_test.go b/klyshko-operator/controllers/config_utils_test.go new file mode 100644 index 0000000..9898265 --- /dev/null +++ b/klyshko-operator/controllers/config_utils_test.go @@ -0,0 +1,176 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + //+kubebuilder:scaffold:imports +) + +var _ = Context("TODO", func() { // TODO Description? + + ctx := context.Background() + var vcp *Vcp + + BeforeEach(func() { + var err error + vcp, err = setupVcp() + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := vcp.tearDownVcp() + Expect(err).NotTo(HaveOccurred()) + }) + + Describe("Getting the local player Id", func() { + + When("when no configuration has been provided", func() { + It("fails", func() { + _, err := localPlayerID(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + + When("when valid configuration has been provided", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": "2", + "playerId": "0", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("gives the right local player ID", func() { + Expect(localPlayerID(ctx, &vcp.k8sClient, "default")).To(Equal(uint(0))) + }) + }) + + When("when the playerId K/V pair is missing", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": "2", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("fails", func() { + _, err := localPlayerID(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + + When("when the playerId is out of range", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": "2", + "playerId": "-1", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("fails", func() { + _, err := localPlayerID(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + + When("when the playerId can't be parsed", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": "2", + "playerId": "a1b2", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("fails", func() { + _, err := localPlayerID(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Describe("Getting the number of players", func() { + + When("when no configuration has been provided", func() { + It("fails", func() { + _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + + When("when valid configuration has been provided", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": "2", + "playerId": "0", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("gives the right number of Players", func() { + Expect(numberOfPlayers(ctx, &vcp.k8sClient, "default")).To(Equal(uint(2))) + }) + }) + + When("when the playerCount K/V pair is missing", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerId": "0", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("fails", func() { + _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + + When("when the playerCount is out of range", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": "-1", + "playerId": "0", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("fails", func() { + _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + + When("when the playerId can't be parsed", func() { + BeforeEach(func() { + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": "a1b2", + "playerId": "0", + }) + }) + AfterEach(func() { + vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + }) + It("fails", func() { + _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) + }) + }) + }) +}) diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go new file mode 100644 index 0000000..a0bc7b0 --- /dev/null +++ b/klyshko-operator/controllers/controller_test.go @@ -0,0 +1,243 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "fmt" + klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/jarcoal/httpmock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + clientv3 "go.etcd.io/etcd/client/v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "path/filepath" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" + "strconv" + "time" +) + +const NumberOfVcps = 2 + +type Vcp struct { + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment +} + +func setupVcp() (*Vcp, error) { + env := Vcp{} + env.testEnv = &envtest.Environment{ + ErrorIfCRDPathMissing: true, + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + AttachControlPlaneOutput: true, + } + var err error + env.cfg, err = env.testEnv.Start() + if err != nil { + return nil, err + } + err = klyshkov1alpha1.AddToScheme(scheme.Scheme) + if err != nil { + return nil, err + } + env.k8sClient, err = client.New(env.cfg, client.Options{Scheme: scheme.Scheme}) + return &env, err +} + +func (vcp Vcp) tearDownVcp() error { + return vcp.testEnv.Stop() +} + +func (vcp Vcp) createVcpConfig(ctx context.Context, name string, namespace string, data map[string]string) { + vcpConfig := v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } + err := vcp.k8sClient.Create(ctx, &vcpConfig) + if err != nil { + Fail(fmt.Sprintf("couldn't create VCP configuration: %s", err)) + } +} + +func (vcp Vcp) deleteVcpConfig(ctx context.Context, name string, namespace string) { + vcpConfig := &v1.ConfigMap{} + err := vcp.k8sClient.Get(ctx, client.ObjectKey{ + Name: name, + Namespace: namespace, + }, vcpConfig) + if err != nil { + Fail(fmt.Sprintf("couldn't get VCP configuration: %s", err)) + } + err = vcp.k8sClient.Delete(ctx, vcpConfig) + if err != nil { + Fail(fmt.Sprintf("couldn't delete VCP configuration: %s", err)) + } +} + +type Controller interface { + SetupWithManager(manager.Manager) error +} + +func (vcp Vcp) setupControllers(ctx context.Context, etcdClient *clientv3.Client, castorUrl string) error { + k8sManager, err := ctrl.NewManager(vcp.cfg, ctrl.Options{ + Scheme: scheme.Scheme, + MetricsBindAddress: "0", // Avoid colliding metrics servers by disabling + }) + if err != nil { + return err + } + castorClient := NewCastorClient(castorUrl) + controllers := []Controller{ + NewTupleGenerationJobReconciler( + k8sManager.GetClient(), k8sManager.GetScheme(), etcdClient, castorClient), + &TupleGenerationTaskReconciler{ // TODO Replace with constructors + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + EtcdClient: etcdClient, + }, + &TupleGenerationSchedulerReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + CastorClient: castorClient, + }, + } + for _, controller := range controllers { + err := controller.SetupWithManager(k8sManager) + if err != nil { + return err + } + } + go func() { + defer GinkgoRecover() + err = k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + }() + return nil +} + +type Vc struct { + vcps []Vcp + ectd *envtest.Etcd +} + +func setupVc(ctx context.Context, numberOfVcps int) (*Vc, error) { + vc := Vc{} + for i := 0; i < numberOfVcps; i++ { + vcp, err := setupVcp() + if err != nil { + return nil, err + } + + // Create the VCP configuration + vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + "playerCount": strconv.Itoa(1), + "playerId": strconv.Itoa(i), + }) + + // Setup controllers using ectd client connected to first VCPs control plane etcd + if i == 0 { + vc.ectd = vcp.testEnv.ControlPlane.Etcd + } + etcdClient, err := clientv3.New(clientv3.Config{ + Endpoints: []string{vc.ectd.URL.String()}, + DialTimeout: 5 * time.Second, + }) + err = vcp.setupControllers(ctx, etcdClient, "http://cs-castor.default.svc.cluster.local:10100") + if err != nil { + return nil, err + } + + vc.vcps = append(vc.vcps, *vcp) + } + return &vc, nil +} + +func (vc *Vc) teardown() error { + for _, vcp := range vc.vcps { + err := vcp.testEnv.Stop() + if err != nil { + return err + } + } + return nil +} + +var _ = Describe("Generating tuples", func() { + ctx, cancel := context.WithCancel(context.TODO()) + var vc *Vc + + BeforeEach(func() { + httpmock.Activate() + httpmock.RegisterResponder( + "GET", + "=~^http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/.*", + httpmock.NewStringResponder(200, ""), + ) + telemetry := Telemetry{TupleMetrics: []TupleMetrics{ + { + Available: 0, + ConsumptionRate: 0, + TupleType: "MULTIPLICATION_TRIPLE_GFP", + }, + }} + responder, err := httpmock.NewJsonResponder(200, telemetry) + Expect(err).NotTo(HaveOccurred()) + httpmock.RegisterResponder( + "GET", + "=~^http://cs-castor.default.svc.cluster.local:10100/intra-vcp/telemetry", + responder, + ) + vc, err = setupVc(ctx, NumberOfVcps) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + cancel() + err := vc.teardown() + Expect(err).NotTo(HaveOccurred()) + httpmock.DeactivateAndReset() + }) + + When("a scheduler is deployed", func() { + It("succeed", func() { + scheduler := &klyshkov1alpha1.TupleGenerationScheduler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-scheduler", + Namespace: "default", + }, + Spec: klyshkov1alpha1.TupleGenerationSchedulerSpec{ + Concurrency: 1, + Threshold: 50000, + }, + } + Expect(vc.vcps[0].k8sClient.Create(ctx, scheduler)).Should(Succeed()) + + schedulerLookupKey := types.NamespacedName{Name: "test-scheduler", Namespace: "default"} + createdScheduler := &klyshkov1alpha1.TupleGenerationScheduler{} + Eventually(func() bool { + err := vc.vcps[0].k8sClient.Get(ctx, schedulerLookupKey, createdScheduler) + if err != nil { + return false + } + return true + }, 60*time.Second, 5*time.Second).Should(BeTrue()) + time.Sleep(10 * time.Second) + }) + }) +}) diff --git a/klyshko-operator/controllers/etcd_keys_test.go b/klyshko-operator/controllers/etcd_keys_test.go new file mode 100644 index 0000000..a9c0bc8 --- /dev/null +++ b/klyshko-operator/controllers/etcd_keys_test.go @@ -0,0 +1,63 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + "math" +) + +var _ = When("Parsing a key", func() { + rosterKey := RosterKey{ + NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}, + } + rosterEntryKey := RosterEntryKey{ + RosterKey: rosterKey, + PlayerID: 1, + } + + Context("from a serialized roster key", func() { + It("should return original key", func() { + Expect(ParseKey(rosterKey.ToEtcdKey())).To(Equal(rosterKey)) + }) + }) + + Context("from a serialized roster entry key", func() { + It("should return original key", func() { + Expect(ParseKey(rosterEntryKey.ToEtcdKey())).To(Equal(rosterEntryKey)) + }) + }) + + Context("from a roster entry key with non-integer playerId", func() { + It("should fail", func() { + malformedKey := rosterEntryKey.ToEtcdKey() + "m" + _, err := ParseKey(malformedKey) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("from a roster entry key with non-unit32 playerId", func() { + It("should fail", func() { + malformedKey := rosterEntryKey.ToEtcdKey() + fmt.Sprint(math.MaxUint32*2) + _, err := ParseKey(malformedKey) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("from a roster entry key with non-atoi parseable playerId", func() { + It("should fail", func() { + malformedKey := rosterEntryKey.ToEtcdKey() + fmt.Sprint(math.MaxUint64/2) + _, err := ParseKey(malformedKey) + Expect(err).To(HaveOccurred()) + }) + }) + +}) diff --git a/klyshko-operator/controllers/suite_test.go b/klyshko-operator/controllers/suite_test.go index cbf4005..d7dd757 100644 --- a/klyshko-operator/controllers/suite_test.go +++ b/klyshko-operator/controllers/suite_test.go @@ -8,64 +8,20 @@ SPDX-License-Identifier: Apache-2.0 package controllers import ( - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + . "github.com/onsi/ginkgo/v2" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "testing" - klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + . "github.com/onsi/gomega" //+kubebuilder:scaffold:imports ) -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecsWithDefaultAndCustomReporters(t, - "Controller Suite", - []Reporter{printer.NewlineReporter{}}) -} - var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - cfg, err := testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = klyshkov1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - -}, 60) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) }) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Klyshko Controller Suite") +} diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 734647f..b382114 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -31,14 +31,16 @@ type TupleGenerationJobReconciler struct { Scheme *runtime.Scheme EtcdClient *clientv3.Client rosterWatcherCh clientv3.WatchChan + CastorClient *CastorClient } -func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Scheme, etcdClient *clientv3.Client) *TupleGenerationJobReconciler { +func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Scheme, etcdClient *clientv3.Client, castorClient *CastorClient) *TupleGenerationJobReconciler { r := &TupleGenerationJobReconciler{ Client: client, Scheme: scheme, EtcdClient: etcdClient, rosterWatcherCh: etcdClient.Watch(context.Background(), rosterKey, clientv3.WithPrefix()), // TODO Close channel? + CastorClient: castorClient, } go r.handleWatchEvents() return r @@ -356,7 +358,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R logger.Error(err, "invalid job id encountered") return ctrl.Result{}, nil } - err = activateTupleChunk(ctx, tupleChunkID) + err = r.CastorClient.activateTupleChunk(ctx, tupleChunkID) if err != nil { logger.Error(err, "tuple activation failed") return ctrl.Result{}, nil diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index eeb844d..b87d19f 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -27,7 +27,8 @@ const MinimumTuplesPerJob = 10000 type TupleGenerationSchedulerReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + CastorClient *CastorClient } //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationschedulers,verbs=get;list;watch;create;update;patch;delete @@ -69,7 +70,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req // 1. Compute available and in generation number of tuples per type // 2. Filter out all above threshold // 3. Sort ascending wrt to sum from step 1 - telemetry, err := getTelemetry(ctx) + telemetry, err := r.CastorClient.getTelemetry(ctx) if err != nil { return ctrl.Result{RequeueAfter: 60 * time.Second}, err } diff --git a/klyshko-operator/go.mod b/klyshko-operator/go.mod index 932b985..aa89a92 100644 --- a/klyshko-operator/go.mod +++ b/klyshko-operator/go.mod @@ -4,8 +4,9 @@ go 1.16 require ( github.com/google/uuid v1.1.2 - github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.13.0 + github.com/jarcoal/httpmock v1.2.0 + github.com/onsi/ginkgo/v2 v2.1.4 + github.com/onsi/gomega v1.19.0 go.etcd.io/etcd/api/v3 v3.5.2 go.etcd.io/etcd/client/v3 v3.5.2 k8s.io/api v0.21.2 diff --git a/klyshko-operator/go.sum b/klyshko-operator/go.sum index 662c1a7..126fa74 100644 --- a/klyshko-operator/go.sum +++ b/klyshko-operator/go.sum @@ -195,6 +195,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -239,10 +240,13 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= +github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -280,6 +284,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk= +github.com/maxatome/go-testdeep v1.11.0/go.mod h1:011SgQ6efzZYAen6fDn4BqQ+lUR72ysdyKe7Dyogw70= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -316,12 +322,17 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -404,6 +415,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -440,8 +452,9 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -476,6 +489,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -507,9 +521,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -571,12 +588,17 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -584,8 +606,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -634,8 +657,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index dc8058c..1c90818 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -45,11 +45,13 @@ func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string + var castorUrl string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&castorUrl, "castor-url", "http://cs-castor.default.svc.cluster.local:10100", "The base url of the castor service.") opts := zap.Options{ Development: true, } @@ -79,9 +81,14 @@ func main() { setupLog.Error(err, "unable to create etcd client", "controller", "TupleGenerationJob") os.Exit(1) } - defer etcdClient.Close() + defer func() { + err := etcdClient.Close() + setupLog.Error(err, "closing etcd client failed") + }() - if err = controllers.NewTupleGenerationJobReconciler(mgr.GetClient(), mgr.GetScheme(), etcdClient).SetupWithManager(mgr); err != nil { + castorClient := controllers.NewCastorClient(castorUrl) + + if err = controllers.NewTupleGenerationJobReconciler(mgr.GetClient(), mgr.GetScheme(), etcdClient, castorClient).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationJob") os.Exit(1) } @@ -96,8 +103,9 @@ func main() { } if err = (&controllers.TupleGenerationSchedulerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + CastorClient: castorClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationScheduler") os.Exit(1) From 306af578a59ce08f405e029ea70c193738422be9 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 29 Jul 2022 15:42:35 +0200 Subject: [PATCH 30/69] Improve tests and refactor codebase Signed-off-by: Sven Trieflinger --- .../api/v1alpha1/tuplegenerationtask_types.go | 8 +- klyshko-operator/controllers/castor_client.go | 8 +- .../controllers/castor_client_test.go | 24 +- klyshko-operator/controllers/config_utils.go | 12 +- .../controllers/config_utils_test.go | 54 +-- .../controllers/controller_test.go | 330 ++++++++++--- .../tuplegenerationjob_controller.go | 440 ++++++++++-------- .../tuplegenerationscheduler_controller.go | 81 +++- .../tuplegenerationtask_controller.go | 256 +++++----- klyshko-operator/go.mod | 3 +- 10 files changed, 754 insertions(+), 462 deletions(-) diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go index 6d5cbcc..970fa56 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go @@ -16,10 +16,10 @@ type TupleGenerationTaskState string const ( TaskLaunching TupleGenerationTaskState = "Launching" - TaskGenerating = "Generating" - TaskProvisioning = "Provisioning" - TaskCompleted = "Completed" - TaskFailed = "Failed" + TaskGenerating TupleGenerationTaskState = "Generating" + TaskProvisioning TupleGenerationTaskState = "Provisioning" + TaskCompleted TupleGenerationTaskState = "Completed" + TaskFailed TupleGenerationTaskState = "Failed" ) func (s TupleGenerationTaskState) IsValid() bool { diff --git a/klyshko-operator/controllers/castor_client.go b/klyshko-operator/controllers/castor_client.go index 78c8e61..215f32e 100644 --- a/klyshko-operator/controllers/castor_client.go +++ b/klyshko-operator/controllers/castor_client.go @@ -20,20 +20,20 @@ import ( ) type CastorClient struct { - Url string + URL string client *http.Client } func NewCastorClient(url string) *CastorClient { return &CastorClient{ - Url: url, + URL: url, client: &http.Client{}, } } func (castorClient CastorClient) activateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkID) - url := fmt.Sprintf("%s/intra-vcp/tuple-chunks/activate/%s", castorClient.Url, chunkID) + url := fmt.Sprintf("%s/intra-vcp/tuple-chunks/activate/%s", castorClient.URL, chunkID) logger.Info("activating tuple chunk with castor URL", "URL", url) req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, nil) if err != nil { @@ -77,7 +77,7 @@ func (castorClient CastorClient) getTelemetry(ctx context.Context) (Telemetry, e req, err := http.NewRequestWithContext( ctx, "GET", - fmt.Sprintf("%s/intra-vcp/telemetry", castorClient.Url), + fmt.Sprintf("%s/intra-vcp/telemetry", castorClient.URL), nil, ) if err != nil { diff --git a/klyshko-operator/controllers/castor_client_test.go b/klyshko-operator/controllers/castor_client_test.go index 05c8fe4..fd4e69c 100644 --- a/klyshko-operator/controllers/castor_client_test.go +++ b/klyshko-operator/controllers/castor_client_test.go @@ -16,8 +16,8 @@ import ( . "github.com/onsi/gomega" ) -const validCastorUrl = "http://cs-castor.default.svc.cluster.local:10100" -const invalidCastorUrl = "http://cs-castor.default.svc.cluster.local:10101" +const validCastorURL = "http://cs-castor.default.svc.cluster.local:10100" +const invalidCastorURL = "http://cs-castor.default.svc.cluster.local:10101" var _ = Describe("Fetching telemetry", func() { @@ -43,12 +43,12 @@ var _ = Describe("Fetching telemetry", func() { Expect(err).NotTo(HaveOccurred()) httpmock.RegisterResponder( "GET", - fmt.Sprintf("=~^%s/intra-vcp/telemetry", validCastorUrl), + fmt.Sprintf("=~^%s/intra-vcp/telemetry", validCastorURL), responder, ) }) It("succeeds", func() { - castorClient := NewCastorClient(validCastorUrl) + castorClient := NewCastorClient(validCastorURL) telemetry, err := castorClient.getTelemetry(ctx) Expect(err).NotTo(HaveOccurred()) Expect(telemetry).To(Equal(expectedTelemetry)) @@ -60,12 +60,12 @@ var _ = Describe("Fetching telemetry", func() { httpmock.Activate() httpmock.RegisterResponder( "GET", - fmt.Sprintf("=~^%s/intra-vcp/telemetry", validCastorUrl), + fmt.Sprintf("=~^%s/intra-vcp/telemetry", validCastorURL), httpmock.NewStringResponder(404, ""), ) }) It("fails", func() { - castorClient := NewCastorClient(validCastorUrl) + castorClient := NewCastorClient(validCastorURL) _, err := castorClient.getTelemetry(ctx) Expect(err).To(HaveOccurred()) }) @@ -73,7 +73,7 @@ var _ = Describe("Fetching telemetry", func() { When("when Castor service is not available", func() { It("fails", func() { - castorClient := NewCastorClient(invalidCastorUrl) + castorClient := NewCastorClient(invalidCastorURL) _, err := castorClient.getTelemetry(ctx) Expect(err).To(HaveOccurred()) }) @@ -94,13 +94,13 @@ var _ = Describe("Activating a tuple chunk", func() { httpmock.Activate() httpmock.RegisterResponder( "PUT", - fmt.Sprintf("=~^%s/intra-vcp/tuple-chunks/activate/.*", validCastorUrl), + fmt.Sprintf("=~^%s/intra-vcp/tuple-chunks/activate/.*", validCastorURL), httpmock.NewStringResponder(200, ""), ) }) It("succeeds", func() { chunkId := uuid.New() - castorClient := NewCastorClient(validCastorUrl) + castorClient := NewCastorClient(validCastorURL) err := castorClient.activateTupleChunk(ctx, chunkId) Expect(err).NotTo(HaveOccurred()) }) @@ -111,13 +111,13 @@ var _ = Describe("Activating a tuple chunk", func() { httpmock.Activate() httpmock.RegisterResponder( "PUT", - fmt.Sprintf("=~^%s/intra-vcp/tuple-chunks/activate/.*", validCastorUrl), + fmt.Sprintf("=~^%s/intra-vcp/tuple-chunks/activate/.*", validCastorURL), httpmock.NewStringResponder(404, ""), ) }) It("fails", func() { chunkId := uuid.New() - castorClient := NewCastorClient(validCastorUrl) + castorClient := NewCastorClient(validCastorURL) err := castorClient.activateTupleChunk(ctx, chunkId) Expect(err).To(HaveOccurred()) }) @@ -126,7 +126,7 @@ var _ = Describe("Activating a tuple chunk", func() { When("when Castor service is not available", func() { It("fails", func() { chunkId := uuid.New() - castorClient := NewCastorClient(invalidCastorUrl) + castorClient := NewCastorClient(invalidCastorURL) err := castorClient.activateTupleChunk(ctx, chunkId) Expect(err).To(HaveOccurred()) }) diff --git a/klyshko-operator/controllers/config_utils.go b/klyshko-operator/controllers/config_utils.go index a1c02e9..e8d7d88 100644 --- a/klyshko-operator/controllers/config_utils.go +++ b/klyshko-operator/controllers/config_utils.go @@ -18,12 +18,12 @@ import ( "strconv" ) -const VcpConfigMapName = "cs-vcp-config" +const VCPConfigMapName = "cs-vcp-config" -func getVcpConfig(ctx context.Context, client *client.Client, namespace string) (v1.ConfigMap, error) { +func getVCPConfig(ctx context.Context, client *client.Client, namespace string) (v1.ConfigMap, error) { name := types.NamespacedName{ Namespace: namespace, - Name: VcpConfigMapName, + Name: VCPConfigMapName, } cfm := v1.ConfigMap{} err := (*client).Get(ctx, name, &cfm) @@ -34,7 +34,7 @@ func getVcpConfig(ctx context.Context, client *client.Client, namespace string) } func localPlayerID(ctx context.Context, client *client.Client, namespace string) (uint, error) { - cfm, err := getVcpConfig(ctx, client, namespace) + cfm, err := getVCPConfig(ctx, client, namespace) if err != nil { return 0, err } @@ -54,8 +54,8 @@ func localPlayerID(ctx context.Context, client *client.Client, namespace string) return uint(playerID), nil } -func numberOfPlayers(ctx context.Context, client *client.Client, namespace string) (uint, error) { - cfm, err := getVcpConfig(ctx, client, namespace) +func numberOfVCPs(ctx context.Context, client *client.Client, namespace string) (uint, error) { + cfm, err := getVCPConfig(ctx, client, namespace) if err != nil { return 0, err } diff --git a/klyshko-operator/controllers/config_utils_test.go b/klyshko-operator/controllers/config_utils_test.go index 9898265..1fab045 100644 --- a/klyshko-operator/controllers/config_utils_test.go +++ b/klyshko-operator/controllers/config_utils_test.go @@ -14,23 +14,23 @@ import ( //+kubebuilder:scaffold:imports ) -var _ = Context("TODO", func() { // TODO Description? +var _ = Context("Using config utils", func() { ctx := context.Background() - var vcp *Vcp + var vcp *vcp BeforeEach(func() { var err error - vcp, err = setupVcp() + vcp, err = setupVCP() Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { - err := vcp.tearDownVcp() + err := vcp.tearDownVCP() Expect(err).NotTo(HaveOccurred()) }) - Describe("Getting the local player Id", func() { + Describe("getting the local player Id", func() { When("when no configuration has been provided", func() { It("fails", func() { @@ -41,13 +41,13 @@ var _ = Context("TODO", func() { // TODO Description? When("when valid configuration has been provided", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": "2", "playerId": "0", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("gives the right local player ID", func() { Expect(localPlayerID(ctx, &vcp.k8sClient, "default")).To(Equal(uint(0))) @@ -56,12 +56,12 @@ var _ = Context("TODO", func() { // TODO Description? When("when the playerId K/V pair is missing", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": "2", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("fails", func() { _, err := localPlayerID(ctx, &vcp.k8sClient, "default") @@ -71,13 +71,13 @@ var _ = Context("TODO", func() { // TODO Description? When("when the playerId is out of range", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": "2", "playerId": "-1", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("fails", func() { _, err := localPlayerID(ctx, &vcp.k8sClient, "default") @@ -87,13 +87,13 @@ var _ = Context("TODO", func() { // TODO Description? When("when the playerId can't be parsed", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": "2", "playerId": "a1b2", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("fails", func() { _, err := localPlayerID(ctx, &vcp.k8sClient, "default") @@ -102,73 +102,73 @@ var _ = Context("TODO", func() { // TODO Description? }) }) - Describe("Getting the number of players", func() { + Describe("getting the number of players", func() { When("when no configuration has been provided", func() { It("fails", func() { - _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + _, err := numberOfVCPs(ctx, &vcp.k8sClient, "default") Expect(err).To(HaveOccurred()) }) }) When("when valid configuration has been provided", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": "2", "playerId": "0", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("gives the right number of Players", func() { - Expect(numberOfPlayers(ctx, &vcp.k8sClient, "default")).To(Equal(uint(2))) + Expect(numberOfVCPs(ctx, &vcp.k8sClient, "default")).To(Equal(uint(2))) }) }) When("when the playerCount K/V pair is missing", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerId": "0", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("fails", func() { - _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + _, err := numberOfVCPs(ctx, &vcp.k8sClient, "default") Expect(err).To(HaveOccurred()) }) }) When("when the playerCount is out of range", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": "-1", "playerId": "0", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("fails", func() { - _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + _, err := numberOfVCPs(ctx, &vcp.k8sClient, "default") Expect(err).To(HaveOccurred()) }) }) When("when the playerId can't be parsed", func() { BeforeEach(func() { - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": "a1b2", "playerId": "0", }) }) AfterEach(func() { - vcp.deleteVcpConfig(ctx, "cs-vcp-config", "default") + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") }) It("fails", func() { - _, err := numberOfPlayers(ctx, &vcp.k8sClient, "default") + _, err := numberOfVCPs(ctx, &vcp.k8sClient, "default") Expect(err).To(HaveOccurred()) }) }) diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go index a0bc7b0..e9046d5 100644 --- a/klyshko-operator/controllers/controller_test.go +++ b/klyshko-operator/controllers/controller_test.go @@ -11,34 +11,46 @@ import ( "context" "fmt" klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/google/uuid" "github.com/jarcoal/httpmock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" clientv3 "go.etcd.io/etcd/client/v3" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "k8s.io/utils/pointer" + "math" "path/filepath" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "strconv" + "strings" "time" ) -const NumberOfVcps = 2 +const NumberOfVCPs = 2 +const Timeout = 10 * time.Second +const PollingInterval = 1 * time.Second +const SchedulerNamespace = "default" +const SchedulerName = "test-scheduler" +const SchedulerConcurrency = 1 +const SchedulerTupleThreshold = 50000 -type Vcp struct { +type vcp struct { cfg *rest.Config k8sClient client.Client testEnv *envtest.Environment } -func setupVcp() (*Vcp, error) { - env := Vcp{} +func setupVCP() (*vcp, error) { + env := vcp{} env.testEnv = &envtest.Environment{ ErrorIfCRDPathMissing: true, CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, @@ -57,11 +69,11 @@ func setupVcp() (*Vcp, error) { return &env, err } -func (vcp Vcp) tearDownVcp() error { +func (vcp vcp) tearDownVCP() error { return vcp.testEnv.Stop() } -func (vcp Vcp) createVcpConfig(ctx context.Context, name string, namespace string, data map[string]string) { +func (vcp vcp) createVCPConfig(ctx context.Context, name string, namespace string, data map[string]string) { vcpConfig := v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -75,7 +87,7 @@ func (vcp Vcp) createVcpConfig(ctx context.Context, name string, namespace strin } } -func (vcp Vcp) deleteVcpConfig(ctx context.Context, name string, namespace string) { +func (vcp vcp) deleteVCPConfig(ctx context.Context, name string, namespace string) { vcpConfig := &v1.ConfigMap{} err := vcp.k8sClient.Get(ctx, client.ObjectKey{ Name: name, @@ -94,15 +106,16 @@ type Controller interface { SetupWithManager(manager.Manager) error } -func (vcp Vcp) setupControllers(ctx context.Context, etcdClient *clientv3.Client, castorUrl string) error { +func (vcp vcp) setupControllers(ctx context.Context, vcpID int, etcdClient *clientv3.Client, castorURL string) error { k8sManager, err := ctrl.NewManager(vcp.cfg, ctrl.Options{ Scheme: scheme.Scheme, - MetricsBindAddress: "0", // Avoid colliding metrics servers by disabling + MetricsBindAddress: "0", // Avoid colliding metrics servers by disabling + Logger: logf.Log.WithName(fmt.Sprintf("vcp-%d", vcpID)), // use scoped logger to ease debugging }) if err != nil { return err } - castorClient := NewCastorClient(castorUrl) + castorClient := NewCastorClient(castorURL) controllers := []Controller{ NewTupleGenerationJobReconciler( k8sManager.GetClient(), k8sManager.GetScheme(), etcdClient, castorClient), @@ -131,21 +144,21 @@ func (vcp Vcp) setupControllers(ctx context.Context, etcdClient *clientv3.Client return nil } -type Vc struct { - vcps []Vcp +type vc struct { + vcps []vcp ectd *envtest.Etcd } -func setupVc(ctx context.Context, numberOfVcps int) (*Vc, error) { - vc := Vc{} - for i := 0; i < numberOfVcps; i++ { - vcp, err := setupVcp() +func setupVC(ctx context.Context, numberOfVCPs int) (*vc, error) { + vc := vc{} + for i := 0; i < numberOfVCPs; i++ { + vcp, err := setupVCP() if err != nil { return nil, err } // Create the VCP configuration - vcp.createVcpConfig(ctx, "cs-vcp-config", "default", map[string]string{ + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ "playerCount": strconv.Itoa(1), "playerId": strconv.Itoa(i), }) @@ -156,9 +169,9 @@ func setupVc(ctx context.Context, numberOfVcps int) (*Vc, error) { } etcdClient, err := clientv3.New(clientv3.Config{ Endpoints: []string{vc.ectd.URL.String()}, - DialTimeout: 5 * time.Second, + DialTimeout: Timeout, }) - err = vcp.setupControllers(ctx, etcdClient, "http://cs-castor.default.svc.cluster.local:10100") + err = vcp.setupControllers(ctx, i, etcdClient, "http://cs-castor.default.svc.cluster.local:10100") if err != nil { return nil, err } @@ -168,7 +181,7 @@ func setupVc(ctx context.Context, numberOfVcps int) (*Vc, error) { return &vc, nil } -func (vc *Vc) teardown() error { +func (vc *vc) teardown() error { for _, vcp := range vc.vcps { err := vcp.testEnv.Stop() if err != nil { @@ -178,32 +191,38 @@ func (vc *Vc) teardown() error { return nil } +func setupCastorServiceResponders(numberOfAvailableTuples int) { + httpmock.Reset() + httpmock.RegisterResponder( + "PUT", + "=~^http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/.*", + httpmock.NewStringResponder(200, ""), + ) + telemetry := Telemetry{TupleMetrics: []TupleMetrics{ + { + Available: numberOfAvailableTuples, + ConsumptionRate: 0, + TupleType: "MULTIPLICATION_TRIPLE_GFP", + }, + }} + responder, err := httpmock.NewJsonResponder(200, telemetry) + Expect(err).NotTo(HaveOccurred()) + httpmock.RegisterResponder( + "GET", + "=~^http://cs-castor.default.svc.cluster.local:10100/intra-vcp/telemetry", + responder, + ) +} + var _ = Describe("Generating tuples", func() { ctx, cancel := context.WithCancel(context.TODO()) - var vc *Vc + var vc *vc BeforeEach(func() { httpmock.Activate() - httpmock.RegisterResponder( - "GET", - "=~^http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/.*", - httpmock.NewStringResponder(200, ""), - ) - telemetry := Telemetry{TupleMetrics: []TupleMetrics{ - { - Available: 0, - ConsumptionRate: 0, - TupleType: "MULTIPLICATION_TRIPLE_GFP", - }, - }} - responder, err := httpmock.NewJsonResponder(200, telemetry) - Expect(err).NotTo(HaveOccurred()) - httpmock.RegisterResponder( - "GET", - "=~^http://cs-castor.default.svc.cluster.local:10100/intra-vcp/telemetry", - responder, - ) - vc, err = setupVc(ctx, NumberOfVcps) + setupCastorServiceResponders(0) + var err error + vc, err = setupVC(ctx, NumberOfVCPs) Expect(err).NotTo(HaveOccurred()) }) @@ -215,29 +234,220 @@ var _ = Describe("Generating tuples", func() { }) When("a scheduler is deployed", func() { - It("succeed", func() { - scheduler := &klyshkov1alpha1.TupleGenerationScheduler{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-scheduler", - Namespace: "default", - }, - Spec: klyshkov1alpha1.TupleGenerationSchedulerSpec{ - Concurrency: 1, - Threshold: 50000, - }, + It("succeeds", func() { + scheduler := createScheduler(ctx, vc) + jobs := ensureJobCreatedOnEachVcp(ctx, vc, scheduler) + + // Make Castor mock respond from here on with large number of available tuples, to ensure that no other + // jobs are created + setupCastorServiceResponders(math.MaxInt32) + + localTasksByVCP := ensureTasksCreatedOnEachVcp(ctx, vc, scheduler, jobs) + generatorPodsByVCP := ensureGeneratorPodsCreatedOnEachVcp(ctx, vc, localTasksByVCP) + ensureJobState(ctx, vc, scheduler, uuid.MustParse(jobs[0].Spec.ID), klyshkov1alpha1.JobRunning) + + // Update generator pods to be in PodSucceed state + for i, pod := range generatorPodsByVCP { + pod.Status.Phase = v1.PodSucceeded + Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) } - Expect(vc.vcps[0].k8sClient.Create(ctx, scheduler)).Should(Succeed()) - schedulerLookupKey := types.NamespacedName{Name: "test-scheduler", Namespace: "default"} - createdScheduler := &klyshkov1alpha1.TupleGenerationScheduler{} + provisionerPodsByVCP := ensureProvisionerPodsCreatedOnEachVcp(ctx, vc, jobs, localTasksByVCP) + + // Update provisioner pods to be in PodSucceed state + for i, pod := range provisionerPodsByVCP { + pod.Status.Phase = v1.PodSucceeded + Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) + } + + // Ensure that castor activate tuple chunk endpoint has been called on each VCP + activationUrl := fmt.Sprintf("PUT http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", jobs[0].Spec.ID) Eventually(func() bool { - err := vc.vcps[0].k8sClient.Get(ctx, schedulerLookupKey, createdScheduler) - if err != nil { - return false + info := httpmock.GetCallCountInfo() + return info[activationUrl] == NumberOfVCPs + }, Timeout, PollingInterval).Should(BeTrue()) + + // Ensure that resources get deleted. + // As of https://book-v2.book.kubebuilder.io/reference/envtest.html#testing-considerations garbage + // collection does not work in envtest. Hence, we can only check that the jobs get deleted and ensure that + // owner references are set up correctly for all our resources (see respective ensure... methods below). + for i := 0; i < NumberOfVCPs; i++ { + key := client.ObjectKey{ + Namespace: jobs[i].GetNamespace(), + Name: jobs[i].GetName(), } - return true - }, 60*time.Second, 5*time.Second).Should(BeTrue()) - time.Sleep(10 * time.Second) + Eventually(func() bool { + return apierrors.IsNotFound(vc.vcps[i].k8sClient.Get(ctx, key, &jobs[i])) + }, Timeout, PollingInterval).Should(BeTrue()) + } + }) }) }) + +// Ensures that the job with the given identifier eventually assumes the given state. +func ensureJobState(ctx context.Context, vc *vc, owner *klyshkov1alpha1.TupleGenerationScheduler, jobId uuid.UUID, state klyshkov1alpha1.TupleGenerationJobState) { + for i := 0; i < NumberOfVCPs; i++ { + name := client.ObjectKey{ + Namespace: owner.Namespace, + Name: fmt.Sprintf("%s-%s", owner.Name, jobId), + } + Eventually(func() bool { + job := &klyshkov1alpha1.TupleGenerationJob{} + err := vc.vcps[i].k8sClient.Get(ctx, name, job) + if err != nil { + return false + } + return job.Status.State == state + }, Timeout, PollingInterval).Should(BeTrue()) + } +} + +// Ensures that pods with a certain name eventually become available in each VCP of the given VC and are owned by a +// certain object. Both name and owner are computed from the provided functions that take the VCP identifier as an +// argument. +func ensurePodsCreatedOnEachVcp(ctx context.Context, vc *vc, name func(int) types.NamespacedName, owner func(int) client.Object) []v1.Pod { + pods := make([]v1.Pod, NumberOfVCPs) + for i := 0; i < NumberOfVCPs; i++ { + pod := &v1.Pod{} + Eventually(func() bool { + err := vc.vcps[i].k8sClient.Get(ctx, name(i), pod) + if err != nil { + return false + } + return true + }, Timeout, PollingInterval).Should(BeTrue()) + expectedOwnerReference := metav1.OwnerReference{ + Kind: owner(i).GetObjectKind().GroupVersionKind().Kind, + APIVersion: owner(i).GetObjectKind().GroupVersionKind().GroupVersion().String(), + UID: owner(i).GetUID(), + Name: owner(i).GetName(), + Controller: pointer.Bool(true), + BlockOwnerDeletion: pointer.Bool(true), + } + Expect(pod.OwnerReferences).To(ContainElement(expectedOwnerReference)) + pods[i] = *pod + } + return pods +} + +// Ensures that provisioner pods associated with the respective tasks eventually become available for the given job in +// each VCP of the given VC. In addition, it is checked that the pod is owned by the respective task. +func ensureProvisionerPodsCreatedOnEachVcp(ctx context.Context, vc *vc, jobs []klyshkov1alpha1.TupleGenerationJob, localTasks []klyshkov1alpha1.TupleGenerationTask) []v1.Pod { + return ensurePodsCreatedOnEachVcp(ctx, vc, func(i int) types.NamespacedName { + return types.NamespacedName{ + Namespace: jobs[i].Namespace, + Name: fmt.Sprintf("%s-provisioner", jobs[i].Name), + } + }, func(i int) client.Object { + return &localTasks[i] + }) +} + +// Ensures that generator pods associated with the respective tasks eventually become available in each VCP of the +// given VC. In addition, it is checked that the pod is owned by the respective task. +func ensureGeneratorPodsCreatedOnEachVcp(ctx context.Context, vc *vc, localTasks []klyshkov1alpha1.TupleGenerationTask) []v1.Pod { + return ensurePodsCreatedOnEachVcp(ctx, vc, func(i int) types.NamespacedName { + return types.NamespacedName{ + Namespace: localTasks[i].Namespace, + Name: localTasks[i].Name, + } + }, func(i int) client.Object { + return &localTasks[i] + }) +} + +// Ensures that tasks for the respective job resources eventually become available in each VCP of the given VC. It is +// also checked that tasks are owned by the respective job and that the task is in the expected state. +func ensureTasksCreatedOnEachVcp(ctx context.Context, vc *vc, scheduler *klyshkov1alpha1.TupleGenerationScheduler, jobs []klyshkov1alpha1.TupleGenerationJob) []klyshkov1alpha1.TupleGenerationTask { + localTasks := make([]klyshkov1alpha1.TupleGenerationTask, NumberOfVCPs) + for i := 0; i < NumberOfVCPs; i++ { + taskList := &klyshkov1alpha1.TupleGenerationTaskList{} + opts := []client.ListOption{ + client.InNamespace(scheduler.Namespace), + } + Eventually(func() bool { + err := vc.vcps[i].k8sClient.List(ctx, taskList, opts...) + if err != nil { + return false + } + allGenerating := true + for _, t := range taskList.Items { + allGenerating = allGenerating && (t.Status.State == klyshkov1alpha1.TaskGenerating) + } + return len(taskList.Items) == NumberOfVCPs && allGenerating + }, Timeout, PollingInterval).Should(BeTrue()) + for _, t := range taskList.Items { + if strings.HasSuffix(t.Name, fmt.Sprintf("-%d", i)) { + localTasks[i] = t + } + } + Expect(localTasks[i].Status.State).To(Equal(klyshkov1alpha1.TaskGenerating)) + expectedOwnerReference := metav1.OwnerReference{ + Kind: jobs[i].Kind, + APIVersion: jobs[i].APIVersion, + UID: jobs[i].UID, + Name: jobs[i].Name, + Controller: pointer.Bool(true), + BlockOwnerDeletion: pointer.Bool(true), + } + Expect(localTasks[i].OwnerReferences).To(ContainElement(expectedOwnerReference)) + } + return localTasks +} + +// Ensures that a job is created on each VCP of the given VC. Also ensures that job identifiers are the same across all +// VCPs. +func ensureJobCreatedOnEachVcp(ctx context.Context, vc *vc, scheduler *klyshkov1alpha1.TupleGenerationScheduler) []klyshkov1alpha1.TupleGenerationJob { + jobs := make([]klyshkov1alpha1.TupleGenerationJob, NumberOfVCPs) + for i := 0; i < NumberOfVCPs; i++ { + jobList := &klyshkov1alpha1.TupleGenerationJobList{} + opts := []client.ListOption{ + client.InNamespace(scheduler.Namespace), + } + Eventually(func() bool { + err := vc.vcps[i].k8sClient.List(ctx, jobList, opts...) + if err != nil { + return false + } + return len(jobList.Items) != 0 + }, Timeout, PollingInterval).Should(BeTrue()) + job := jobList.Items[0] + Expect(job.Spec.Type).To(Equal("MULTIPLICATION_TRIPLE_GFP")) + Expect(job.Spec.Count > 0).Should(BeTrue()) + if i > 0 { + Expect(job.Spec.ID).To(Equal(jobs[0].Spec.ID)) + } + jobs[i] = job + } + return jobs +} + +// Creates a scheduler and waits until it becomes available. Checks that created scheduler spec properties are as +// expected. +func createScheduler(ctx context.Context, vc *vc) *klyshkov1alpha1.TupleGenerationScheduler { + scheduler := &klyshkov1alpha1.TupleGenerationScheduler{ + ObjectMeta: metav1.ObjectMeta{ + Name: SchedulerName, + Namespace: SchedulerNamespace, + }, + Spec: klyshkov1alpha1.TupleGenerationSchedulerSpec{ + Concurrency: SchedulerConcurrency, + Threshold: SchedulerTupleThreshold, + }, + } + Expect(vc.vcps[0].k8sClient.Create(ctx, scheduler)).Should(Succeed()) + + key := types.NamespacedName{Name: SchedulerName, Namespace: SchedulerNamespace} + createdScheduler := &klyshkov1alpha1.TupleGenerationScheduler{} + Eventually(func() bool { + err := vc.vcps[0].k8sClient.Get(ctx, key, createdScheduler) + if err != nil { + return false + } + return true + }, Timeout, PollingInterval).Should(BeTrue()) + Expect(createdScheduler.Spec.Threshold).To(Equal(SchedulerTupleThreshold)) + Expect(createdScheduler.Spec.Concurrency).To(Equal(SchedulerConcurrency)) + return createdScheduler +} diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index b382114..841bb34 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/go-logr/logr" "github.com/google/uuid" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" @@ -26,14 +27,17 @@ import ( "time" ) +// TupleGenerationJobReconciler reconciles a TupleGenerationJob object. type TupleGenerationJobReconciler struct { client.Client Scheme *runtime.Scheme EtcdClient *clientv3.Client rosterWatcherCh clientv3.WatchChan CastorClient *CastorClient + Logger logr.Logger } +// NewTupleGenerationJobReconciler creates a TupleGenerationJobReconciler. func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Scheme, etcdClient *clientv3.Client, castorClient *CastorClient) *TupleGenerationJobReconciler { r := &TupleGenerationJobReconciler{ Client: client, @@ -41,11 +45,203 @@ func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Schem EtcdClient: etcdClient, rosterWatcherCh: etcdClient.Watch(context.Background(), rosterKey, clientv3.WithPrefix()), // TODO Close channel? CastorClient: castorClient, + Logger: logr.DiscardLogger{}, } go r.handleWatchEvents() return r } +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch + +// Reconcile compares the actual state of TupleGenerationJob resources to their desired state and performs actions to +// bring the actual state closer to the desired one. +func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + + // Assigning logger to field to allow access from etcd event handling logic below + r.Logger = log.FromContext(ctx) + + jobKey := RosterKey{ + req.NamespacedName, + } + logger := r.Logger.WithValues("Job.Key", jobKey) + logger.Info("Reconciling tuple generation jobs") + + // Cleanup if job has been deleted + job := &klyshkov1alpha1.TupleGenerationJob{} + err := r.Get(ctx, req.NamespacedName, job) + if err != nil { + if apierrors.IsNotFound(err) { + // Job resource not available -> has been deleted + _, err := r.EtcdClient.Delete(ctx, jobKey.ToEtcdKey()) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to delete roster for job %v: %w", req.Name, err) + } + logger.Info("Roster deleted") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, fmt.Errorf("failed to read resource for job %v: %w", req.Name, err) + } + logger.Info("Job exists already") + + // Create roster if not existing (no etcd transaction needed as remote job creation is triggered by roster creation) + resp, err := r.EtcdClient.Get(ctx, jobKey.ToEtcdKey()) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to read resource for roster with key %v for task %v: %w", jobKey, req.Name, err) + } + if resp.Count == 0 { + playerId, err := localPlayerID(ctx, &r.Client, req.Namespace) + if playerId != 0 { + logger.Info("Roster not available, retrying later") + return ctrl.Result{}, nil + } + encoded, err := json.Marshal(job.Spec) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to marshal specification for job %v: %w", req.Name, err) + } + _, err = r.EtcdClient.Put(ctx, jobKey.ToEtcdKey(), string(encoded)) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create roster for job %v: %w", req.Name, err) + } + logger.Info("Roster created") + } else { + logger.Info("Roster exists already") + } + + // Create local task if not existing + playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) + if err != nil { + return ctrl.Result{RequeueAfter: 60 * time.Second}, fmt.Errorf("can't read playerId from VCP configuration for job %v: %w", req.Name, err) + } + task := &klyshkov1alpha1.TupleGenerationTask{} + err = r.Get(ctx, types.NamespacedName{ + Namespace: job.Namespace, + Name: r.taskName(job, playerID), + }, task) + if err != nil { + if apierrors.IsNotFound(err) { + // Create new local task for job + task, err = r.taskForJob(job, playerID) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to define local task for job %v: %w", req.Name, err) + } + err = r.Create(ctx, task) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create local task for job %v: %w", req.Name, err) + } + logger.Info("Local task created", "Task.Name", task.Name) + return ctrl.Result{Requeue: true}, nil + } + // Error reading resource, requeue + return ctrl.Result{}, fmt.Errorf("failed to read task resource for job %v: %w", req.Name, err) + } else { + logger.Info("Local task exists already", "Task.Name", task.Name) + } + + // Update job status based on owned task statuses; TODO That might not scale well in case we have many jobs + tasks := &klyshkov1alpha1.TupleGenerationTaskList{} + err = r.List(ctx, tasks) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to fetch task list: %w", err) + } + + // Collecting tasks owned by job; TODO Consider moving this to task class + isOwnedBy := func(task klyshkov1alpha1.TupleGenerationTask) bool { + for _, or := range task.OwnerReferences { + if or.Name == job.Name { + return true + } + } + return false + } + var ownedBy []klyshkov1alpha1.TupleGenerationTask + for _, t := range tasks.Items { + if isOwnedBy(t) { + ownedBy = append(ownedBy, t) + } + } + logger.Info("Collected statuses of owned tasks", "Tasks", ownedBy) + + // Helper functions; TODO Consider moving this to state class + allTerminated := func(tasks []klyshkov1alpha1.TupleGenerationTask) bool { + for _, t := range tasks { + if t.Status.State != klyshkov1alpha1.TaskCompleted && t.Status.State != klyshkov1alpha1.TaskFailed { + return false + } + } + return true + } + anyFailed := func(tasks []klyshkov1alpha1.TupleGenerationTask) bool { + for _, t := range tasks { + if t.Status.State == klyshkov1alpha1.TaskFailed { + return true + } + } + return false + } + nbrOfVCPs, err := numberOfVCPs(ctx, &r.Client, req.Namespace) + if err != nil { + return ctrl.Result{RequeueAfter: 60 * time.Second}, fmt.Errorf("can't read playerCount from VCP configuration: %w", err) + } + var state klyshkov1alpha1.TupleGenerationJobState + if uint(len(ownedBy)) < nbrOfVCPs { + state = klyshkov1alpha1.JobPending + } else if !allTerminated(ownedBy) { + state = klyshkov1alpha1.JobRunning + } else if anyFailed(ownedBy) { + state = klyshkov1alpha1.JobFailed + } else if job.Status.State != klyshkov1alpha1.JobCompleted { + state = klyshkov1alpha1.JobCompleted + + // Activate tuples; TODO How to deal with failures here? Introduce JobActivating state, how to sync between VCPs? + tupleChunkID, err := uuid.Parse(job.Spec.ID) + if err != nil { + return ctrl.Result{}, fmt.Errorf("invalid job id encountered '%v': %w", job.Spec.ID, err) + } + err = r.CastorClient.activateTupleChunk(ctx, tupleChunkID) + if err != nil { + return ctrl.Result{}, fmt.Errorf("tuple chunk activation failed for job %v: %w", job.Name, err) + } + } + if state.IsValid() && state != job.Status.State { + logger.Info("State update", "from", job.Status.State, "to", state) + job.Status.State = state + err = r.Status().Update(ctx, job) + if err != nil { + return ctrl.Result{}, fmt.Errorf("status update failed for job %v: %w", job.Name, err) + } + } + + logger.Info("Desired state reached") + return ctrl.Result{}, nil +} + +// taskName returns the name of the task for a given player associated with the given job. +func (r *TupleGenerationJobReconciler) taskName(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) string { + return job.Name + "-" + strconv.Itoa(int(playerID)) +} + +// taskForJob assembles the TupleGenerationJob resource description for the given job and VCP. +func (r *TupleGenerationJobReconciler) taskForJob(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) (*klyshkov1alpha1.TupleGenerationTask, error) { + task := &klyshkov1alpha1.TupleGenerationTask{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.taskName(job, playerID), + Namespace: job.Namespace, + }, + Spec: klyshkov1alpha1.TupleGenerationTaskSpec{}, + Status: klyshkov1alpha1.TupleGenerationTaskStatus{}, + } + err := ctrl.SetControllerReference(job, task, r.Scheme) + if err != nil { + return nil, fmt.Errorf("setting owner reference failed: %w", err) + } + return task, nil +} + +// handleWatchEvents handles incoming etcd events and dispatches them individually to handleWatchEvent. func (r *TupleGenerationJobReconciler) handleWatchEvents() { ctx := context.Background() for watchResponse := range r.rosterWatcherCh { @@ -55,20 +251,23 @@ func (r *TupleGenerationJobReconciler) handleWatchEvents() { } } +// handleWatchEvent inspects the given event and dispatches to handleRemoteTaskUpdate or handleJobUpdate based on the +// type of contained key. func (r *TupleGenerationJobReconciler) handleWatchEvent(ctx context.Context, ev *clientv3.Event) { - logger := log.FromContext(ctx) key, err := ParseKey(string(ev.Kv.Key)) if err != nil { - logger.Error(err, "unexpected etcd watch event", "Event.Key", ev.Kv.Key) + r.Logger.Error(err, "Unexpected etcd watch event", "Event.Key", ev.Kv.Key) + return } - logger.Info("processing roster event", "Key", key) + logger := r.Logger.WithValues("Key", key, "Value", string(ev.Kv.Value), "Type", ev.Type) + logger.Info("Processing roster event") switch k := key.(type) { case RosterEntryKey: // Skip if update is for local task local, err := isLocalTaskKey(ctx, &r.Client, k) if err != nil { - logger.Error(err, "failed to check task type") + logger.Error(err, "Failed to check task type") return } if local { @@ -81,23 +280,23 @@ func (r *TupleGenerationJobReconciler) handleWatchEvent(ctx context.Context, ev } func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key RosterKey, ev *clientv3.Event) { - logger := log.FromContext(ctx).WithValues("Key", key) + logger := r.Logger.WithValues("Key", key) switch ev.Type { case mvccpb.PUT: // Get job spec from etcd K/V pair jobSpec := &klyshkov1alpha1.TupleGenerationJobSpec{} err := json.Unmarshal(ev.Kv.Value, jobSpec) if err != nil { - logger.Error(err, "failed to unmarshal spec") + logger.Error(err, "Failed to unmarshal spec") return } // TODO Create or update depending on whether Job already exists err = r.createJobIfNotExists(ctx, key.NamespacedName, jobSpec) if err != nil { - logger.Error(err, "failed to create job") + logger.Error(err, "Failed to create job") return } - logger.Info("job created") + logger.Info("Job created") case mvccpb.DELETE: // Delete job iff exists found := &klyshkov1alpha1.TupleGenerationJob{} @@ -106,26 +305,27 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key if apierrors.IsNotFound(err) { return } - logger.Error(err, "failed to read job resource") + logger.Error(err, "Failed to read job resource") return } err = r.Delete(ctx, found) if err != nil { - logger.Error(err, "job deletion failed") + logger.Error(err, "Job deletion failed") return } - logger.Info("job deleted") + logger.Info("Job deleted") } } +// handleRemoteTaskUpdate is responsible for creating, updating, and deleting local tasks and proxies for remote tasks. func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Context, key RosterEntryKey, ev *clientv3.Event) { - logger := log.FromContext(ctx).WithValues("Task.Key", key) + logger := r.Logger.WithValues("Task.Key", key) // Lookup job job := &klyshkov1alpha1.TupleGenerationJob{} err := r.Get(ctx, key.NamespacedName, job) if err != nil { - logger.Error(err, "failed to read job resource") + logger.Error(err, "Failed to read job resource") return } @@ -134,6 +334,7 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex Name: r.taskName(job, key.PlayerID), } + // TODO Failure in one of the below handlers requires reconciliation, how to do that? switch ev.Type { case mvccpb.PUT: found := &klyshkov1alpha1.TupleGenerationTask{} @@ -141,55 +342,63 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex // Update local proxy task status status, err := klyshkov1alpha1.ParseFromJSON(ev.Kv.Value) if err != nil { - logger.Error(err, "extracting state from roster entry failed") + logger.Error(err, "Extracting state from roster entry failed", "Value", string(ev.Kv.Value)) return } found.Status = *status err = r.Client.Status().Update(ctx, found) if err != nil { - logger.Error(err, "failed to update proxy task") + logger.Error(err, "Failed to update proxy task") return } + logger.Info("Updated state", "State.New", status) } else { if !apierrors.IsNotFound(err) { - logger.Error(err, "failed to fetch task") // TODO Failure requires reconciliation (also above/below), how to do that? + logger.Error(err, "Failed to fetch task") return } // Create local proxy for remote task task, err := r.taskForJob(job, key.PlayerID) if err != nil { - logger.Error(err, "failed to define proxy task") + logger.Error(err, "Failed to define proxy task") return } err = r.Create(ctx, task) if err != nil { - logger.Error(err, "failed to create proxy task") + if apierrors.IsAlreadyExists(err) { + // Resource was not yet created when fetching at the beginning of the method, re-enqueue + r.handleRemoteTaskUpdate(ctx, key, ev) + } else { + logger.Error(err, "Failed to create proxy task") + } return } + logger.Info("Proxy task created") } case mvccpb.DELETE: - // Delete task for job iff exists + // Delete task for job if exists task := &klyshkov1alpha1.TupleGenerationTask{} err = r.Get(ctx, taskName, task) if err != nil { if apierrors.IsNotFound(err) { return } - logger.Error(err, "failed to read proxy task resource") + logger.Error(err, "Failed to read proxy task resource") return } err = r.Delete(ctx, task) if err != nil { - logger.Error(err, "proxy task deletion failed") + logger.Error(err, "Proxy task deletion failed") return } - logger.Info("proxy task deleted") + logger.Info("Proxy task deleted") } } +// createJobIfNotExists creates a job according to the given TupleGenerationJobSpec. func (r *TupleGenerationJobReconciler) createJobIfNotExists(ctx context.Context, name types.NamespacedName, jobSpec *klyshkov1alpha1.TupleGenerationJobSpec) error { - logger := log.FromContext(ctx).WithValues("Job.Name", name) + logger := r.Logger.WithValues("Job.Name", name) found := &klyshkov1alpha1.TupleGenerationJob{} err := r.Client.Get(ctx, name, found) if err != nil { @@ -201,7 +410,7 @@ func (r *TupleGenerationJobReconciler) createJobIfNotExists(ctx context.Context, }, Spec: *jobSpec, } - logger.Info("creating a new job") + logger.Info("Creating a new job") return r.Create(ctx, job) } return err @@ -209,188 +418,7 @@ func (r *TupleGenerationJobReconciler) createJobIfNotExists(ctx context.Context, return nil } -//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationjobs/finalizers,verbs=update -//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch - -func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - jobKey := RosterKey{ - req.NamespacedName, - } - logger := log.FromContext(ctx).WithValues("Job.Key", jobKey) - - // Cleanup if job has been deleted - job := &klyshkov1alpha1.TupleGenerationJob{} - err := r.Get(ctx, req.NamespacedName, job) - if err != nil { - if apierrors.IsNotFound(err) { - // Job resource not available -> has been deleted - _, err := r.EtcdClient.Delete(ctx, jobKey.ToEtcdKey()) - if err != nil { - logger.Error(err, "failed to delete roster") - return ctrl.Result{}, err - } - logger.Info("roster deleted") - return ctrl.Result{}, nil - } - // Error reading the object - requeue the request. - logger.Error(err, "failed to read job resource") - return ctrl.Result{}, err - } - logger.Info("job exists already") - - // Create roster if not existing (no etcd transaction needed as remote job creation is triggered by roster creation) - resp, err := r.EtcdClient.Get(ctx, jobKey.ToEtcdKey()) - if err != nil { - logger.Error(err, "failed to fetch roster", "Roster.Key", jobKey) - return ctrl.Result{}, err - } - if resp.Count == 0 { - encoded, err := json.Marshal(job.Spec) - if err != nil { - fmt.Println(err, "failed to marshal specification") - return ctrl.Result{}, err - } - _, err = r.EtcdClient.Put(ctx, jobKey.ToEtcdKey(), string(encoded)) - if err != nil { - logger.Error(err, "failed to create roster") - return ctrl.Result{}, err - } - logger.Info("roster created") - } else { - logger.Info("roster exists already") - } - - // Create local task if not existing - playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) - if err != nil { - logger.Error(err, "can't read playerId from VCP configuration") - return ctrl.Result{RequeueAfter: 60 * time.Second}, err - } - task := &klyshkov1alpha1.TupleGenerationTask{} - err = r.Get(ctx, types.NamespacedName{ - Namespace: job.Namespace, - Name: r.taskName(job, playerID), - }, task) - if err != nil { - if apierrors.IsNotFound(err) { - // Create new local task for job - task, err = r.taskForJob(job, playerID) - if err != nil { - logger.Error(err, "failed to define local task", "Task.Name", task.Name) - return ctrl.Result{}, err - } - err = r.Create(ctx, task) - if err != nil { - logger.Error(err, "failed to create local task", "Task.Name", task.Name) - return ctrl.Result{}, err - } - logger.Info("local task created", "Task.Name", task.Name) - return ctrl.Result{Requeue: true}, nil - } - // Error reading resource, requeue - logger.Error(err, "failed to read task resource") - return ctrl.Result{}, err - } else { - logger.Info("local task exists already") - } - - // Update job status based on owned task statuses - tasks := &klyshkov1alpha1.TupleGenerationTaskList{} - err = r.List(ctx, tasks) // TODO Use ListOption filter based on owner reference, if possible - if err != nil { - return ctrl.Result{}, err - } - - // Collecting tasks owned by job; TODO Move this to task class - isOwnedBy := func(task klyshkov1alpha1.TupleGenerationTask) bool { - for _, or := range task.OwnerReferences { - if or.Name == job.Name { - return true - } - } - return false - } - var ownedBy []klyshkov1alpha1.TupleGenerationTask - for _, t := range tasks.Items { - if isOwnedBy(t) { - ownedBy = append(ownedBy, t) - } - } - logger.Info("Collected statues of owned tasks", "States", ownedBy) - - // Helper functions; TODO Move this to state class? - allTerminated := func(tasks []klyshkov1alpha1.TupleGenerationTask) bool { - for _, t := range tasks { - if t.Status.State != klyshkov1alpha1.TaskCompleted && t.Status.State != klyshkov1alpha1.TaskFailed { - return false - } - } - return true - } - anyFailed := func(tasks []klyshkov1alpha1.TupleGenerationTask) bool { - for _, t := range tasks { - if t.Status.State == klyshkov1alpha1.TaskFailed { - return true - } - } - return false - } - nbrOfPlayers, err := numberOfPlayers(ctx, &r.Client, req.Namespace) - if err != nil { - logger.Error(err, "can't read playerCount from VCP configuration") - return ctrl.Result{RequeueAfter: 60 * time.Second}, err - } - var state klyshkov1alpha1.TupleGenerationJobState - if uint(len(ownedBy)) < nbrOfPlayers { - state = klyshkov1alpha1.JobPending - } else if !allTerminated(ownedBy) { - state = klyshkov1alpha1.JobRunning - } else if anyFailed(ownedBy) { - state = klyshkov1alpha1.JobFailed - } else { - state = klyshkov1alpha1.JobCompleted - - // Activate tuples; TODO How to deal with failures here? Introduce JobActivating state, how to sync between VCPs? Is activation still used in new Castor implementation? - tupleChunkID, err := uuid.Parse(job.Spec.ID) - if err != nil { - logger.Error(err, "invalid job id encountered") - return ctrl.Result{}, nil - } - err = r.CastorClient.activateTupleChunk(ctx, tupleChunkID) - if err != nil { - logger.Error(err, "tuple activation failed") - return ctrl.Result{}, nil - } - } - job.Status.State = state - err = r.Status().Update(ctx, job) - if err != nil { - return ctrl.Result{}, err - } - - logger.Info("desired state reached") - return ctrl.Result{}, nil -} - -func (r *TupleGenerationJobReconciler) taskName(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) string { - return job.Name + "-" + strconv.Itoa(int(playerID)) -} - -func (r *TupleGenerationJobReconciler) taskForJob(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) (*klyshkov1alpha1.TupleGenerationTask, error) { - task := &klyshkov1alpha1.TupleGenerationTask{ - ObjectMeta: metav1.ObjectMeta{ - Name: r.taskName(job, playerID), - Namespace: job.Namespace, - }, - Spec: klyshkov1alpha1.TupleGenerationTaskSpec{}, - Status: klyshkov1alpha1.TupleGenerationTaskStatus{}, - } - err := ctrl.SetControllerReference(job, task, r.Scheme) - return task, err -} - +// SetupWithManager sets up the controller with the Manager. func (r *TupleGenerationJobReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&klyshkov1alpha1.TupleGenerationJob{}). diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index b87d19f..30c7df5 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -9,6 +9,7 @@ package controllers import ( "context" + "fmt" "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,8 +24,10 @@ import ( klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" ) +// MinimumTuplesPerJob is the minimum number of tuples generated by a TupleGenerationJob. const MinimumTuplesPerJob = 10000 +// TupleGenerationSchedulerReconciler reconciles a TupleGenerationScheduler object. type TupleGenerationSchedulerReconciler struct { client.Client Scheme *runtime.Scheme @@ -35,8 +38,11 @@ type TupleGenerationSchedulerReconciler struct { //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationschedulers/status,verbs=get;update;patch //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationschedulers/finalizers,verbs=update +// Reconcile compares the actual state of TupleGenerationScheduler resources to their desired state and performs actions +// to bring the actual state closer to the desired one. func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) + logger.Info("Reconciling tuple generation schedulers") // Fetch scheduler resource scheduler := &klyshkov1alpha1.TupleGenerationScheduler{} @@ -47,60 +53,76 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, nil } // Error reading the object - requeue the request. - logger.Error(err, "failed to read scheduler resource") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to read scheduler resource: %w", err) } + // Remove all finished jobs + if r.cleanupFinishedJobs(ctx) != nil { + return ctrl.Result{}, fmt.Errorf("failed to delete finished jobs: %w", err) + } + + // Fetch active jobs activeJobs, err := r.getMatchingJobs(ctx, func(job klyshkov1alpha1.TupleGenerationJob) bool { return !job.Status.State.IsDone() }) if err != nil { - logger.Error(err, "failed to fetch active jobs") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to fetch active jobs: %w", err) } // Stop if already at maximum concurrency level activeJobCount := len(activeJobs) if scheduler.Spec.Concurrency <= activeJobCount { - logger.Info("at maximum concurrency level", "Jobs.Active", activeJobCount, "Scheduler.Concurrency", scheduler.Spec.Concurrency) + logger.Info("At maximum concurrency level", "Jobs.Active", activeJobCount, "Scheduler.Concurrency", scheduler.Spec.Concurrency) return ctrl.Result{RequeueAfter: 60 * time.Second}, nil } - // Least available first strategy for selecting tuples to create a job for + // Least available first strategy for selecting tuple types to create a job for // 1. Compute available and in generation number of tuples per type // 2. Filter out all above threshold // 3. Sort ascending wrt to sum from step 1 telemetry, err := r.CastorClient.getTelemetry(ctx) if err != nil { + logger.Error(err, "Fetching telemetry data from Castor failed", "Castor.URL", r.CastorClient.URL) return ctrl.Result{RequeueAfter: 60 * time.Second}, err } - logger.Info("tuple telemetry data fetched", "Metrics.Available", telemetry.TupleMetrics) + logger.Info("Tuple telemetry data fetched", "Metrics.Available", telemetry.TupleMetrics) for _, j := range activeJobs { for idx := range telemetry.TupleMetrics { if j.Spec.Type == telemetry.TupleMetrics[idx].TupleType { - telemetry.TupleMetrics[idx].Available = telemetry.TupleMetrics[idx].Available + int(j.Spec.Count) + telemetry.TupleMetrics[idx].Available = telemetry.TupleMetrics[idx].Available + j.Spec.Count break } } } - logger.Info("with in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) + logger.Info("With in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) var belowThreshold []TupleMetrics for _, m := range telemetry.TupleMetrics { if m.Available < scheduler.Spec.Threshold { belowThreshold = append(belowThreshold, m) } } - logger.Info("filtered for eligible types", "Metrics.Eligible", belowThreshold) + logger.Info("Filtered for eligible types", "Metrics.Eligible", belowThreshold) if len(belowThreshold) == 0 { - logger.Info("above threshold for all tuple types - do nothing") + logger.Info("Above threshold for all tuple types - do nothing") return ctrl.Result{RequeueAfter: 60 * time.Second}, nil } sort.Slice(belowThreshold, func(i, j int) bool { return belowThreshold[i].Available < belowThreshold[j].Available }) - logger.Info("sorted by priority", "Metrics.Sorted", belowThreshold) + logger.Info("Sorted by priority", "Metrics.Sorted", belowThreshold) + + // Create job for first tuple type below threshold + err = r.createJob(ctx, scheduler, belowThreshold[0].TupleType) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create tuple generation job: %w", err) + } + + return ctrl.Result{}, nil +} - // Create job for first below threshold +// Creates a tuple generation job for the given tuple type in the namespace where the scheduler lives in. +func (r *TupleGenerationSchedulerReconciler) createJob(ctx context.Context, scheduler *klyshkov1alpha1.TupleGenerationScheduler, tupleType string) error { + logger := log.FromContext(ctx) jobID := uuid.New().String() job := &klyshkov1alpha1.TupleGenerationJob{ TypeMeta: metav1.TypeMeta{ @@ -109,51 +131,61 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req }, ObjectMeta: metav1.ObjectMeta{ Name: scheduler.Name + "-" + jobID, - Namespace: req.Namespace, + Namespace: scheduler.Namespace, }, Spec: klyshkov1alpha1.TupleGenerationJobSpec{ ID: jobID, - Type: belowThreshold[0].TupleType, + Type: tupleType, Count: MinimumTuplesPerJob, // TODO Make this configurable }, + Status: klyshkov1alpha1.TupleGenerationJobStatus{ + State: klyshkov1alpha1.JobPending, + }, } - err = ctrl.SetControllerReference(scheduler, job, r.Scheme) + err := ctrl.SetControllerReference(scheduler, job, r.Scheme) if err != nil { logger.Error(err, "could not set owner reference on job", "Job", job) - return ctrl.Result{}, err + return err } err = r.Create(ctx, job) if err != nil { logger.Error(err, "job creation failed", "Job", job) - return ctrl.Result{}, err + return err } - logger.Info("job created", "Job", job) + logger.Info("Job created", "Job", job) + return nil +} - // Delete all finished jobs +// Deletes all jobs that are done, i.e., either complete or failed +func (r *TupleGenerationSchedulerReconciler) cleanupFinishedJobs(ctx context.Context) error { + logger := log.FromContext(ctx) finishedJobs, err := r.getMatchingJobs(ctx, func(job klyshkov1alpha1.TupleGenerationJob) bool { return job.Status.State.IsDone() }) if err != nil { logger.Error(err, "failed to fetch finished jobs") - return ctrl.Result{}, err + return err } + logger.Info("Deleting finished jobs", "jobs", finishedJobs) for _, j := range finishedJobs { err := r.Delete(ctx, &j) if err != nil { logger.Error(err, "failed to delete finished job", "Job", j) - return ctrl.Result{}, err + return err } } - - return ctrl.Result{}, nil + return nil } +// Returns all jobs that match the given predicate. func (r *TupleGenerationSchedulerReconciler) getMatchingJobs(ctx context.Context, pred func(klyshkov1alpha1.TupleGenerationJob) bool) ([]klyshkov1alpha1.TupleGenerationJob, error) { + logger := log.FromContext(ctx) allJobs := &klyshkov1alpha1.TupleGenerationJobList{} err := r.List(ctx, allJobs) if err != nil { return nil, err } + logger.Info("Considering potentially matching jobs", "jobs", allJobs) var matchingJobs []klyshkov1alpha1.TupleGenerationJob for _, j := range allJobs.Items { if pred(j) { @@ -163,6 +195,7 @@ func (r *TupleGenerationSchedulerReconciler) getMatchingJobs(ctx context.Context return matchingJobs, nil } +// SetupWithManager sets up the controller with the Manager. func (r *TupleGenerationSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&klyshkov1alpha1.TupleGenerationScheduler{}). diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 20970b9..ed6ec0b 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -28,88 +28,34 @@ import ( klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" ) +// TupleGenerationTaskReconciler reconciles a TupleGenerationTask object. type TupleGenerationTaskReconciler struct { client.Client Scheme *runtime.Scheme EtcdClient *clientv3.Client } -func (r *TupleGenerationTaskReconciler) taskKeyFromName(namespace string, name string) (*RosterEntryKey, error) { - p := strings.Split(name, "-") - pid := p[len(p)-1] - jobName := strings.Join(p[:len(p)-1], "-") - key, err := ParseKey(rosterKey + "/" + namespace + "/" + jobName + "/" + pid) - if err != nil { - return nil, err - } - rev, ok := key.(RosterEntryKey) - if !ok { - return nil, fmt.Errorf("not a task key: %v", key) - } - return &rev, nil -} - -func isLocalTaskKey(ctx context.Context, client *client.Client, key RosterEntryKey) (bool, error) { - playerID, err := localPlayerID(ctx, client, key.Namespace) - if err != nil { - return false, fmt.Errorf("can't read local player ID: %w", err) - } - return playerID == key.PlayerID, nil -} - -func (r *TupleGenerationTaskReconciler) getStatus(ctx context.Context, taskKey RosterEntryKey) (*klyshkov1alpha1.TupleGenerationTaskStatus, error) { - resp, err := r.EtcdClient.Get(ctx, taskKey.ToEtcdKey()) - if err != nil { - return nil, err - } - if len(resp.Kvs) != 1 { - return nil, fmt.Errorf("no status available for roster entry: %v", taskKey) - } - status, err := klyshkov1alpha1.ParseFromJSON(resp.Kvs[0].Value) - if err != nil { - return nil, err - } - if !status.State.IsValid() { - return nil, fmt.Errorf("status contains invalid state: %s", status.State) - } - return status, nil -} - -func (r *TupleGenerationTaskReconciler) setStatus(ctx context.Context, taskKey RosterEntryKey, status *klyshkov1alpha1.TupleGenerationTaskStatus) error { - encoded, err := json.Marshal(status) - if err != nil { - return err - } - _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), string(encoded)) - return err -} - -func (r *TupleGenerationTaskReconciler) setState(ctx context.Context, taskKey RosterEntryKey, status *klyshkov1alpha1.TupleGenerationTaskStatus, state klyshkov1alpha1.TupleGenerationTaskState) error { - logger := log.FromContext(ctx).WithValues("Task.Key", taskKey) - logger.Info("task transitioning into new state", "from", status.State, "to", state) - status.State = state - return r.setStatus(ctx, taskKey, status) -} - //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/status,verbs=get;update;patch //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete +// Reconcile compares the actual state of TupleGenerationTask resources to their desired state and performs actions to +// bring the actual state closer to the desired one. func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues("Task.Name", req.Name) + logger.Info("Reconciling tuple generation tasks") + taskKey, err := r.taskKeyFromName(req.Namespace, req.Name) if err != nil { - logger.Error(err, "failed to get task key") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to get key for task %v: %w", req.Name, err) } // Skip if remote task local, err := isLocalTaskKey(ctx, &r.Client, *taskKey) if err != nil { - logger.Error(err, "failed to check task type") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to check type for task %v: %w", req.Name, err) } if !local { return ctrl.Result{}, nil @@ -123,88 +69,81 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. // Task resource not available -> has been deleted _, err := r.EtcdClient.Delete(ctx, taskKey.ToEtcdKey()) if err != nil { - logger.Error(err, "failed to delete roster entry") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to delete roster entry for task %v: %w", req.Name, err) } - logger.Info("roster entry deleted") + logger.Info("Roster entry deleted") return ctrl.Result{}, nil } // Error reading the object - requeue the request. - logger.Error(err, "failed to read task resource") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to read resource for task %v: %w", req.Name, err) } - logger.Info("task exists already") + logger.Info("Task exists already") // Create roster entry if not existing resp, err := r.EtcdClient.Get(ctx, taskKey.ToEtcdKey()) if err != nil { - logger.Error(err, "failed to fetch roster entry", "RosterEntry.Key", taskKey) - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to read resource for roster entry with key %v for task %v: %w", taskKey, req.Name, err) } if resp.Count == 0 { status, err := json.Marshal(&klyshkov1alpha1.TupleGenerationTaskStatus{State: klyshkov1alpha1.TaskLaunching}) _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), string(status)) if err != nil { - logger.Error(err, "failed to create roster entry") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to create roster entry for task %v: %w", req.Name, err) } - logger.Info("roster entry created") + logger.Info("Roster entry created") } else { - logger.Info("roster entry exists already") + logger.Info("Roster entry exists already") } status, err := r.getStatus(ctx, *taskKey) if err != nil { - logger.Error(err, "task status not available") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("status not available for task %v: %w", req.Name, err) } - // Lookup job + // Lookup job that owns the task job := &klyshkov1alpha1.TupleGenerationJob{} err = r.Get(ctx, taskKey.NamespacedName, job) if err != nil { - logger.Error(err, "failed to lookup job") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to lookup job for task %v: %w", req.Name, err) } - // Update the status according to state in etcd + // Update the task status according to state in etcd taskStatus, err := r.getStatus(ctx, *taskKey) if err != nil { return ctrl.Result{}, err } task.Status = *taskStatus if err := r.Status().Update(ctx, task); err != nil { - logger.Error(err, "unable to update task status") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("unable to update status for task %v: %w", req.Name, err) } - if status.State == klyshkov1alpha1.TaskLaunching { + // Proceed based on current task state + switch status.State { + case klyshkov1alpha1.TaskLaunching: // Create persistent volume claim used to store generated tuples, if not existing err = r.createPVC(ctx, taskKey) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("unable to create PVC for task %v: %w", req.Name, err) } // Create generator pod if not existing _, err := r.createGeneratorPod(ctx, *taskKey, job, task) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("unable to create generator pod for task %v: %w", req.Name, err) } return ctrl.Result{ Requeue: true, }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskGenerating) - } - - if status.State == klyshkov1alpha1.TaskGenerating { + case klyshkov1alpha1.TaskGenerating: genPod, err := r.getGeneratorPod(ctx, task) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("unable to get generator pod for task %v: %w", req.Name, err) } switch genPod.Status.Phase { case v1.PodSucceeded: // Generation successful, create provisioner pod to upload tuple shares to VCP-local castor _, err := r.createProvisionerPod(ctx, *taskKey, job, task) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("unable to create provisioner pod for task %v: %w", req.Name, err) } return ctrl.Result{ Requeue: true, @@ -214,12 +153,10 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. Requeue: true, }, r.setState(ctx, *taskKey, status, klyshkov1alpha1.TaskFailed) } - } - - if status.State == klyshkov1alpha1.TaskProvisioning { + case klyshkov1alpha1.TaskProvisioning: provPod, err := r.getProvisionerPod(ctx, *taskKey) if err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("unable to get provisioner pod for task %v: %w", req.Name, err) } switch provPod.Status.Phase { case v1.PodSucceeded: @@ -233,10 +170,11 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. } } - logger.Info("desired state reached") + logger.Info("Desired state reached") return ctrl.Result{}, nil } +// SetupWithManager sets up the controller with the Manager. func (r *TupleGenerationTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&klyshkov1alpha1.TupleGenerationTask{}). @@ -244,10 +182,79 @@ func (r *TupleGenerationTaskReconciler) SetupWithManager(mgr ctrl.Manager) error Complete(r) } +// taskKeyFromName creates a RosterEntryKey from the given name and namespace. Expects that the zero-based VCP +// identifier is appended with a hyphen to the name. +func (r *TupleGenerationTaskReconciler) taskKeyFromName(namespace string, name string) (*RosterEntryKey, error) { + parts := strings.Split(name, "-") + vcpId := parts[len(parts)-1] + jobName := strings.Join(parts[:len(parts)-1], "-") + keyStr := rosterKey + "/" + namespace + "/" + jobName + "/" + vcpId + key, err := ParseKey(keyStr) + if err != nil { + return nil, fmt.Errorf("can't parse task key from '%s': %w", keyStr, err) + } + rev, ok := key.(RosterEntryKey) + if !ok { + return nil, fmt.Errorf("not a task key: %v", key) + } + return &rev, nil +} + +// isLocalTaskKey checks whether a given key is the key of task running in the local VCP. +func isLocalTaskKey(ctx context.Context, client *client.Client, key RosterEntryKey) (bool, error) { + playerID, err := localPlayerID(ctx, client, key.Namespace) + if err != nil { + return false, fmt.Errorf("can't read local player ID: %w", err) + } + return playerID == key.PlayerID, nil +} + +// getStatus reads the task status from the respective value stored in etcd. +func (r *TupleGenerationTaskReconciler) getStatus(ctx context.Context, taskKey RosterEntryKey) (*klyshkov1alpha1.TupleGenerationTaskStatus, error) { + resp, err := r.EtcdClient.Get(ctx, taskKey.ToEtcdKey()) + if err != nil { + return nil, fmt.Errorf("can't get status from etcd: %w", err) + } + if len(resp.Kvs) != 1 { + return nil, fmt.Errorf("no status available for roster entry: %v", taskKey) + } + status, err := klyshkov1alpha1.ParseFromJSON(resp.Kvs[0].Value) + if err != nil { + return nil, fmt.Errorf("parsing status from '%s' failed: %w", string(resp.Kvs[0].Value), err) + } + if !status.State.IsValid() { + return nil, fmt.Errorf("status contains invalid state: %s", status.State) + } + return status, nil +} + +// setStatus writes the given status to etcd. +func (r *TupleGenerationTaskReconciler) setStatus(ctx context.Context, taskKey RosterEntryKey, status *klyshkov1alpha1.TupleGenerationTaskStatus) error { + encoded, err := json.Marshal(status) + if err != nil { + return fmt.Errorf("marshalling status failed: %w", err) + } + _, err = r.EtcdClient.Put(ctx, taskKey.ToEtcdKey(), string(encoded)) + if err != nil { + return fmt.Errorf("storing marshalled status in etcd failed: %w", err) + } + return nil +} + +// setState updates the given status object with the given state and writes the status to etcd. +func (r *TupleGenerationTaskReconciler) setState(ctx context.Context, taskKey RosterEntryKey, status *klyshkov1alpha1.TupleGenerationTaskStatus, state klyshkov1alpha1.TupleGenerationTaskState) error { + logger := log.FromContext(ctx).WithValues("Task.Key", taskKey) + logger.Info("Task transitioning into new state", "from", status.State, "to", state) + status.State = state + return r.setStatus(ctx, taskKey, status) +} + +// pvcName returns the name of the PVC used for the task with the given key. func pvcName(key RosterEntryKey) string { return key.Name + "-" + strconv.Itoa(int(key.PlayerID)) } +// createPVC creates a PVC used to transfer tuples between generator and provision pod for a task with the given key. func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key *RosterEntryKey) error { logger := log.FromContext(ctx).WithValues("Task.Key", key) name := types.NamespacedName{ @@ -257,7 +264,7 @@ func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key *Rost found := &v1.PersistentVolumeClaim{} err := r.Get(ctx, name, found) if err == nil { - logger.Info("persistent volume claim already exists") + logger.Info("Persistent volume claim already exists") return nil } pvc := &v1.PersistentVolumeClaim{ @@ -271,24 +278,25 @@ func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key *Rost }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "storage": resource.MustParse("100Mi"), + "storage": resource.MustParse("100Mi"), // TODO Make configurable }, }, }, } - logger.Info("creating persistent volume claim", "PVC", pvc) + logger.Info("Creating persistent volume claim", "PVC", pvc) err = r.Create(ctx, pvc) if err != nil { - logger.Error(err, "persistent volume claim creation failed") - return err + return fmt.Errorf("persistent volume claim creation failed for task %v: %w", key, err) } return nil } +// provisionerPodName returns the name for the provisioner pod used for the task with the given key. func provisionerPodName(key RosterEntryKey) string { return key.Name + "-provisioner" } +// getProvisionerPod returns the provisioner pod for the task with given key. func (r *TupleGenerationTaskReconciler) getProvisionerPod(ctx context.Context, key RosterEntryKey) (*v1.Pod, error) { name := types.NamespacedName{ Name: provisionerPodName(key), @@ -296,9 +304,14 @@ func (r *TupleGenerationTaskReconciler) getProvisionerPod(ctx context.Context, k } found := &v1.Pod{} err := r.Get(ctx, name, found) - return found, err + if err != nil { + return nil, fmt.Errorf("can't get the provisioner pod for task %v: %w", name, err) + } + return found, nil } +// createProvisionerPod creates a provisioner pod for the task with given key. The pod takes the tuples from the PV +// shared with the respective generator pod and uploads them to Castor. func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context, key RosterEntryKey, job *klyshkov1alpha1.TupleGenerationJob, task *klyshkov1alpha1.TupleGenerationTask) (*v1.Pod, error) { logger := log.FromContext(ctx).WithValues("Task.Key", key) name := types.NamespacedName{ @@ -307,7 +320,7 @@ func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context } found, err := r.getProvisionerPod(ctx, key) if err == nil { - logger.Info("provisioner pod already exists") + logger.Info("Provisioner pod already exists") return found, nil } pod := &v1.Pod{ @@ -318,7 +331,7 @@ func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context Spec: v1.PodSpec{ Containers: []v1.Container{{ Name: "generator", - Image: "carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT", + Image: "carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT", // TODO Read from config Env: []v1.EnvVar{ { Name: "KII_JOB_ID", @@ -353,36 +366,45 @@ func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context }, }, } - logger.Info("creating provisioner pod", "Pod", pod) + logger.Info("Creating provisioner pod", "Pod", pod) err = ctrl.SetControllerReference(task, pod, r.Scheme) if err != nil { - logger.Error(err, "setting owner reference failed") - return nil, err + return nil, fmt.Errorf("setting the owner reference for task %v failed: %w", name, err) } err = r.Create(ctx, pod) if err != nil { - logger.Error(err, "provisioner pod creation failed") - return nil, err + return nil, fmt.Errorf("pod creation for task %v failed: %w", name, err) } return pod, nil } +// getGeneratorPod returns the generator pod for the task with given key. func (r *TupleGenerationTaskReconciler) getGeneratorPod(ctx context.Context, task *klyshkov1alpha1.TupleGenerationTask) (*v1.Pod, error) { found := &v1.Pod{} - err := r.Get(ctx, types.NamespacedName{ + name := types.NamespacedName{ Name: task.Name, Namespace: task.Namespace, - }, found) - return found, err + } + err := r.Get(ctx, name, found) + if err != nil { + return nil, fmt.Errorf("can't get the generator pod for task %v: %w", name, err) + } + return found, nil } +// createGeneratorPod creates a generator pod for the task with given key. The pod generates tuples according to the +// parameter of the given TupleGenerationJob and stores them on the PV shared with the respective provisioner pod. func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, key RosterEntryKey, job *klyshkov1alpha1.TupleGenerationJob, task *klyshkov1alpha1.TupleGenerationTask) (*v1.Pod, error) { logger := log.FromContext(ctx).WithValues("Task.Key", key) found, err := r.getGeneratorPod(ctx, task) if err == nil { - logger.Info("pod already exists") + logger.Info("Pod already exists") return found, nil } + vcpCount, err := numberOfVCPs(ctx, &r.Client, job.Namespace) + if err != nil { + return found, fmt.Errorf("can't get number of VCPs for task %v: %w", task.Name, err) + } pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: task.Name, @@ -407,7 +429,7 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, }, { Name: "KII_PLAYER_COUNT", - Value: "2", // TODO Read from config + Value: strconv.Itoa(int(vcpCount)), }, { Name: "KII_TUPLE_TYPE", @@ -423,7 +445,7 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, }, { Name: "KII_SHARED_FOLDER", - Value: "/kii", // TODO Read from config + Value: "/kii", }, { Name: "KII_TUPLE_FILE", @@ -494,16 +516,14 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, }, }, } - logger.Info("creating pod", "Pod", pod) + logger.Info("Creating generator pod", "Pod", pod) err = ctrl.SetControllerReference(task, pod, r.Scheme) if err != nil { - logger.Error(err, "setting owner reference failed") - return nil, err + return nil, fmt.Errorf("setting the owner reference for task %v failed: %w", task.Name, err) } err = r.Create(ctx, pod) if err != nil { - logger.Error(err, "pod creation failed") - return nil, err + return nil, fmt.Errorf("pod creation for task %v failed: %w", task.Name, err) } return pod, nil } diff --git a/klyshko-operator/go.mod b/klyshko-operator/go.mod index aa89a92..2c4e408 100644 --- a/klyshko-operator/go.mod +++ b/klyshko-operator/go.mod @@ -3,6 +3,7 @@ module github.com/carbynestack/klyshko go 1.16 require ( + github.com/go-logr/logr v0.4.0 github.com/google/uuid v1.1.2 github.com/jarcoal/httpmock v1.2.0 github.com/onsi/ginkgo/v2 v2.1.4 @@ -12,6 +13,6 @@ require ( k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 sigs.k8s.io/controller-runtime v0.9.2 ) From 0f7eb64b99578b00fe640a8be5c40a49d820dfb3 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 29 Jul 2022 17:24:30 +0200 Subject: [PATCH 31/69] Verify eventual proxy task deletion Signed-off-by: Sven Trieflinger --- .../controllers/controller_test.go | 18 +++++++ .../tuplegenerationjob_controller.go | 47 +++++++++++-------- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go index e9046d5..b954ae1 100644 --- a/klyshko-operator/controllers/controller_test.go +++ b/klyshko-operator/controllers/controller_test.go @@ -281,6 +281,24 @@ var _ = Describe("Generating tuples", func() { }, Timeout, PollingInterval).Should(BeTrue()) } + // Ensure that proxy tasks get deleted on all VCPs eventually after local tasks are deleted. + for i := 0; i < NumberOfVCPs; i++ { + Expect(vc.vcps[i].k8sClient.Delete(ctx, &localTasksByVCP[i])).Should(Succeed()) + for j := 0; j < NumberOfVCPs; j++ { + if i == j { + continue + } + key := client.ObjectKey{ + Namespace: jobs[j].GetNamespace(), + Name: fmt.Sprintf("%s-%d", jobs[j].GetName(), i), + } + proxyTask := &klyshkov1alpha1.TupleGenerationTask{} + Eventually(func() bool { + return apierrors.IsNotFound(vc.vcps[j].k8sClient.Get(ctx, key, proxyTask)) + }, Timeout, PollingInterval).Should(BeTrue()) + } + } + }) }) }) diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 841bb34..496bff4 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -119,7 +119,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R task := &klyshkov1alpha1.TupleGenerationTask{} err = r.Get(ctx, types.NamespacedName{ Namespace: job.Namespace, - Name: r.taskName(job, playerID), + Name: r.taskNameForJob(job, playerID), }, task) if err != nil { if apierrors.IsNotFound(err) { @@ -219,16 +219,21 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } -// taskName returns the name of the task for a given player associated with the given job. -func (r *TupleGenerationJobReconciler) taskName(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) string { - return job.Name + "-" + strconv.Itoa(int(playerID)) +// taskNameForJob returns the name of the task for a given player associated with the given job. +func (r *TupleGenerationJobReconciler) taskNameForJob(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) string { + return r.taskName(job.Name, playerID) +} + +// taskName returns the name of the task for a given player derived from a given job name. +func (r *TupleGenerationJobReconciler) taskName(jobName string, playerID uint) string { + return jobName + "-" + strconv.Itoa(int(playerID)) } // taskForJob assembles the TupleGenerationJob resource description for the given job and VCP. func (r *TupleGenerationJobReconciler) taskForJob(job *klyshkov1alpha1.TupleGenerationJob, playerID uint) (*klyshkov1alpha1.TupleGenerationTask, error) { task := &klyshkov1alpha1.TupleGenerationTask{ ObjectMeta: metav1.ObjectMeta{ - Name: r.taskName(job, playerID), + Name: r.taskNameForJob(job, playerID), Namespace: job.Namespace, }, Spec: klyshkov1alpha1.TupleGenerationTaskSpec{}, @@ -321,22 +326,22 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Context, key RosterEntryKey, ev *clientv3.Event) { logger := r.Logger.WithValues("Task.Key", key) - // Lookup job - job := &klyshkov1alpha1.TupleGenerationJob{} - err := r.Get(ctx, key.NamespacedName, job) - if err != nil { - logger.Error(err, "Failed to read job resource") - return - } - - taskName := types.NamespacedName{ - Namespace: key.Namespace, - Name: r.taskName(job, key.PlayerID), - } - // TODO Failure in one of the below handlers requires reconciliation, how to do that? switch ev.Type { case mvccpb.PUT: + + // Lookup job + job := &klyshkov1alpha1.TupleGenerationJob{} + err := r.Get(ctx, key.NamespacedName, job) + if err != nil { + logger.Error(err, "Failed to read job resource") + return + } + taskName := types.NamespacedName{ + Namespace: key.Namespace, + Name: r.taskNameForJob(job, key.PlayerID), + } + found := &klyshkov1alpha1.TupleGenerationTask{} if err := r.Client.Get(ctx, taskName, found); err == nil { // Update local proxy task status @@ -378,8 +383,12 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex } case mvccpb.DELETE: // Delete task for job if exists + taskName := types.NamespacedName{ + Namespace: key.Namespace, + Name: r.taskName(key.Name, key.PlayerID), + } task := &klyshkov1alpha1.TupleGenerationTask{} - err = r.Get(ctx, taskName, task) + err := r.Get(ctx, taskName, task) if err != nil { if apierrors.IsNotFound(err) { return From f9ae3aa6d7f71a8d2651b54abadb19ddfeccc8cf Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 2 Aug 2022 14:22:56 +0200 Subject: [PATCH 32/69] Update SBOM and fullfil OSS obligations Signed-off-by: Sven Trieflinger --- .../github.com/jarcoal/httpmock/LICENSE | 20 +++++++++++++ .../github.com/nxadm/tail/LICENSE | 21 -------------- .../github.com/nxadm/tail/ratelimiter/Licence | 7 ----- .../stenographer/support/go-colorable/LICENSE | 21 -------------- .../stenographer/support/go-isatty/LICENSE | 9 ------ .../github.com/onsi/ginkgo/{ => v2}/LICENSE | 0 .../gopkg.in/tomb.v1/LICENSE | 29 ------------------- klyshko-operator/3RD-PARTY-LICENSES/sbom.json | 26 ++++------------- 8 files changed, 25 insertions(+), 108 deletions(-) create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/jarcoal/httpmock/LICENSE delete mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE delete mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence delete mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE delete mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE rename klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/{ => v2}/LICENSE (100%) delete mode 100644 klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/jarcoal/httpmock/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/jarcoal/httpmock/LICENSE new file mode 100644 index 0000000..438fbf5 --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/jarcoal/httpmock/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Jared Morse + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE deleted file mode 100644 index 818d802..0000000 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -# The MIT License (MIT) - -# © Copyright 2015 Hewlett Packard Enterprise Development LP -Copyright (c) 2014 ActiveState - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence b/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence deleted file mode 100644 index 434aab1..0000000 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/nxadm/tail/ratelimiter/Licence +++ /dev/null @@ -1,7 +0,0 @@ -Copyright (C) 2013 99designs - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE deleted file mode 100644 index 91b5cef..0000000 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Yasuhiro Matsumoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE deleted file mode 100644 index 65dc692..0000000 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty/LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -Copyright (c) Yasuhiro MATSUMOTO - -MIT License (Expat) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/v2/LICENSE similarity index 100% rename from klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/LICENSE rename to klyshko-operator/3RD-PARTY-LICENSES/github.com/onsi/ginkgo/v2/LICENSE diff --git a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE deleted file mode 100644 index a4249bb..0000000 --- a/klyshko-operator/3RD-PARTY-LICENSES/gopkg.in/tomb.v1/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -tomb - support for clean goroutine termination in Go. - -Copyright (c) 2010-2011 - Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/klyshko-operator/3RD-PARTY-LICENSES/sbom.json b/klyshko-operator/3RD-PARTY-LICENSES/sbom.json index f98a25f..3b91726 100644 --- a/klyshko-operator/3RD-PARTY-LICENSES/sbom.json +++ b/klyshko-operator/3RD-PARTY-LICENSES/sbom.json @@ -36,15 +36,7 @@ "license": "Apache-2.0" }, { - "project": "vendor/github.com/onsi/ginkgo", - "license": "MIT" - }, - { - "project": "vendor/github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty", - "license": "MIT" - }, - { - "project": "vendor/github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable", + "project": "vendor/github.com/onsi/ginkgo/v2", "license": "MIT" }, { @@ -75,14 +67,6 @@ "project": "vendor/github.com/coreos/go-semver", "license": "Apache-2.0" }, - { - "project": "vendor/github.com/nxadm/tail", - "license": "MIT" - }, - { - "project": "vendor/github.com/nxadm/tail/ratelimiter", - "license": "MIT" - }, { "project": "vendor/github.com/json-iterator/go", "license": "MIT" @@ -111,6 +95,10 @@ "project": "vendor/github.com/imdario/mergo", "license": "BSD-3-Clause" }, + { + "project": "vendor/github.com/jarcoal/httpmock", + "license": "MIT" + }, { "project": "vendor/github.com/golang/groupcache", "license": "Apache-2.0" @@ -283,10 +271,6 @@ "project": "vendor/gopkg.in/yaml.v3", "license": "Apache-2.0" }, - { - "project": "vendor/gopkg.in/tomb.v1", - "license": "BSD-3-Clause" - }, { "project": "vendor/gopkg.in/inf.v0", "license": "BSD-3-Clause" From c02217f266a9059479c5fe4d5c509435ffc0174b Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 2 Aug 2022 14:25:25 +0200 Subject: [PATCH 33/69] Fix SCA findings Signed-off-by: Sven Trieflinger --- .../api/v1alpha1/tuplegenerationjob_types.go | 16 +++++++---- .../tuplegenerationscheduler_types.go | 4 +++ .../api/v1alpha1/tuplegenerationtask_types.go | 27 +++++++++++++++---- klyshko-operator/controllers/castor_client.go | 5 ++-- .../controllers/castor_client_test.go | 12 ++++----- .../controllers/controller_test.go | 8 +++--- .../tuplegenerationjob_controller.go | 9 +++---- .../tuplegenerationtask_controller.go | 6 ++--- klyshko-operator/main.go | 6 ++--- 9 files changed, 59 insertions(+), 34 deletions(-) diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go index 2dc8221..e1f632f 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// TupleGenerationJobState encodes the state of a TupleGenerationJob. type TupleGenerationJobState string const ( @@ -18,17 +19,16 @@ const ( JobPending TupleGenerationJobState = "Pending" // JobRunning means all tasks for the job have been spawned but have not terminated yet. - JobRunning = "Running" + JobRunning TupleGenerationJobState = "Running" // JobCompleted means all tasks have completed successfully. - JobCompleted = "Completed" + JobCompleted TupleGenerationJobState = "Completed" // JobFailed means that all tasks for the job have terminated but at least on failed. - JobFailed = "Failed" + JobFailed TupleGenerationJobState = "Failed" ) -// IsValid returns true if this state is among the defined ones and false -// otherwise. +// IsValid returns true if state s is among the defined ones and false otherwise. func (s TupleGenerationJobState) IsValid() bool { switch s { case JobPending, JobRunning, JobCompleted, JobFailed: @@ -38,10 +38,13 @@ func (s TupleGenerationJobState) IsValid() bool { } } +// IsDone returns true if s is among the set of TupleGenerationJobState that describe a job that is done, i.e., is +// either JobCompleted or JobFailed, and false otherwise. func (s TupleGenerationJobState) IsDone() bool { return s == JobCompleted || s == JobFailed } +// TupleGenerationJobSpec defines the desired state of a TupleGenerationJob. type TupleGenerationJobSpec struct { ID string `json:"id"` @@ -53,6 +56,7 @@ type TupleGenerationJobSpec struct { Count int `json:"count"` } +// TupleGenerationJobStatus defines the observed state of a TupleGenerationJob. type TupleGenerationJobStatus struct { State TupleGenerationJobState `json:"state"` } @@ -65,6 +69,7 @@ type TupleGenerationJobStatus struct { //+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state` //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// TupleGenerationJob is the Schema for the TupleGenerationJob API. type TupleGenerationJob struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -75,6 +80,7 @@ type TupleGenerationJob struct { //+kubebuilder:object:root=true +// TupleGenerationJobList contains a list of TupleGenerationJobs. type TupleGenerationJobList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go index a2e2e8f..81b0683 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// TupleGenerationSchedulerSpec defines the desired state of a TupleGenerationScheduler. type TupleGenerationSchedulerSpec struct { //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:ExclusiveMinimum=false @@ -21,12 +22,14 @@ type TupleGenerationSchedulerSpec struct { Threshold int `json:"threshold"` } +// TupleGenerationSchedulerStatus defines the observed state of a TupleGenerationScheduler. type TupleGenerationSchedulerStatus struct { } //+kubebuilder:object:root=true //+kubebuilder:subresource:status +// TupleGenerationScheduler is the Schema for the TupleGenerationScheduler API. type TupleGenerationScheduler struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -37,6 +40,7 @@ type TupleGenerationScheduler struct { //+kubebuilder:object:root=true +// TupleGenerationSchedulerList contains a list of TupleGenerationSchedulers. type TupleGenerationSchedulerList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go index 970fa56..bac0599 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationtask_types.go @@ -12,16 +12,28 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// TupleGenerationTaskState encodes the state of a TupleGenerationTask. type TupleGenerationTaskState string const ( - TaskLaunching TupleGenerationTaskState = "Launching" - TaskGenerating TupleGenerationTaskState = "Generating" + + // TaskLaunching means that the tuple generation process is being initiated. + TaskLaunching TupleGenerationTaskState = "Launching" + + // TaskGenerating means that tuples are being generated. + TaskGenerating TupleGenerationTaskState = "Generating" + + // TaskProvisioning means that tuples are being uploaded to Castor. TaskProvisioning TupleGenerationTaskState = "Provisioning" - TaskCompleted TupleGenerationTaskState = "Completed" - TaskFailed TupleGenerationTaskState = "Failed" + + // TaskCompleted means that the task has been finished successfully. + TaskCompleted TupleGenerationTaskState = "Completed" + + // TaskFailed means that an error occurred while performing the task. + TaskFailed TupleGenerationTaskState = "Failed" ) +// IsValid returns true if state s is among the defined ones and false otherwise. func (s TupleGenerationTaskState) IsValid() bool { switch s { case TaskLaunching, TaskGenerating, TaskProvisioning, TaskCompleted, TaskFailed: @@ -31,14 +43,17 @@ func (s TupleGenerationTaskState) IsValid() bool { } } +// TupleGenerationTaskSpec defines the desired state of a TupleGenerationTask. type TupleGenerationTaskSpec struct { } +// TupleGenerationTaskStatus defines the observed state of a TupleGenerationTask. type TupleGenerationTaskStatus struct { State TupleGenerationTaskState `json:"state"` } -func ParseFromJSON(data []byte) (*TupleGenerationTaskStatus, error) { +// Unmarshal parses a JSON serialized TupleGenerationTask. +func Unmarshal(data []byte) (*TupleGenerationTaskStatus, error) { status := &TupleGenerationTaskStatus{} if err := json.Unmarshal(data, status); err != nil { return status, err @@ -52,6 +67,7 @@ func ParseFromJSON(data []byte) (*TupleGenerationTaskStatus, error) { //+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state` //+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// TupleGenerationTask is the Schema for the TupleGenerationTask API. type TupleGenerationTask struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -62,6 +78,7 @@ type TupleGenerationTask struct { //+kubebuilder:object:root=true +// TupleGenerationTaskList contains a list of TupleGenerationTasks. type TupleGenerationTaskList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/klyshko-operator/controllers/castor_client.go b/klyshko-operator/controllers/castor_client.go index 215f32e..03bc230 100644 --- a/klyshko-operator/controllers/castor_client.go +++ b/klyshko-operator/controllers/castor_client.go @@ -10,7 +10,6 @@ package controllers import ( "context" "encoding/json" - "errors" "fmt" "github.com/google/uuid" "io" @@ -44,7 +43,7 @@ func (castorClient CastorClient) activateTupleChunk(ctx context.Context, chunkID return err } if resp.StatusCode != http.StatusOK { - return errors.New(fmt.Sprintf("received response with status code %d", resp.StatusCode)) + return fmt.Errorf("received response with status code %d", resp.StatusCode) } defer func() { _, err := io.Copy(ioutil.Discard, resp.Body) @@ -94,7 +93,7 @@ func (castorClient CastorClient) getTelemetry(ctx context.Context) (Telemetry, e return Telemetry{}, err } if resp.StatusCode != http.StatusOK { - err := errors.New(fmt.Sprintf("received response with status code %d", resp.StatusCode)) + err := fmt.Errorf("received response with status code %d", resp.StatusCode) logger.Error(err, "failed to fetch castor telemetry data") return Telemetry{}, err } diff --git a/klyshko-operator/controllers/castor_client_test.go b/klyshko-operator/controllers/castor_client_test.go index fd4e69c..e4ed0c4 100644 --- a/klyshko-operator/controllers/castor_client_test.go +++ b/klyshko-operator/controllers/castor_client_test.go @@ -99,9 +99,9 @@ var _ = Describe("Activating a tuple chunk", func() { ) }) It("succeeds", func() { - chunkId := uuid.New() + chunkID := uuid.New() castorClient := NewCastorClient(validCastorURL) - err := castorClient.activateTupleChunk(ctx, chunkId) + err := castorClient.activateTupleChunk(ctx, chunkID) Expect(err).NotTo(HaveOccurred()) }) }) @@ -116,18 +116,18 @@ var _ = Describe("Activating a tuple chunk", func() { ) }) It("fails", func() { - chunkId := uuid.New() + chunkID := uuid.New() castorClient := NewCastorClient(validCastorURL) - err := castorClient.activateTupleChunk(ctx, chunkId) + err := castorClient.activateTupleChunk(ctx, chunkID) Expect(err).To(HaveOccurred()) }) }) When("when Castor service is not available", func() { It("fails", func() { - chunkId := uuid.New() + chunkID := uuid.New() castorClient := NewCastorClient(invalidCastorURL) - err := castorClient.activateTupleChunk(ctx, chunkId) + err := castorClient.activateTupleChunk(ctx, chunkID) Expect(err).To(HaveOccurred()) }) }) diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go index b954ae1..42fbcbb 100644 --- a/klyshko-operator/controllers/controller_test.go +++ b/klyshko-operator/controllers/controller_test.go @@ -261,10 +261,10 @@ var _ = Describe("Generating tuples", func() { } // Ensure that castor activate tuple chunk endpoint has been called on each VCP - activationUrl := fmt.Sprintf("PUT http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", jobs[0].Spec.ID) + activationURL := fmt.Sprintf("PUT http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", jobs[0].Spec.ID) Eventually(func() bool { info := httpmock.GetCallCountInfo() - return info[activationUrl] == NumberOfVCPs + return info[activationURL] == NumberOfVCPs }, Timeout, PollingInterval).Should(BeTrue()) // Ensure that resources get deleted. @@ -304,11 +304,11 @@ var _ = Describe("Generating tuples", func() { }) // Ensures that the job with the given identifier eventually assumes the given state. -func ensureJobState(ctx context.Context, vc *vc, owner *klyshkov1alpha1.TupleGenerationScheduler, jobId uuid.UUID, state klyshkov1alpha1.TupleGenerationJobState) { +func ensureJobState(ctx context.Context, vc *vc, owner *klyshkov1alpha1.TupleGenerationScheduler, jobID uuid.UUID, state klyshkov1alpha1.TupleGenerationJobState) { for i := 0; i < NumberOfVCPs; i++ { name := client.ObjectKey{ Namespace: owner.Namespace, - Name: fmt.Sprintf("%s-%s", owner.Name, jobId), + Name: fmt.Sprintf("%s-%s", owner.Name, jobID), } Eventually(func() bool { job := &klyshkov1alpha1.TupleGenerationJob{} diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 496bff4..590737d 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -93,8 +93,8 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, fmt.Errorf("failed to read resource for roster with key %v for task %v: %w", jobKey, req.Name, err) } if resp.Count == 0 { - playerId, err := localPlayerID(ctx, &r.Client, req.Namespace) - if playerId != 0 { + playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) + if playerID != 0 { logger.Info("Roster not available, retrying later") return ctrl.Result{}, nil } @@ -137,9 +137,8 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R } // Error reading resource, requeue return ctrl.Result{}, fmt.Errorf("failed to read task resource for job %v: %w", req.Name, err) - } else { - logger.Info("Local task exists already", "Task.Name", task.Name) } + logger.Info("Local task exists already", "Task.Name", task.Name) // Update job status based on owned task statuses; TODO That might not scale well in case we have many jobs tasks := &klyshkov1alpha1.TupleGenerationTaskList{} @@ -345,7 +344,7 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex found := &klyshkov1alpha1.TupleGenerationTask{} if err := r.Client.Get(ctx, taskName, found); err == nil { // Update local proxy task status - status, err := klyshkov1alpha1.ParseFromJSON(ev.Kv.Value) + status, err := klyshkov1alpha1.Unmarshal(ev.Kv.Value) if err != nil { logger.Error(err, "Extracting state from roster entry failed", "Value", string(ev.Kv.Value)) return diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index ed6ec0b..6ea5450 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -186,9 +186,9 @@ func (r *TupleGenerationTaskReconciler) SetupWithManager(mgr ctrl.Manager) error // identifier is appended with a hyphen to the name. func (r *TupleGenerationTaskReconciler) taskKeyFromName(namespace string, name string) (*RosterEntryKey, error) { parts := strings.Split(name, "-") - vcpId := parts[len(parts)-1] + vcpID := parts[len(parts)-1] jobName := strings.Join(parts[:len(parts)-1], "-") - keyStr := rosterKey + "/" + namespace + "/" + jobName + "/" + vcpId + keyStr := rosterKey + "/" + namespace + "/" + jobName + "/" + vcpID key, err := ParseKey(keyStr) if err != nil { return nil, fmt.Errorf("can't parse task key from '%s': %w", keyStr, err) @@ -218,7 +218,7 @@ func (r *TupleGenerationTaskReconciler) getStatus(ctx context.Context, taskKey R if len(resp.Kvs) != 1 { return nil, fmt.Errorf("no status available for roster entry: %v", taskKey) } - status, err := klyshkov1alpha1.ParseFromJSON(resp.Kvs[0].Value) + status, err := klyshkov1alpha1.Unmarshal(resp.Kvs[0].Value) if err != nil { return nil, fmt.Errorf("parsing status from '%s' failed: %w", string(resp.Kvs[0].Value), err) } diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index 1c90818..232a815 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -45,13 +45,13 @@ func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string - var castorUrl string + var castorURL string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&castorUrl, "castor-url", "http://cs-castor.default.svc.cluster.local:10100", "The base url of the castor service.") + flag.StringVar(&castorURL, "castor-url", "http://cs-castor.default.svc.cluster.local:10100", "The base url of the castor service.") opts := zap.Options{ Development: true, } @@ -86,7 +86,7 @@ func main() { setupLog.Error(err, "closing etcd client failed") }() - castorClient := controllers.NewCastorClient(castorUrl) + castorClient := controllers.NewCastorClient(castorURL) if err = controllers.NewTupleGenerationJobReconciler(mgr.GetClient(), mgr.GetScheme(), etcdClient, castorClient).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationJob") From 43e364b9e5c0b451718112271ab067a55c83ec78 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 2 Aug 2022 14:59:46 +0200 Subject: [PATCH 34/69] Move castor client to separate package Signed-off-by: Sven Trieflinger --- .../{controllers => castor}/castor_client.go | 20 +++++++------- .../castor_client_test.go | 26 +++++++++---------- .../controllers/controller_test.go | 5 ++-- .../tuplegenerationjob_controller.go | 7 ++--- .../tuplegenerationscheduler_controller.go | 7 ++--- klyshko-operator/main.go | 3 ++- 6 files changed, 36 insertions(+), 32 deletions(-) rename klyshko-operator/{controllers => castor}/castor_client.go (82%) rename klyshko-operator/{controllers => castor}/castor_client_test.go (81%) diff --git a/klyshko-operator/controllers/castor_client.go b/klyshko-operator/castor/castor_client.go similarity index 82% rename from klyshko-operator/controllers/castor_client.go rename to klyshko-operator/castor/castor_client.go index 03bc230..9b80a7c 100644 --- a/klyshko-operator/controllers/castor_client.go +++ b/klyshko-operator/castor/castor_client.go @@ -5,7 +5,7 @@ see the NOTICE file and/or the repository https://github.com/carbynestack/klyshk SPDX-License-Identifier: Apache-2.0 */ -package controllers +package castor import ( "context" @@ -18,27 +18,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -type CastorClient struct { +type Client struct { URL string client *http.Client } -func NewCastorClient(url string) *CastorClient { - return &CastorClient{ +func NewClient(url string) *Client { + return &Client{ URL: url, client: &http.Client{}, } } -func (castorClient CastorClient) activateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { +func (c Client) ActivateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkID) - url := fmt.Sprintf("%s/intra-vcp/tuple-chunks/activate/%s", castorClient.URL, chunkID) + url := fmt.Sprintf("%s/intra-vcp/tuple-chunks/activate/%s", c.URL, chunkID) logger.Info("activating tuple chunk with castor URL", "URL", url) req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, nil) if err != nil { return err } - resp, err := castorClient.client.Do(req) + resp, err := c.client.Do(req) if err != nil { return err } @@ -69,14 +69,14 @@ type Telemetry struct { TupleMetrics []TupleMetrics `json:"metrics"` } -func (castorClient CastorClient) getTelemetry(ctx context.Context) (Telemetry, error) { +func (c Client) GetTelemetry(ctx context.Context) (Telemetry, error) { logger := log.FromContext(ctx) // Building the request req, err := http.NewRequestWithContext( ctx, "GET", - fmt.Sprintf("%s/intra-vcp/telemetry", castorClient.URL), + fmt.Sprintf("%s/intra-vcp/telemetry", c.URL), nil, ) if err != nil { @@ -87,7 +87,7 @@ func (castorClient CastorClient) getTelemetry(ctx context.Context) (Telemetry, e req.Header.Add("Content-Type", "application/json") // Doing the request - resp, err := castorClient.client.Do(req) + resp, err := c.client.Do(req) if err != nil { logger.Error(err, "failed to fetch castor telemetry data") return Telemetry{}, err diff --git a/klyshko-operator/controllers/castor_client_test.go b/klyshko-operator/castor/castor_client_test.go similarity index 81% rename from klyshko-operator/controllers/castor_client_test.go rename to klyshko-operator/castor/castor_client_test.go index e4ed0c4..9f17a2d 100644 --- a/klyshko-operator/controllers/castor_client_test.go +++ b/klyshko-operator/castor/castor_client_test.go @@ -5,7 +5,7 @@ see the NOTICE file and/or the repository https://github.com/carbynestack/klyshk SPDX-License-Identifier: Apache-2.0 */ -package controllers +package castor import ( "context" @@ -48,8 +48,8 @@ var _ = Describe("Fetching telemetry", func() { ) }) It("succeeds", func() { - castorClient := NewCastorClient(validCastorURL) - telemetry, err := castorClient.getTelemetry(ctx) + castorClient := NewClient(validCastorURL) + telemetry, err := castorClient.GetTelemetry(ctx) Expect(err).NotTo(HaveOccurred()) Expect(telemetry).To(Equal(expectedTelemetry)) }) @@ -65,16 +65,16 @@ var _ = Describe("Fetching telemetry", func() { ) }) It("fails", func() { - castorClient := NewCastorClient(validCastorURL) - _, err := castorClient.getTelemetry(ctx) + castorClient := NewClient(validCastorURL) + _, err := castorClient.GetTelemetry(ctx) Expect(err).To(HaveOccurred()) }) }) When("when Castor service is not available", func() { It("fails", func() { - castorClient := NewCastorClient(invalidCastorURL) - _, err := castorClient.getTelemetry(ctx) + castorClient := NewClient(invalidCastorURL) + _, err := castorClient.GetTelemetry(ctx) Expect(err).To(HaveOccurred()) }) }) @@ -100,8 +100,8 @@ var _ = Describe("Activating a tuple chunk", func() { }) It("succeeds", func() { chunkID := uuid.New() - castorClient := NewCastorClient(validCastorURL) - err := castorClient.activateTupleChunk(ctx, chunkID) + castorClient := NewClient(validCastorURL) + err := castorClient.ActivateTupleChunk(ctx, chunkID) Expect(err).NotTo(HaveOccurred()) }) }) @@ -117,8 +117,8 @@ var _ = Describe("Activating a tuple chunk", func() { }) It("fails", func() { chunkID := uuid.New() - castorClient := NewCastorClient(validCastorURL) - err := castorClient.activateTupleChunk(ctx, chunkID) + castorClient := NewClient(validCastorURL) + err := castorClient.ActivateTupleChunk(ctx, chunkID) Expect(err).To(HaveOccurred()) }) }) @@ -126,8 +126,8 @@ var _ = Describe("Activating a tuple chunk", func() { When("when Castor service is not available", func() { It("fails", func() { chunkID := uuid.New() - castorClient := NewCastorClient(invalidCastorURL) - err := castorClient.activateTupleChunk(ctx, chunkID) + castorClient := NewClient(invalidCastorURL) + err := castorClient.ActivateTupleChunk(ctx, chunkID) Expect(err).To(HaveOccurred()) }) }) diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go index 42fbcbb..89d9315 100644 --- a/klyshko-operator/controllers/controller_test.go +++ b/klyshko-operator/controllers/controller_test.go @@ -11,6 +11,7 @@ import ( "context" "fmt" klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/carbynestack/klyshko/castor" "github.com/google/uuid" "github.com/jarcoal/httpmock" . "github.com/onsi/ginkgo/v2" @@ -115,7 +116,7 @@ func (vcp vcp) setupControllers(ctx context.Context, vcpID int, etcdClient *clie if err != nil { return err } - castorClient := NewCastorClient(castorURL) + castorClient := castor.NewClient(castorURL) controllers := []Controller{ NewTupleGenerationJobReconciler( k8sManager.GetClient(), k8sManager.GetScheme(), etcdClient, castorClient), @@ -198,7 +199,7 @@ func setupCastorServiceResponders(numberOfAvailableTuples int) { "=~^http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/.*", httpmock.NewStringResponder(200, ""), ) - telemetry := Telemetry{TupleMetrics: []TupleMetrics{ + telemetry := castor.Telemetry{TupleMetrics: []castor.TupleMetrics{ { Available: numberOfAvailableTuples, ConsumptionRate: 0, diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 590737d..4b6b007 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/carbynestack/klyshko/castor" "github.com/go-logr/logr" "github.com/google/uuid" "go.etcd.io/etcd/api/v3/mvccpb" @@ -33,12 +34,12 @@ type TupleGenerationJobReconciler struct { Scheme *runtime.Scheme EtcdClient *clientv3.Client rosterWatcherCh clientv3.WatchChan - CastorClient *CastorClient + CastorClient *castor.Client Logger logr.Logger } // NewTupleGenerationJobReconciler creates a TupleGenerationJobReconciler. -func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Scheme, etcdClient *clientv3.Client, castorClient *CastorClient) *TupleGenerationJobReconciler { +func NewTupleGenerationJobReconciler(client client.Client, scheme *runtime.Scheme, etcdClient *clientv3.Client, castorClient *castor.Client) *TupleGenerationJobReconciler { r := &TupleGenerationJobReconciler{ Client: client, Scheme: scheme, @@ -200,7 +201,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R if err != nil { return ctrl.Result{}, fmt.Errorf("invalid job id encountered '%v': %w", job.Spec.ID, err) } - err = r.CastorClient.activateTupleChunk(ctx, tupleChunkID) + err = r.CastorClient.ActivateTupleChunk(ctx, tupleChunkID) if err != nil { return ctrl.Result{}, fmt.Errorf("tuple chunk activation failed for job %v: %w", job.Name, err) } diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 30c7df5..57a5739 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -10,6 +10,7 @@ package controllers import ( "context" "fmt" + "github.com/carbynestack/klyshko/castor" "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,7 +32,7 @@ const MinimumTuplesPerJob = 10000 type TupleGenerationSchedulerReconciler struct { client.Client Scheme *runtime.Scheme - CastorClient *CastorClient + CastorClient *castor.Client } //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationschedulers,verbs=get;list;watch;create;update;patch;delete @@ -80,7 +81,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req // 1. Compute available and in generation number of tuples per type // 2. Filter out all above threshold // 3. Sort ascending wrt to sum from step 1 - telemetry, err := r.CastorClient.getTelemetry(ctx) + telemetry, err := r.CastorClient.GetTelemetry(ctx) if err != nil { logger.Error(err, "Fetching telemetry data from Castor failed", "Castor.URL", r.CastorClient.URL) return ctrl.Result{RequeueAfter: 60 * time.Second}, err @@ -95,7 +96,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req } } logger.Info("With in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) - var belowThreshold []TupleMetrics + var belowThreshold []castor.TupleMetrics for _, m := range telemetry.TupleMetrics { if m.Available < scheduler.Spec.Threshold { belowThreshold = append(belowThreshold, m) diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index 232a815..dfe5fe3 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -9,6 +9,7 @@ package main import ( "flag" + "github.com/carbynestack/klyshko/castor" "os" "time" @@ -86,7 +87,7 @@ func main() { setupLog.Error(err, "closing etcd client failed") }() - castorClient := controllers.NewCastorClient(castorURL) + castorClient := castor.NewClient(castorURL) if err = controllers.NewTupleGenerationJobReconciler(mgr.GetClient(), mgr.GetScheme(), etcdClient, castorClient).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationJob") From a8a0f6249f097381f373716c8620fdecd967d3c4 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 2 Aug 2022 15:10:46 +0200 Subject: [PATCH 35/69] Add documentation to Castor client and improve logging / error handling Signed-off-by: Sven Trieflinger --- klyshko-operator/castor/castor_client.go | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/klyshko-operator/castor/castor_client.go b/klyshko-operator/castor/castor_client.go index 9b80a7c..27dc8c0 100644 --- a/klyshko-operator/castor/castor_client.go +++ b/klyshko-operator/castor/castor_client.go @@ -18,11 +18,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) +// Client is a client for the Castor tuple store. type Client struct { URL string client *http.Client } +// NewClient creates a Castor client talking to the given URL. func NewClient(url string) *Client { return &Client{ URL: url, @@ -30,10 +32,11 @@ func NewClient(url string) *Client { } } +// ActivateTupleChunk activates the tuple chunk with the given chunk identifier stored by the Castor service. func (c Client) ActivateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkID) url := fmt.Sprintf("%s/intra-vcp/tuple-chunks/activate/%s", c.URL, chunkID) - logger.Info("activating tuple chunk with castor URL", "URL", url) + logger.Info("Activating tuple chunk with castor URL", "URL", url) req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, nil) if err != nil { return err @@ -48,27 +51,30 @@ func (c Client) ActivateTupleChunk(ctx context.Context, chunkID uuid.UUID) error defer func() { _, err := io.Copy(ioutil.Discard, resp.Body) if err != nil { - logger.Error(err, "failed to discard response from castor") + logger.Error(err, "Failed to discard response from castor") } err = resp.Body.Close() if err != nil { - logger.Error(err, "failed to close response from castor") + logger.Error(err, "Failed to close response from castor") } }() - logger.Info("response from castor", "Status", resp.Status) + logger.Info("Response from castor", "Status", resp.Status) return nil } +// TupleMetrics stores how many tuples are available for a given tuple type and how fast they are consumed. type TupleMetrics struct { Available int `json:"available"` ConsumptionRate int `json:"consumptionRate"` TupleType string `json:"type"` } +// Telemetry stores a TupleMetrics object per tuple type. type Telemetry struct { TupleMetrics []TupleMetrics `json:"metrics"` } +// GetTelemetry fetches telemetry data from the Castor service. func (c Client) GetTelemetry(ctx context.Context) (Telemetry, error) { logger := log.FromContext(ctx) @@ -80,8 +86,7 @@ func (c Client) GetTelemetry(ctx context.Context) (Telemetry, error) { nil, ) if err != nil { - logger.Error(err, "failed to build request for castor telemetry data") - return Telemetry{}, err + return Telemetry{}, fmt.Errorf("failed to build request for castor telemetry data") } req.Header.Add("Accept", "application/json") req.Header.Add("Content-Type", "application/json") @@ -89,26 +94,23 @@ func (c Client) GetTelemetry(ctx context.Context) (Telemetry, error) { // Doing the request resp, err := c.client.Do(req) if err != nil { - logger.Error(err, "failed to fetch castor telemetry data") + logger.Error(err, "Failed to fetch castor telemetry data") return Telemetry{}, err } if resp.StatusCode != http.StatusOK { - err := fmt.Errorf("received response with status code %d", resp.StatusCode) - logger.Error(err, "failed to fetch castor telemetry data") - return Telemetry{}, err + return Telemetry{}, fmt.Errorf("received response with status code %d", resp.StatusCode) } // Read, parse, and return telemetry response defer func() { err := resp.Body.Close() if err != nil { - logger.Error(err, "failed to close response from castor") + logger.Error(err, "Failed to close response from castor") } }() bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - logger.Error(err, "failed to read response body") - return Telemetry{}, err + return Telemetry{}, fmt.Errorf("failed to read response body") } var response Telemetry err = json.Unmarshal(bodyBytes, &response) From 5129e5914c3101989142fea85d50ab1cab8ddb9b Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 2 Aug 2022 15:38:46 +0200 Subject: [PATCH 36/69] Add documentation to etcd key implementation Signed-off-by: Sven Trieflinger --- klyshko-operator/controllers/config_utils.go | 4 ++-- klyshko-operator/controllers/etcd_keys.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/klyshko-operator/controllers/config_utils.go b/klyshko-operator/controllers/config_utils.go index e8d7d88..ad28dad 100644 --- a/klyshko-operator/controllers/config_utils.go +++ b/klyshko-operator/controllers/config_utils.go @@ -18,12 +18,12 @@ import ( "strconv" ) -const VCPConfigMapName = "cs-vcp-config" +const vcpConfigMapName = "cs-vcp-config" func getVCPConfig(ctx context.Context, client *client.Client, namespace string) (v1.ConfigMap, error) { name := types.NamespacedName{ Namespace: namespace, - Name: VCPConfigMapName, + Name: vcpConfigMapName, } cfm := v1.ConfigMap{} err := (*client).Get(ctx, name, &cfm) diff --git a/klyshko-operator/controllers/etcd_keys.go b/klyshko-operator/controllers/etcd_keys.go index 380095a..8e191fb 100644 --- a/klyshko-operator/controllers/etcd_keys.go +++ b/klyshko-operator/controllers/etcd_keys.go @@ -17,31 +17,40 @@ import ( const rosterKey = "/klyshko/roster" +// Key is a key for data stored in an etcd cluster. type Key interface { ToEtcdKey() string } +// RosterKey is a Key referencing a set of RosterEntryKey instances. The data referenced by a RosterKey consists of +// information related to a tuple generation job managed mainly by the TupleGenerationJobReconciler. type RosterKey struct { types.NamespacedName } +// ToEtcdKey converts RosterKey k to an etcd key. func (k RosterKey) ToEtcdKey() string { return fmt.Sprintf("%s/%s/%s", rosterKey, k.Namespace, k.Name) } +// String returns a string representation of RosterKey k. func (k RosterKey) String() string { return k.ToEtcdKey() } +// RosterEntryKey is a Key referencing data that is related to a tuple generation task managed mainly by the +// TupleGenerationTaskReconciler. type RosterEntryKey struct { RosterKey PlayerID uint } +// ToEtcdKey converts RosterEntryKey k to an etcd key. func (k RosterEntryKey) ToEtcdKey() string { return fmt.Sprintf("%s/%d", k.RosterKey.ToEtcdKey(), k.PlayerID) } +// String returns a string representation of RosterEntryKey k. func (k RosterEntryKey) String() string { return k.ToEtcdKey() } @@ -62,6 +71,8 @@ func etcdKeyParts(s string) map[string]string { return result } +// ParseKey parses string s into a RosterKey or RosterEntryKey. In case s cannot be parsed in any of the those, +// an error is returned. func ParseKey(s string) (Key, error) { parts := etcdKeyParts(s) if parts == nil { From 967a1c396b491e5c0c8e55d33e970f656bd6076d Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 2 Aug 2022 16:12:33 +0200 Subject: [PATCH 37/69] Reenable castor client tests after moving to dedicated package Signed-off-by: Sven Trieflinger --- klyshko-operator/castor/suite_test.go | 27 +++++++++++++++++++ ...o.carbnyestack.io_tuplegenerationjobs.yaml | 5 ++++ ...nyestack.io_tuplegenerationschedulers.yaml | 6 +++++ ....carbnyestack.io_tuplegenerationtasks.yaml | 6 +++++ 4 files changed, 44 insertions(+) create mode 100644 klyshko-operator/castor/suite_test.go diff --git a/klyshko-operator/castor/suite_test.go b/klyshko-operator/castor/suite_test.go new file mode 100644 index 0000000..088e853 --- /dev/null +++ b/klyshko-operator/castor/suite_test.go @@ -0,0 +1,27 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package castor + +import ( + . "github.com/onsi/ginkgo/v2" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "testing" + + . "github.com/onsi/gomega" + //+kubebuilder:scaffold:imports +) + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) +}) + +func TestCastorClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Klyshko Castor Client Suite") +} diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml index 936495f..4956528 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -35,6 +35,7 @@ spec: name: v1alpha1 schema: openAPIV3Schema: + description: TupleGenerationJob is the Schema for the TupleGenerationJob API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -49,6 +50,7 @@ spec: metadata: type: object spec: + description: TupleGenerationJobSpec defines the desired state of a TupleGenerationJob. properties: count: exclusiveMinimum: true @@ -75,8 +77,11 @@ spec: - type type: object status: + description: TupleGenerationJobStatus defines the observed state of a + TupleGenerationJob. properties: state: + description: TupleGenerationJobState encodes the state of a TupleGenerationJob. type: string required: - state diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml index 44ce6f6..cab5695 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml @@ -19,6 +19,8 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: + description: TupleGenerationScheduler is the Schema for the TupleGenerationScheduler + API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -33,6 +35,8 @@ spec: metadata: type: object spec: + description: TupleGenerationSchedulerSpec defines the desired state of + a TupleGenerationScheduler. properties: concurrency: minimum: 0 @@ -46,6 +50,8 @@ spec: - threshold type: object status: + description: TupleGenerationSchedulerStatus defines the observed state + of a TupleGenerationScheduler. type: object type: object served: true diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml index 5ffa1e2..36a61d7 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationtasks.yaml @@ -29,6 +29,8 @@ spec: name: v1alpha1 schema: openAPIV3Schema: + description: TupleGenerationTask is the Schema for the TupleGenerationTask + API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -43,10 +45,14 @@ spec: metadata: type: object spec: + description: TupleGenerationTaskSpec defines the desired state of a TupleGenerationTask. type: object status: + description: TupleGenerationTaskStatus defines the observed state of a + TupleGenerationTask. properties: state: + description: TupleGenerationTaskState encodes the state of a TupleGenerationTask. type: string required: - state From 1284066bed436a407473b223403a481fa035a696 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 9 Sep 2022 15:05:46 +0200 Subject: [PATCH 38/69] Improve controller test and TTL for jobs Signed-off-by: Sven Trieflinger --- klyshko-operator/Makefile | 2 +- .../api/v1alpha1/tuplegenerationjob_types.go | 3 +- .../tuplegenerationscheduler_types.go | 5 + .../api/v1alpha1/zz_generated.deepcopy.go | 3 +- ...o.carbnyestack.io_tuplegenerationjobs.yaml | 4 + ...nyestack.io_tuplegenerationschedulers.yaml | 6 + .../controllers/controller_test.go | 192 +++++++++++------- .../tuplegenerationjob_controller.go | 1 + .../tuplegenerationscheduler_controller.go | 18 +- 9 files changed, 152 insertions(+), 82 deletions(-) diff --git a/klyshko-operator/Makefile b/klyshko-operator/Makefile index 7e1e9ad..5e44643 100644 --- a/klyshko-operator/Makefile +++ b/klyshko-operator/Makefile @@ -90,7 +90,7 @@ vet: ## Run go vet against code. go vet ./... test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" GINKGO_EDITOR_INTEGRATION=true go test ./... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" GINKGO_EDITOR_INTEGRATION=true go test -v ./... -coverprofile cover.out ##@ Build diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go index e1f632f..005f362 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -58,7 +58,8 @@ type TupleGenerationJobSpec struct { // TupleGenerationJobStatus defines the observed state of a TupleGenerationJob. type TupleGenerationJobStatus struct { - State TupleGenerationJobState `json:"state"` + State TupleGenerationJobState `json:"state"` + LastStateTransitionTime metav1.Time `json:"lastStateTransitionTime"` } //+kubebuilder:object:root=true diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go index 81b0683..812204b 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go @@ -20,6 +20,11 @@ type TupleGenerationSchedulerSpec struct { //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:ExclusiveMinimum=true Threshold int `json:"threshold"` + + //+kubebuilder:default=600 + //+kubebuilder:validation:Minimum=0 + //+kubebuilder:validation:ExclusiveMinimum=true + TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` } // TupleGenerationSchedulerStatus defines the observed state of a TupleGenerationScheduler. diff --git a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go index 71149bf..4a96bdc 100644 --- a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -30,7 +30,7 @@ func (in *TupleGenerationJob) DeepCopyInto(out *TupleGenerationJob) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationJob. @@ -101,6 +101,7 @@ func (in *TupleGenerationJobSpec) DeepCopy() *TupleGenerationJobSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TupleGenerationJobStatus) DeepCopyInto(out *TupleGenerationJobStatus) { *out = *in + in.LastStateTransitionTime.DeepCopyInto(&out.LastStateTransitionTime) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationJobStatus. diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml index 4956528..bbdef54 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -80,10 +80,14 @@ spec: description: TupleGenerationJobStatus defines the observed state of a TupleGenerationJob. properties: + lastStateTransitionTime: + format: date-time + type: string state: description: TupleGenerationJobState encodes the state of a TupleGenerationJob. type: string required: + - lastStateTransitionTime - state type: object type: object diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml index cab5695..4d21510 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml @@ -45,9 +45,15 @@ spec: exclusiveMinimum: true minimum: 0 type: integer + ttlSecondsAfterFinished: + default: 600 + exclusiveMinimum: true + minimum: 0 + type: integer required: - concurrency - threshold + - ttlSecondsAfterFinished type: object status: description: TupleGenerationSchedulerStatus defines the observed state diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go index 89d9315..f1e9ae7 100644 --- a/klyshko-operator/controllers/controller_test.go +++ b/klyshko-operator/controllers/controller_test.go @@ -37,7 +37,7 @@ import ( ) const NumberOfVCPs = 2 -const Timeout = 10 * time.Second +const Timeout = 30 * time.Second const PollingInterval = 1 * time.Second const SchedulerNamespace = "default" const SchedulerName = "test-scheduler" @@ -55,7 +55,7 @@ func setupVCP() (*vcp, error) { env.testEnv = &envtest.Environment{ ErrorIfCRDPathMissing: true, CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, - AttachControlPlaneOutput: true, + AttachControlPlaneOutput: false, } var err error env.cfg, err = env.testEnv.Start() @@ -70,11 +70,11 @@ func setupVCP() (*vcp, error) { return &env, err } -func (vcp vcp) tearDownVCP() error { +func (vcp *vcp) tearDownVCP() error { return vcp.testEnv.Stop() } -func (vcp vcp) createVCPConfig(ctx context.Context, name string, namespace string, data map[string]string) { +func (vcp *vcp) createVCPConfig(ctx context.Context, name string, namespace string, data map[string]string) { vcpConfig := v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -88,7 +88,7 @@ func (vcp vcp) createVCPConfig(ctx context.Context, name string, namespace strin } } -func (vcp vcp) deleteVCPConfig(ctx context.Context, name string, namespace string) { +func (vcp *vcp) deleteVCPConfig(ctx context.Context, name string, namespace string) { vcpConfig := &v1.ConfigMap{} err := vcp.k8sClient.Get(ctx, client.ObjectKey{ Name: name, @@ -107,7 +107,7 @@ type Controller interface { SetupWithManager(manager.Manager) error } -func (vcp vcp) setupControllers(ctx context.Context, vcpID int, etcdClient *clientv3.Client, castorURL string) error { +func (vcp *vcp) setupControllers(ctx context.Context, vcpID int, etcdClient *clientv3.Client, castorURL string) error { k8sManager, err := ctrl.NewManager(vcp.cfg, ctrl.Options{ Scheme: scheme.Scheme, MetricsBindAddress: "0", // Avoid colliding metrics servers by disabling @@ -125,11 +125,13 @@ func (vcp vcp) setupControllers(ctx context.Context, vcpID int, etcdClient *clie Scheme: k8sManager.GetScheme(), EtcdClient: etcdClient, }, - &TupleGenerationSchedulerReconciler{ + } + if vcpID == 0 { + controllers = append(controllers, &TupleGenerationSchedulerReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), CastorClient: castorClient, - }, + }) } for _, controller := range controllers { err := controller.SetupWithManager(k8sManager) @@ -160,7 +162,7 @@ func setupVC(ctx context.Context, numberOfVCPs int) (*vc, error) { // Create the VCP configuration vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": strconv.Itoa(1), + "playerCount": strconv.Itoa(NumberOfVCPs), "playerId": strconv.Itoa(i), }) @@ -184,7 +186,7 @@ func setupVC(ctx context.Context, numberOfVCPs int) (*vc, error) { func (vc *vc) teardown() error { for _, vcp := range vc.vcps { - err := vcp.testEnv.Stop() + err := vcp.tearDownVCP() if err != nil { return err } @@ -216,90 +218,131 @@ func setupCastorServiceResponders(numberOfAvailableTuples int) { } var _ = Describe("Generating tuples", func() { - ctx, cancel := context.WithCancel(context.TODO()) - var vc *vc - - BeforeEach(func() { - httpmock.Activate() - setupCastorServiceResponders(0) - var err error - vc, err = setupVC(ctx, NumberOfVCPs) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - cancel() - err := vc.teardown() - Expect(err).NotTo(HaveOccurred()) - httpmock.DeactivateAndReset() - }) When("a scheduler is deployed", func() { - It("succeeds", func() { - scheduler := createScheduler(ctx, vc) - jobs := ensureJobCreatedOnEachVcp(ctx, vc, scheduler) + + var ( + ctx context.Context + cancel context.CancelFunc + vc *vc + scheduler *klyshkov1alpha1.TupleGenerationScheduler + jobs []klyshkov1alpha1.TupleGenerationJob + localTasksByVCP []klyshkov1alpha1.TupleGenerationTask + generatorPodsByVCP []v1.Pod + ) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.TODO()) + httpmock.Activate() + setupCastorServiceResponders(0) + var err error + vc, err = setupVC(ctx, NumberOfVCPs) + Expect(err).NotTo(HaveOccurred()) + + scheduler = createScheduler(ctx, vc) + jobs = ensureJobCreatedOnEachVcp(ctx, vc, scheduler) // Make Castor mock respond from here on with large number of available tuples, to ensure that no other // jobs are created setupCastorServiceResponders(math.MaxInt32) - localTasksByVCP := ensureTasksCreatedOnEachVcp(ctx, vc, scheduler, jobs) - generatorPodsByVCP := ensureGeneratorPodsCreatedOnEachVcp(ctx, vc, localTasksByVCP) + localTasksByVCP = ensureTasksCreatedOnEachVcp(ctx, vc, scheduler, jobs) + generatorPodsByVCP = ensureGeneratorPodsCreatedOnEachVcp(ctx, vc, localTasksByVCP) ensureJobState(ctx, vc, scheduler, uuid.MustParse(jobs[0].Spec.ID), klyshkov1alpha1.JobRunning) + }) - // Update generator pods to be in PodSucceed state - for i, pod := range generatorPodsByVCP { - pod.Status.Phase = v1.PodSucceeded - Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) - } + AfterEach(func() { + cancel() + err := vc.teardown() + Expect(err).NotTo(HaveOccurred()) + httpmock.DeactivateAndReset() + }) - provisionerPodsByVCP := ensureProvisionerPodsCreatedOnEachVcp(ctx, vc, jobs, localTasksByVCP) + Context("and the generator pod fails", func() { + It("fails", func() { + // Update generator pods to be in PodFailed state + for i, pod := range generatorPodsByVCP { + pod.Status.Phase = v1.PodFailed + Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) + } + ensureJobState(ctx, vc, scheduler, uuid.MustParse(jobs[0].Spec.ID), klyshkov1alpha1.JobFailed) + }) + }) - // Update provisioner pods to be in PodSucceed state - for i, pod := range provisionerPodsByVCP { - pod.Status.Phase = v1.PodSucceeded - Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) - } + Context("and the provisioner pod fails", func() { + It("fails", func() { + // Update generator pods to be in PodSucceed state + for i, pod := range generatorPodsByVCP { + pod.Status.Phase = v1.PodSucceeded + Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) + } - // Ensure that castor activate tuple chunk endpoint has been called on each VCP - activationURL := fmt.Sprintf("PUT http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", jobs[0].Spec.ID) - Eventually(func() bool { - info := httpmock.GetCallCountInfo() - return info[activationURL] == NumberOfVCPs - }, Timeout, PollingInterval).Should(BeTrue()) - - // Ensure that resources get deleted. - // As of https://book-v2.book.kubebuilder.io/reference/envtest.html#testing-considerations garbage - // collection does not work in envtest. Hence, we can only check that the jobs get deleted and ensure that - // owner references are set up correctly for all our resources (see respective ensure... methods below). - for i := 0; i < NumberOfVCPs; i++ { - key := client.ObjectKey{ - Namespace: jobs[i].GetNamespace(), - Name: jobs[i].GetName(), + provisionerPodsByVCP := ensureProvisionerPodsCreatedOnEachVcp(ctx, vc, jobs, localTasksByVCP) + + // Update provisioner pods to be in PodFailed state + for i, pod := range provisionerPodsByVCP { + pod.Status.Phase = v1.PodFailed + Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) + } + ensureJobState(ctx, vc, scheduler, uuid.MustParse(jobs[0].Spec.ID), klyshkov1alpha1.JobFailed) + }) + }) + + Context("the generator pod succeeds", func() { + It("succeeds", func() { + // Update generator pods to be in PodSucceed state + for i, pod := range generatorPodsByVCP { + pod.Status.Phase = v1.PodSucceeded + Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) + } + + provisionerPodsByVCP := ensureProvisionerPodsCreatedOnEachVcp(ctx, vc, jobs, localTasksByVCP) + + // Update provisioner pods to be in PodSucceed state + for i, pod := range provisionerPodsByVCP { + pod.Status.Phase = v1.PodSucceeded + Expect(vc.vcps[i].k8sClient.Status().Update(ctx, &pod)).Should(Succeed()) } + + // Ensure that castor activate tuple chunk endpoint has been called on each VCP + activationURL := fmt.Sprintf("PUT http://cs-castor.default.svc.cluster.local:10100/intra-vcp/tuple-chunks/activate/%s", jobs[0].Spec.ID) Eventually(func() bool { - return apierrors.IsNotFound(vc.vcps[i].k8sClient.Get(ctx, key, &jobs[i])) + info := httpmock.GetCallCountInfo() + return info[activationURL] == NumberOfVCPs }, Timeout, PollingInterval).Should(BeTrue()) - } - // Ensure that proxy tasks get deleted on all VCPs eventually after local tasks are deleted. - for i := 0; i < NumberOfVCPs; i++ { - Expect(vc.vcps[i].k8sClient.Delete(ctx, &localTasksByVCP[i])).Should(Succeed()) - for j := 0; j < NumberOfVCPs; j++ { - if i == j { - continue - } + // Ensure that resources get deleted. + // As of https://book-v2.book.kubebuilder.io/reference/envtest.html#testing-considerations garbage + // collection does not work in envtest. Hence, we can only check that the jobs get deleted and ensure that + // owner references are set up correctly for all our resources (see respective ensure... methods below). + for i := 0; i < NumberOfVCPs; i++ { key := client.ObjectKey{ - Namespace: jobs[j].GetNamespace(), - Name: fmt.Sprintf("%s-%d", jobs[j].GetName(), i), + Namespace: jobs[i].GetNamespace(), + Name: jobs[i].GetName(), } - proxyTask := &klyshkov1alpha1.TupleGenerationTask{} Eventually(func() bool { - return apierrors.IsNotFound(vc.vcps[j].k8sClient.Get(ctx, key, proxyTask)) + return apierrors.IsNotFound(vc.vcps[i].k8sClient.Get(ctx, key, &jobs[i])) }, Timeout, PollingInterval).Should(BeTrue()) } - } + // Ensure that proxy tasks get deleted on all VCPs eventually after local tasks are deleted. + for i := 0; i < NumberOfVCPs; i++ { + Expect(vc.vcps[i].k8sClient.Delete(ctx, &localTasksByVCP[i])).Should(Succeed()) + for j := 0; j < NumberOfVCPs; j++ { + if i == j { + continue + } + key := client.ObjectKey{ + Namespace: jobs[j].GetNamespace(), + Name: fmt.Sprintf("%s-%d", jobs[j].GetName(), i), + } + proxyTask := &klyshkov1alpha1.TupleGenerationTask{} + Eventually(func() bool { + return apierrors.IsNotFound(vc.vcps[j].k8sClient.Get(ctx, key, proxyTask)) + }, Timeout, PollingInterval).Should(BeTrue()) + } + } + }) }) }) }) @@ -451,8 +494,9 @@ func createScheduler(ctx context.Context, vc *vc) *klyshkov1alpha1.TupleGenerati Namespace: SchedulerNamespace, }, Spec: klyshkov1alpha1.TupleGenerationSchedulerSpec{ - Concurrency: SchedulerConcurrency, - Threshold: SchedulerTupleThreshold, + Concurrency: SchedulerConcurrency, + Threshold: SchedulerTupleThreshold, + TTLSecondsAfterFinished: 5, }, } Expect(vc.vcps[0].k8sClient.Create(ctx, scheduler)).Should(Succeed()) diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 4b6b007..b77bcac 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -209,6 +209,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R if state.IsValid() && state != job.Status.State { logger.Info("State update", "from", job.Status.State, "to", state) job.Status.State = state + job.Status.LastStateTransitionTime = metav1.Now() err = r.Status().Update(ctx, job) if err != nil { return ctrl.Result{}, fmt.Errorf("status update failed for job %v: %w", job.Name, err) diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 57a5739..d28461f 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -28,6 +28,9 @@ import ( // MinimumTuplesPerJob is the minimum number of tuples generated by a TupleGenerationJob. const MinimumTuplesPerJob = 10000 +// PeriodicReconciliationDuration is the maximum time between two successive reconciliations +const PeriodicReconciliationDuration = 10 * time.Second + // TupleGenerationSchedulerReconciler reconciles a TupleGenerationScheduler object. type TupleGenerationSchedulerReconciler struct { client.Client @@ -58,7 +61,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req } // Remove all finished jobs - if r.cleanupFinishedJobs(ctx) != nil { + if r.cleanupFinishedJobs(ctx, scheduler) != nil { return ctrl.Result{}, fmt.Errorf("failed to delete finished jobs: %w", err) } @@ -118,7 +121,9 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, fmt.Errorf("failed to create tuple generation job: %w", err) } - return ctrl.Result{}, nil + return ctrl.Result{ + RequeueAfter: PeriodicReconciliationDuration, + }, nil } // Creates a tuple generation job for the given tuple type in the namespace where the scheduler lives in. @@ -157,11 +162,14 @@ func (r *TupleGenerationSchedulerReconciler) createJob(ctx context.Context, sche return nil } -// Deletes all jobs that are done, i.e., either complete or failed -func (r *TupleGenerationSchedulerReconciler) cleanupFinishedJobs(ctx context.Context) error { +// Deletes all jobs that are done, i.e., either complete or failed, and beyond the TTL +func (r *TupleGenerationSchedulerReconciler) cleanupFinishedJobs(ctx context.Context, scheduler *klyshkov1alpha1.TupleGenerationScheduler) error { logger := log.FromContext(ctx) finishedJobs, err := r.getMatchingJobs(ctx, func(job klyshkov1alpha1.TupleGenerationJob) bool { - return job.Status.State.IsDone() + isBeyondTTL := func() bool { + return time.Now().After(job.Status.LastStateTransitionTime.Add(time.Duration(scheduler.Spec.TTLSecondsAfterFinished) * time.Second)) + } + return job.Status.State.IsDone() && isBeyondTTL() }) if err != nil { logger.Error(err, "failed to fetch finished jobs") From 1a32ccded997e76ec1372fc7cc42645fe04f7104 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 9 Sep 2022 15:31:03 +0200 Subject: [PATCH 39/69] Add package documentation Signed-off-by: Sven Trieflinger --- klyshko-operator/api/v1alpha1/doc.go | 10 ++++++++++ klyshko-operator/controllers/doc.go | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 klyshko-operator/api/v1alpha1/doc.go create mode 100644 klyshko-operator/controllers/doc.go diff --git a/klyshko-operator/api/v1alpha1/doc.go b/klyshko-operator/api/v1alpha1/doc.go new file mode 100644 index 0000000..14df302 --- /dev/null +++ b/klyshko-operator/api/v1alpha1/doc.go @@ -0,0 +1,10 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package v1alpha1 contains the code defining the v1alpha1 version of the +// Klyshko API. +package v1alpha1 diff --git a/klyshko-operator/controllers/doc.go b/klyshko-operator/controllers/doc.go new file mode 100644 index 0000000..14b54bc --- /dev/null +++ b/klyshko-operator/controllers/doc.go @@ -0,0 +1,10 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package controllers contains the K8s controllers and utilities that +// implement the Klyshko operator core logic. +package controllers From cb859c4d7f734fdb7d961e602eeeab71b2b732ab Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 9 Sep 2022 16:04:56 +0200 Subject: [PATCH 40/69] Address minor review comments Signed-off-by: Sven Trieflinger --- klyshko-operator/controllers/config_utils.go | 47 +++++--- .../controllers/config_utils_test.go | 108 +++++++----------- .../controllers/etcd_keys_test.go | 12 +- .../tuplegenerationjob_controller.go | 8 +- 4 files changed, 88 insertions(+), 87 deletions(-) diff --git a/klyshko-operator/controllers/config_utils.go b/klyshko-operator/controllers/config_utils.go index ad28dad..12f7039 100644 --- a/klyshko-operator/controllers/config_utils.go +++ b/klyshko-operator/controllers/config_utils.go @@ -33,44 +33,53 @@ func getVCPConfig(ctx context.Context, client *client.Client, namespace string) return cfm, nil } -func localPlayerID(ctx context.Context, client *client.Client, namespace string) (uint, error) { +func parseVCPConfig(ctx context.Context, client *client.Client, namespace string) (uint, uint, error) { cfm, err := getVCPConfig(ctx, client, namespace) if err != nil { - return 0, err + return 0, 0, err + } + + // Extract playerCount + playerCountStr, ok := cfm.Data["playerCount"] + if !ok { + return 0, 0, errors.New("invalid VCP configuration - missing playerCount") + } + playerCount, err := strconv.Atoi(playerCountStr) + if err != nil { + return 0, 0, err + } + if playerCount < 0 || playerCount > math.MaxUint32 { + return 0, 0, fmt.Errorf("invalid playerCount '%d' - must be in range [0,%d]", playerCount, math.MaxUint32) } // Extract playerId playerIDStr, ok := cfm.Data["playerId"] if !ok { - return 0, errors.New("invalid VCP configuration - missing playerId") + return 0, 0, errors.New("invalid VCP configuration - missing playerId") } playerID, err := strconv.Atoi(playerIDStr) if err != nil { - return 0, err + return 0, 0, err } - if playerID < 0 || playerID > math.MaxUint32 { - return 0, fmt.Errorf("invalid playerId '%d'- must be in range [0,%d]", playerID, math.MaxUint32) + if playerID < 0 || playerID >= playerCount { + return 0, 0, fmt.Errorf("invalid playerId '%d' - must be in range [0,%d]", playerID, playerCount) } - return uint(playerID), nil + + return uint(playerID), uint(playerCount), nil } -func numberOfVCPs(ctx context.Context, client *client.Client, namespace string) (uint, error) { - cfm, err := getVCPConfig(ctx, client, namespace) +func localPlayerID(ctx context.Context, client *client.Client, namespace string) (uint, error) { + playerID, _, err := parseVCPConfig(ctx, client, namespace) if err != nil { return 0, err } + return playerID, nil +} - // Extract playerCount - playerCountStr, ok := cfm.Data["playerCount"] - if !ok { - return 0, errors.New("invalid VCP configuration - missing playerCount") - } - playerCount, err := strconv.Atoi(playerCountStr) +func numberOfVCPs(ctx context.Context, client *client.Client, namespace string) (uint, error) { + _, playerCount, err := parseVCPConfig(ctx, client, namespace) if err != nil { return 0, err } - if playerCount < 0 || playerCount > math.MaxUint32 { - return 0, fmt.Errorf("invalid playerCount '%d'- must be in range [0,%d]", playerCount, math.MaxUint32) - } - return uint(playerCount), nil + return playerCount, nil } diff --git a/klyshko-operator/controllers/config_utils_test.go b/klyshko-operator/controllers/config_utils_test.go index 1fab045..37fc560 100644 --- a/klyshko-operator/controllers/config_utils_test.go +++ b/klyshko-operator/controllers/config_utils_test.go @@ -30,6 +30,15 @@ var _ = Context("Using config utils", func() { Expect(err).NotTo(HaveOccurred()) }) + usingConfigMap := func(ctx context.Context, data map[string]string) { + BeforeEach(func() { + vcp.createVCPConfig(ctx, "cs-vcp-config", "default", data) + }) + AfterEach(func() { + vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + }) + } + Describe("getting the local player Id", func() { When("when no configuration has been provided", func() { @@ -40,28 +49,29 @@ var _ = Context("Using config utils", func() { }) When("when valid configuration has been provided", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": "2", - "playerId": "0", - }) - }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + usingConfigMap(ctx, map[string]string{ + "playerCount": "2", + "playerId": "1", }) It("gives the right local player ID", func() { - Expect(localPlayerID(ctx, &vcp.k8sClient, "default")).To(Equal(uint(0))) + Expect(localPlayerID(ctx, &vcp.k8sClient, "default")).To(Equal(uint(1))) }) }) When("when the playerId K/V pair is missing", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": "2", - }) + usingConfigMap(ctx, map[string]string{ + "playerCount": "2", + }) + It("fails", func() { + _, err := localPlayerID(ctx, &vcp.k8sClient, "default") + Expect(err).To(HaveOccurred()) }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + }) + + When("when the playerId is negative", func() { + usingConfigMap(ctx, map[string]string{ + "playerCount": "2", + "playerId": "-1", }) It("fails", func() { _, err := localPlayerID(ctx, &vcp.k8sClient, "default") @@ -70,14 +80,9 @@ var _ = Context("Using config utils", func() { }) When("when the playerId is out of range", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": "2", - "playerId": "-1", - }) - }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + usingConfigMap(ctx, map[string]string{ + "playerCount": "2", + "playerId": "2", }) It("fails", func() { _, err := localPlayerID(ctx, &vcp.k8sClient, "default") @@ -86,14 +91,9 @@ var _ = Context("Using config utils", func() { }) When("when the playerId can't be parsed", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": "2", - "playerId": "a1b2", - }) - }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + usingConfigMap(ctx, map[string]string{ + "playerCount": "2", + "playerId": "a1b2", }) It("fails", func() { _, err := localPlayerID(ctx, &vcp.k8sClient, "default") @@ -112,14 +112,9 @@ var _ = Context("Using config utils", func() { }) When("when valid configuration has been provided", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": "2", - "playerId": "0", - }) - }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + usingConfigMap(ctx, map[string]string{ + "playerCount": "2", + "playerId": "0", }) It("gives the right number of Players", func() { Expect(numberOfVCPs(ctx, &vcp.k8sClient, "default")).To(Equal(uint(2))) @@ -127,13 +122,8 @@ var _ = Context("Using config utils", func() { }) When("when the playerCount K/V pair is missing", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerId": "0", - }) - }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + usingConfigMap(ctx, map[string]string{ + "playerId": "0", }) It("fails", func() { _, err := numberOfVCPs(ctx, &vcp.k8sClient, "default") @@ -141,15 +131,10 @@ var _ = Context("Using config utils", func() { }) }) - When("when the playerCount is out of range", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": "-1", - "playerId": "0", - }) - }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + When("when the playerCount is negative", func() { + usingConfigMap(ctx, map[string]string{ + "playerCount": "-1", + "playerId": "0", }) It("fails", func() { _, err := numberOfVCPs(ctx, &vcp.k8sClient, "default") @@ -157,15 +142,10 @@ var _ = Context("Using config utils", func() { }) }) - When("when the playerId can't be parsed", func() { - BeforeEach(func() { - vcp.createVCPConfig(ctx, "cs-vcp-config", "default", map[string]string{ - "playerCount": "a1b2", - "playerId": "0", - }) - }) - AfterEach(func() { - vcp.deleteVCPConfig(ctx, "cs-vcp-config", "default") + When("when playerCount can't be parsed", func() { + usingConfigMap(ctx, map[string]string{ + "playerCount": "a1b2", + "playerId": "0", }) It("fails", func() { _, err := numberOfVCPs(ctx, &vcp.k8sClient, "default") diff --git a/klyshko-operator/controllers/etcd_keys_test.go b/klyshko-operator/controllers/etcd_keys_test.go index a9c0bc8..52b9c05 100644 --- a/klyshko-operator/controllers/etcd_keys_test.go +++ b/klyshko-operator/controllers/etcd_keys_test.go @@ -36,7 +36,7 @@ var _ = When("Parsing a key", func() { }) }) - Context("from a roster entry key with non-integer playerId", func() { + Context("from a string that is not a valid roster entry key", func() { It("should fail", func() { malformedKey := rosterEntryKey.ToEtcdKey() + "m" _, err := ParseKey(malformedKey) @@ -44,7 +44,7 @@ var _ = When("Parsing a key", func() { }) }) - Context("from a roster entry key with non-unit32 playerId", func() { + Context("from a roster entry key with non-uint32 playerId", func() { It("should fail", func() { malformedKey := rosterEntryKey.ToEtcdKey() + fmt.Sprint(math.MaxUint32*2) _, err := ParseKey(malformedKey) @@ -60,4 +60,12 @@ var _ = When("Parsing a key", func() { }) }) + Context("from a wrongly prefixed roster entry key", func() { + It("should fail", func() { + wronglyPrefixedKey := "/klyshko/toaster/foo/bar" + _, err := ParseKey(wronglyPrefixedKey) + Expect(err).To(HaveOccurred()) + }) + }) + }) diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index b77bcac..15914be 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -182,12 +182,12 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R } return false } - nbrOfVCPs, err := numberOfVCPs(ctx, &r.Client, req.Namespace) + numberOfVCPs, err := numberOfVCPs(ctx, &r.Client, req.Namespace) if err != nil { return ctrl.Result{RequeueAfter: 60 * time.Second}, fmt.Errorf("can't read playerCount from VCP configuration: %w", err) } var state klyshkov1alpha1.TupleGenerationJobState - if uint(len(ownedBy)) < nbrOfVCPs { + if uint(len(ownedBy)) < numberOfVCPs { state = klyshkov1alpha1.JobPending } else if !allTerminated(ownedBy) { state = klyshkov1alpha1.JobRunning @@ -282,6 +282,8 @@ func (r *TupleGenerationJobReconciler) handleWatchEvent(ctx context.Context, ev r.handleRemoteTaskUpdate(ctx, k, ev) case RosterKey: r.handleJobUpdate(ctx, k, ev) + default: + panic(fmt.Sprintf("Unexpected key type encountered: %v", key)) } } @@ -320,6 +322,8 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key return } logger.Info("Job deleted") + default: + panic(fmt.Sprintf("Unexpected etcd event encounter: %v", ev)) } } From ff24388b303da4281bae9b9c0b06e1ebcec4cb17 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 09:27:05 +0100 Subject: [PATCH 41/69] Improve hack script logic Signed-off-by: Sven Trieflinger --- hack/deploy.sh | 17 +++++++++++++++-- klyshko-mp-spdz/hack/deploy.sh | 3 +++ klyshko-operator/Dockerfile | 1 + klyshko-operator/hack/deploy.sh | 28 +++++++++++++++++++++++++--- klyshko-provisioner/hack/deploy.sh | 3 +++ 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/hack/deploy.sh b/hack/deploy.sh index d314f3a..15d5589 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -9,6 +9,11 @@ # Fail, if any command fails set -e +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + # Make sure that the following versions "match", i.e., the charts application # version matches ETCD_VERSION (see https://artifacthub.io/packages/helm/bitnami/etcd). # Upgrading the Bitnami etcd chart does not work at the moment (see https://github.com/bitnami/bitnami-docker-etcd/pull/47). @@ -21,8 +26,16 @@ export ETCD_VERSION=v3.5.4 export ETCD_CHART_VERSION=8.3.1 kubectl config use-context "kind-apollo" -helm repo add bitnami https://charts.bitnami.com/bitnami --force-update -helm install test-etcd --set auth.rbac.enabled=false --set service.type=LoadBalancer bitnami/etcd --version ${ETCD_CHART_VERSION} || true +if ! kubectl wait --for=condition=ready pod test-etcd-0 --timeout=0s; then + echo -e "${YELLOW}Deploying etcd${NC}" + helm repo add bitnami https://charts.bitnami.com/bitnami --force-update + helm upgrade test-etcd --install --wait --set auth.rbac.create=false --set service.type=LoadBalancer bitnami/etcd --version ${ETCD_CHART_VERSION} + echo -e "${GREEN}Waiting for etcd to become available${NC}" + if ! kubectl wait --for=condition=ready pod test-etcd-0 --timeout=120s; then + echo -e "${RED}Failed to start etcd${NC}" + exit 1 + fi +fi ( cd klyshko-mp-spdz diff --git a/klyshko-mp-spdz/hack/deploy.sh b/klyshko-mp-spdz/hack/deploy.sh index 8c0b204..42a135c 100755 --- a/klyshko-mp-spdz/hack/deploy.sh +++ b/klyshko-mp-spdz/hack/deploy.sh @@ -10,8 +10,11 @@ set -e GREEN='\033[0;32m' +YELLOW='\033[1;33m' NC='\033[0m' # No Color +echo -e "${YELLOW}Deploying Klyshko MP-SPDZ${NC}" + declare -a CLUSTERS=("starbuck" "apollo") echo -e "${GREEN}Building code and image${NC}" diff --git a/klyshko-operator/Dockerfile b/klyshko-operator/Dockerfile index a0be8f1..12f1faa 100644 --- a/klyshko-operator/Dockerfile +++ b/klyshko-operator/Dockerfile @@ -14,6 +14,7 @@ RUN go mod download # Copy the go source COPY main.go main.go COPY api/ api/ +COPY castor/ castor/ COPY controllers/ controllers/ # Build diff --git a/klyshko-operator/hack/deploy.sh b/klyshko-operator/hack/deploy.sh index 1d8225f..81a164b 100755 --- a/klyshko-operator/hack/deploy.sh +++ b/klyshko-operator/hack/deploy.sh @@ -10,8 +10,25 @@ set -e GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' NC='\033[0m' # No Color +# Checks whether the given Klyshko CRD exists. +function crd_exists() { + kubectl get crds | grep "$1" && true + return $? +} + +# Delete all resources of the given Klyshko CRD (CRD to be provided without the klyshko.carbynestack.io/v1alpha1 suffix). +function delete_all() { + CRD=$1 + if crd_exists "$CRD.klyshko.carbnyestack.io/v1alpha1"; then + echo "Deleting CRD $CRD and associated resources" + kubectl delete --all "$CRD" + fi +} + # Makes etcdctl with version specified as environment variable ETCD_VERSION available in ./bin folder function provide_etcdctl() { REQUIRED_VERSION=$1 @@ -31,10 +48,13 @@ function provide_etcdctl() { DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download rm -f "/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" curl -L "${DOWNLOAD_URL}/${REQUIRED_VERSION}/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" -o "/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" + mkdir -p bin tar --extract --file="/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" -C bin/ "etcd-${REQUIRED_VERSION}-linux-amd64/etcdctl" --strip-components=1 rm -f "/tmp/etcd-${REQUIRED_VERSION}-linux-amd64.tar.gz" } +echo -e "${YELLOW}Deploying Klyshko Operator${NC}" + declare -a CLUSTERS=("starbuck" "apollo") echo -e "${GREEN}Undeploying from clusters${NC}" @@ -42,9 +62,11 @@ for c in "${CLUSTERS[@]}" do echo -e "${GREEN}Undeploying from $c${NC}" kubectl config use-context "kind-$c" - kubectl delete --all tuplegenerationjobs - kubectl delete --all tuplegenerationtasks - make undeploy IMG="carbynestack/klyshko-operator:v0.0.1" + delete_all tuplegenerationjobs + delete_all tuplegenerationtasks + if ! make undeploy IMG="carbynestack/klyshko-operator:v0.0.1"; then + echo -e "${RED}Undeploying operator failed. This is fine, if the operator has not been deployed before.${NC}" + fi done echo -e "${GREEN}Cleaning up etcd${NC}" diff --git a/klyshko-provisioner/hack/deploy.sh b/klyshko-provisioner/hack/deploy.sh index ae2b9e9..1ca2038 100755 --- a/klyshko-provisioner/hack/deploy.sh +++ b/klyshko-provisioner/hack/deploy.sh @@ -10,8 +10,11 @@ set -e GREEN='\033[0;32m' +YELLOW='\033[1;33m' NC='\033[0m' # No Color +echo -e "${YELLOW}Deploying Klyshko Provisioner${NC}" + declare -a CLUSTERS=("starbuck" "apollo") echo -e "${GREEN}Building code and image${NC}" From ab32c585da300b1509a18aa7452be92985836eba Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 10:02:37 +0100 Subject: [PATCH 42/69] Refactor flag handling and make etcd endpoint configurable Signed-off-by: Sven Trieflinger --- klyshko-operator/main.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index dfe5fe3..264b420 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -37,22 +37,20 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(klyshkov1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } +var ( + metricsAddr = flag.String("metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + enableLeaderElection = flag.Bool("leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + probeAddr = flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + etcdEndpoint = flag.String("etcd-endpoint", "172.18.1.129:2379", "The address of the etcd service used for cross VCP coordination.") + etcdDialTimeout = flag.Int("etcd-dial-timeout", 5, "The timeout (in seconds) for failing to establish a connection to the etcd service.") + castorURL = flag.String("castor-url", "http://cs-castor.default.svc.cluster.local:10100", "The base url of the castor service used to upload generated tuples.") +) + func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - var castorURL string - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&castorURL, "castor-url", "http://cs-castor.default.svc.cluster.local:10100", "The base url of the castor service.") opts := zap.Options{ Development: true, } @@ -63,10 +61,10 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, - MetricsBindAddress: metricsAddr, + MetricsBindAddress: *metricsAddr, Port: 9443, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, + HealthProbeBindAddress: *probeAddr, + LeaderElection: *enableLeaderElection, LeaderElectionID: "operator.klyshko.carbynestack.io", }) if err != nil { @@ -75,8 +73,8 @@ func main() { } etcdClient, err := clientv3.New(clientv3.Config{ - Endpoints: []string{"172.18.1.129:2379"}, - DialTimeout: 5 * time.Second, + Endpoints: []string{*etcdEndpoint}, + DialTimeout: time.Duration(*etcdDialTimeout) * time.Second, }) if err != nil { setupLog.Error(err, "unable to create etcd client", "controller", "TupleGenerationJob") @@ -87,7 +85,7 @@ func main() { setupLog.Error(err, "closing etcd client failed") }() - castorClient := castor.NewClient(castorURL) + castorClient := castor.NewClient(*castorURL) if err = controllers.NewTupleGenerationJobReconciler(mgr.GetClient(), mgr.GetScheme(), etcdClient, castorClient).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationJob") From afde2898adcd8ddfa41f2acace96bb06da415a8f Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 10:40:01 +0100 Subject: [PATCH 43/69] Improve documentation for configuration resources Signed-off-by: Sven Trieflinger --- README.md | 10 +++++----- klyshko-mp-spdz/README.md | 40 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1c4a8c5..239802d 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,11 @@ Remember to do this on all VCPs of your VC. ### Provide CRG Configuration -CRGs require some configuration that has to be provided using K8s config maps -and secrets. +CRGs typically require some configuration that has to be provided using K8s +config maps and secrets. While the existence of these resources is dictated by +the KII, their content is CRG implementation specific. Please refer to the CRG +documentation for detailed information on what is expected. The following +examples are for the [MP-SPDZ CRG](klyshko-mp-spdz/README.md). Public parameters are provided in a config map with name `io.carbynestack.engine.params` as follows: @@ -117,9 +120,6 @@ data: <>: <> ``` -Please consult the documentation of the CRG you want to use to understand -whether any extra arguments are expected. - ### Instantiating a Scheduler After configuration is done, you create a scheduler on **one** of the clusters diff --git a/klyshko-mp-spdz/README.md b/klyshko-mp-spdz/README.md index 2dac7f3..f5ec7cf 100644 --- a/klyshko-mp-spdz/README.md +++ b/klyshko-mp-spdz/README.md @@ -6,7 +6,45 @@ ability to generate fake tuples. For a high-level description of the Klyshko subsystem, its components, and how these interact, please see the [README] at the root of this repository. -## Foreign MAC Key Shares +## Configuration + +The MP-SPDZ fake tuple CRG expects the following configuration resources. + +## Public Parameters + +The only required public configuration parameter is the prime (`<>` +placeholder below) used for generating tuples for the prime field arithmetic. It +is specified using a config map as follows: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params +data: + prime: <> +``` + +## Secret Parameters + +The required secret parameters are the MAC key shares for the prime field +(`<>` placeholder below) and for the field of characteristic 2 +(`<>` below). They are specified using a K8s secret as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: io.carbynestack.engine.params.secret +type: Opaque +data: + mac_key_share_p: | + <> + mac_key_share_2: | + <> +``` + +## Additional Parameters The MP-SPDZ fake tuple generator requires all MAC key shares to be available to all parties. They are expected to be made available using the From 866d9a36bddd9d914a6fc00ea2b2b7a978df7bd9 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 10:59:39 +0100 Subject: [PATCH 44/69] Support for generic entrypoint instead of kii-run.sh script Signed-off-by: Sven Trieflinger --- README.md | 7 +++---- klyshko-mp-spdz/Dockerfile.fake-offline | 3 +++ .../controllers/tuplegenerationtask_controller.go | 7 ------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 239802d..d2d754b 100644 --- a/README.md +++ b/README.md @@ -160,10 +160,9 @@ during execution. See below for a detailed description. ### Entrypoint -The CRG docker image must contain a script called `kii-run.sh` in the working -directory that spawns the tuple generation process. The script must terminate -with a non-zero exit code if and only if the tuples can not be generated for -some reason. +The CRG docker image must spawn the tuple generation process when launched as a +container. The command given as the entrypoint must terminate with a non-zero +exit code if and only if the tuples could not be generated for some reason. ### Environment Variables diff --git a/klyshko-mp-spdz/Dockerfile.fake-offline b/klyshko-mp-spdz/Dockerfile.fake-offline index 50ea9e4..ee5ea25 100644 --- a/klyshko-mp-spdz/Dockerfile.fake-offline +++ b/klyshko-mp-spdz/Dockerfile.fake-offline @@ -8,3 +8,6 @@ FROM ghcr.io/carbynestack/spdz:5350e66 # Copy resources COPY kii-run.sh . + +# Run the kii-run script +ENTRYPOINT ["/bin/bash", "-c", "./kii-run.sh"] diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 6ea5450..2ff92cc 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -415,13 +415,6 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, { Name: "generator", Image: "carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT", // TODO Read from config - Command: []string{ - "/bin/bash", - "-c", - }, - Args: []string{ - "./kii-run.sh", - }, Env: []v1.EnvVar{ { Name: "KII_JOB_ID", From c85c85e57fdd343a26b50fafeadaaece75827d0f Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 11:19:21 +0100 Subject: [PATCH 45/69] Improve KII documentation by removing MP-SPDZ CRG specific information Signed-off-by: Sven Trieflinger --- README.md | 120 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index d2d754b..1c97139 100644 --- a/README.md +++ b/README.md @@ -71,54 +71,12 @@ make deploy IMG="carbynestack/klyshko-operator:v0.0.1" Remember to do this on all VCPs of your VC. -### Provide CRG Configuration +### Provide the Configuration -CRGs typically require some configuration that has to be provided using K8s -config maps and secrets. While the existence of these resources is dictated by -the KII, their content is CRG implementation specific. Please refer to the CRG -documentation for detailed information on what is expected. The following -examples are for the [MP-SPDZ CRG](klyshko-mp-spdz/README.md). - -Public parameters are provided in a config map with name -`io.carbynestack.engine.params` as follows: - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: io.carbynestack.engine.params -data: - prime: <> -``` - -Sensitive parameters are provided using a K8s secret with name -`io.carbynestack.engine.params.secret` as follows: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: io.carbynestack.engine.params.secret -type: Opaque -data: - mac_key_share_p: | - <> - mac_key_share_2: | - <> -``` - -Additional parameters _may_ be provided using a K8s config map with name -`io.carbynestack.engine.params.extra` as follows: - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: io.carbynestack.engine.params.extra -data: - <>: <> - <>: <> -``` +Klyshko requires CRG-specific configuration that is provided via K8s config maps +and secrets (see [here](#configuration-parameters) for details). Consult the +documentation of the [MP-SPDZ CRG](klyshko-mp-spdz/README.md) for information of +what has to be provided. ### Instantiating a Scheduler @@ -189,21 +147,69 @@ the tuple generation and provisioning process. - `KII_TUPLE_FILE`: The file the generated tuples must be written to. -### Prime +### Configuration Parameters + +CRGs typically require some configuration that has to be provided using K8s +config maps and secrets. While the existence of these resources is dictated by +the KII, their content is CRG implementation specific. Please refer to the CRG +documentation for detailed information on what is expected. The following +examples are for the [MP-SPDZ CRG](klyshko-mp-spdz/README.md). -The prime to be used for generating prime field tuples is provided in the file -`/etc/kii/params/prime`. +#### Public Parameters -### MAC Key Shares +Public, i.e., non-sensitive, parameters are provided in a config map with name +`io.carbynestack.engine.params` as follows: -The MAC key shares for prime and binary fields are made available as files -`mac_key_share_p` and `mac_key_share_2` in folder `/etc/kii/secret-params`. +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params +data: + prime: <> +``` + +They are provided to CRGs as files in the `/etc/kii/params/` folder. The file +`/etc/kii/params/prime` contains the `<>` in the example above. -### Additional Parameters +#### Secret Parameters + +Sensitive parameters are provided using a K8s secret with name +`io.carbynestack.engine.params.secret` as follows: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: io.carbynestack.engine.params.secret +type: Opaque +data: + mac_key_share_p: | + <> + mac_key_share_2: | + <> +``` + +They are made available to CRGs as files in the folder `/etc/kii/secret-params`. + +#### Additional Parameters + +Additional parameters *may* be provided using a K8s config map with name +`io.carbynestack.engine.params.extra` as follows: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: io.carbynestack.engine.params.extra +data: + <>: <> + <>: <> +``` -Some CRGs might require additional _non-standard_ parameters. These are made -available by the Klyshko runtime in folder `/etc/kii/extra-params`. For an -example of how this is used see the [MP-SPDZ fake tuple CRG][mp-spdz-fake]. +These are made available to CRGs by the Klyshko runtime as files in folder +`/etc/kii/extra-params`. For an example of how this is used see the +[MP-SPDZ fake tuple CRG][mp-spdz-fake]. ## Development From f43f25233b9482949ca3a7221f06f1399fc9616c Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 13:57:44 +0100 Subject: [PATCH 46/69] Make generator image configurable and replace CMD by ENTRYPOINT in provisioner Signed-off-by: Sven Trieflinger --- README.md | 14 +++++++----- klyshko-operator/api/v1alpha1/common_types.go | 21 ++++++++++++++++++ .../api/v1alpha1/tuplegenerationjob_types.go | 2 ++ .../tuplegenerationscheduler_types.go | 7 +++++- .../api/v1alpha1/zz_generated.deepcopy.go | 17 ++++++++++++++ ...o.carbnyestack.io_tuplegenerationjobs.yaml | 20 +++++++++++++++++ ...nyestack.io_tuplegenerationschedulers.yaml | 22 ++++++++++++++++++- .../klyshko_v1alpha1_tuplegenerationjob.yaml | 3 +++ ...hko_v1alpha1_tuplegenerationscheduler.yaml | 3 +++ .../controllers/controller_test.go | 3 +++ .../tuplegenerationscheduler_controller.go | 7 +++--- .../tuplegenerationtask_controller.go | 7 +++--- klyshko-provisioner/Dockerfile | 2 +- 13 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 klyshko-operator/api/v1alpha1/common_types.go diff --git a/README.md b/README.md index 1c97139..a03c583 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,16 @@ metadata: spec: concurrency: 3 threshold: 500000 + generator: + image: carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT + imagePullPolicy: IfNotPresent ``` -Klyshko will start producing correlated randomness by creating respective jobs -whenever the number of tuples for a specific type drops below the given -`threshold`. `concurrency` specifies the maximum number of jobs that are allowed -to run concurrently. This is the upper limit across jobs for all tuple types. +Klyshko will start producing correlated randomness using the given CRG image by +creating jobs whenever the number of tuples for a specific type drops below the +given `threshold`. `concurrency` specifies the maximum number of jobs that are +allowed to run concurrently. This is the upper limit across jobs for all tuple +types. ## Klyshko Integration Interface (KII) @@ -236,4 +240,4 @@ Please see the Carbyne Stack [Contributor's Guide](https://github.com/carbynestack/carbynestack/blob/master/CONTRIBUTING.md) . -[mp-spdz-fake]: klyshko-mp-spdz/README.md#foreign-mac-key-shares +[mp-spdz-fake]: klyshko-mp-spdz/README.md#additional-parameters diff --git a/klyshko-operator/api/v1alpha1/common_types.go b/klyshko-operator/api/v1alpha1/common_types.go new file mode 100644 index 0000000..9fff484 --- /dev/null +++ b/klyshko-operator/api/v1alpha1/common_types.go @@ -0,0 +1,21 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package v1alpha1 + +import v1 "k8s.io/api/core/v1" + +// GeneratorSpec is a description of a Correlated Randomness Generator. +type GeneratorSpec struct { + // Container image name. + Image string `json:"image"` + + // Image pull policy specifies under which circumstances the image is pulled from the registry. + //+kubebuilder:default=IfNotPresent + //+kubebuilder:validation:Enum=IfNotPresent;Always;Never + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` +} diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go index 005f362..a1a6a80 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationjob_types.go @@ -54,6 +54,8 @@ type TupleGenerationJobSpec struct { //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:ExclusiveMinimum=true Count int `json:"count"` + + Generator GeneratorSpec `json:"generator"` } // TupleGenerationJobStatus defines the observed state of a TupleGenerationJob. diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go index 812204b..61cb34d 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go @@ -13,10 +13,13 @@ import ( // TupleGenerationSchedulerSpec defines the desired state of a TupleGenerationScheduler. type TupleGenerationSchedulerSpec struct { + + //+kubebuilder:default=1 //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:ExclusiveMinimum=false - Concurrency int `json:"concurrency"` + Concurrency int `json:"concurrency,omitempty"` + //+kubebuilder:validation:Required //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:ExclusiveMinimum=true Threshold int `json:"threshold"` @@ -25,6 +28,8 @@ type TupleGenerationSchedulerSpec struct { //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:ExclusiveMinimum=true TTLSecondsAfterFinished int `json:"ttlSecondsAfterFinished"` + + Generator GeneratorSpec `json:"generator"` } // TupleGenerationSchedulerStatus defines the observed state of a TupleGenerationScheduler. diff --git a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go index 4a96bdc..13138fb 100644 --- a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GeneratorSpec) DeepCopyInto(out *GeneratorSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GeneratorSpec. +func (in *GeneratorSpec) DeepCopy() *GeneratorSpec { + if in == nil { + return nil + } + out := new(GeneratorSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TupleGenerationJob) DeepCopyInto(out *TupleGenerationJob) { *out = *in @@ -86,6 +101,7 @@ func (in *TupleGenerationJobList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TupleGenerationJobSpec) DeepCopyInto(out *TupleGenerationJobSpec) { *out = *in + out.Generator = in.Generator } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationJobSpec. @@ -176,6 +192,7 @@ func (in *TupleGenerationSchedulerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TupleGenerationSchedulerSpec) DeepCopyInto(out *TupleGenerationSchedulerSpec) { *out = *in + out.Generator = in.Generator } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TupleGenerationSchedulerSpec. diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml index bbdef54..a36fe74 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationjobs.yaml @@ -56,6 +56,25 @@ spec: exclusiveMinimum: true minimum: 0 type: integer + generator: + description: GeneratorSpec is a description of a Correlated Randomness + Generator. + properties: + image: + description: Container image name. + type: string + imagePullPolicy: + default: IfNotPresent + description: Image pull policy specifies under which circumstances + the image is pulled from the registry. + enum: + - IfNotPresent + - Always + - Never + type: string + required: + - image + type: object id: type: string type: @@ -73,6 +92,7 @@ spec: type: string required: - count + - generator - id - type type: object diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml index 4d21510..8effedb 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml @@ -39,8 +39,28 @@ spec: a TupleGenerationScheduler. properties: concurrency: + default: 1 minimum: 0 type: integer + generator: + description: GeneratorSpec is a description of a Correlated Randomness + Generator. + properties: + image: + description: Container image name. + type: string + imagePullPolicy: + default: IfNotPresent + description: Image pull policy specifies under which circumstances + the image is pulled from the registry. + enum: + - IfNotPresent + - Always + - Never + type: string + required: + - image + type: object threshold: exclusiveMinimum: true minimum: 0 @@ -51,7 +71,7 @@ spec: minimum: 0 type: integer required: - - concurrency + - generator - threshold - ttlSecondsAfterFinished type: object diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml index c022a57..f09da98 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationjob.yaml @@ -12,3 +12,6 @@ spec: id: 0f990412-e8a0-4ef3-8e3f-a6a3a9191948 type: MULTIPLICATION_TRIPLE_GFP count: 10000 + generator: + image: carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT + imagePullPolicy: IfNotPresent diff --git a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml index 69a9ecf..f4b9645 100644 --- a/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml +++ b/klyshko-operator/config/samples/klyshko_v1alpha1_tuplegenerationscheduler.yaml @@ -11,3 +11,6 @@ metadata: spec: concurrency: 2 threshold: 50000 + generator: + image: carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT + imagePullPolicy: IfNotPresent diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go index f1e9ae7..41c233f 100644 --- a/klyshko-operator/controllers/controller_test.go +++ b/klyshko-operator/controllers/controller_test.go @@ -497,6 +497,9 @@ func createScheduler(ctx context.Context, vc *vc) *klyshkov1alpha1.TupleGenerati Concurrency: SchedulerConcurrency, Threshold: SchedulerTupleThreshold, TTLSecondsAfterFinished: 5, + Generator: klyshkov1alpha1.GeneratorSpec{ + Image: "carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT", + }, }, } Expect(vc.vcps[0].k8sClient.Create(ctx, scheduler)).Should(Succeed()) diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index d28461f..8e67128 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -140,9 +140,10 @@ func (r *TupleGenerationSchedulerReconciler) createJob(ctx context.Context, sche Namespace: scheduler.Namespace, }, Spec: klyshkov1alpha1.TupleGenerationJobSpec{ - ID: jobID, - Type: tupleType, - Count: MinimumTuplesPerJob, // TODO Make this configurable + ID: jobID, + Type: tupleType, + Count: MinimumTuplesPerJob, // TODO Make this configurable + Generator: scheduler.Spec.Generator, }, Status: klyshkov1alpha1.TupleGenerationJobStatus{ State: klyshkov1alpha1.JobPending, diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 2ff92cc..471f4cc 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -278,7 +278,7 @@ func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key *Rost }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "storage": resource.MustParse("100Mi"), // TODO Make configurable + "storage": resource.MustParse("100Mi"), // TODO Can this be computed from the number of tuples? }, }, }, @@ -413,8 +413,9 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "generator", - Image: "carbynestack/klyshko-mp-spdz:1.0.0-SNAPSHOT", // TODO Read from config + Name: "generator", + Image: job.Spec.Generator.Image, + ImagePullPolicy: job.Spec.Generator.ImagePullPolicy, Env: []v1.EnvVar{ { Name: "KII_JOB_ID", diff --git a/klyshko-provisioner/Dockerfile b/klyshko-provisioner/Dockerfile index e2d70fe..fa70753 100644 --- a/klyshko-provisioner/Dockerfile +++ b/klyshko-provisioner/Dockerfile @@ -13,4 +13,4 @@ ADD https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-$CLI_ # Copy resources COPY kii-provisioner-run.sh . -CMD ["/bin/bash", "-c", "./kii-provisioner-run.sh"] +ENTRYPOINT ["/bin/bash", "-c", "./kii-provisioner-run.sh"] From 19f22e0f2a581ca19936204caa6ed518022384e7 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 16:22:19 +0100 Subject: [PATCH 47/69] Improve logging Signed-off-by: Sven Trieflinger --- README.md | 35 +++++++++++++++++ klyshko-operator/Dockerfile | 1 + klyshko-operator/castor/castor_client.go | 5 ++- .../default/manager_auth_proxy_patch.yaml | 1 + .../tuplegenerationjob_controller.go | 38 ++++++++++--------- .../tuplegenerationscheduler_controller.go | 19 +++++----- .../tuplegenerationtask_controller.go | 27 ++++++------- klyshko-operator/logging/levels.go | 17 +++++++++ klyshko-operator/main.go | 3 ++ 9 files changed, 104 insertions(+), 42 deletions(-) create mode 100644 klyshko-operator/logging/levels.go diff --git a/README.md b/README.md index a03c583..320257f 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,40 @@ be used to (re-)deploy Klyshko to a 2-party Carbyne Stack VC setup as described in the [tutorials](https://carbynestack.io/getting-started) on the Carbyne Stack website. +### Logging + +#### Verbosity + +The Klyshko operator uses the [logging infrastructure][o-sdk-logging] provided +by the Operator SDK. To adjust the logging verbosity set the `zap-log-level` +flag to either `info`, `error`, or any integer value > 0 (higher values = more +verbose, see table below). + +```yaml +apiVersion: apps/v1 +kind: Deployment +# ... +spec: + template: + spec: + containers: + # ... + - name: manager + args: + # ... + - "--zap-log-level=<>" +``` + +#### Choosing Log Levels + +We use the following logging level convention in the Klyshko code basis. + +| Meaning | Level | Command | +| --------- | ----- | -------------------------------- | +| Essential | 0 | `logger.Info()/Error()` | +| Debug | 5 | `logger.V(DEBUG).Info()/Error()` | +| Tracing | 10 | `logger.V(TRACE).Info()/Error()` | + ## License Carbyne Stack *Klyshko Correlated Randomness Generation Subsystem* is @@ -241,3 +275,4 @@ Please see the Carbyne Stack . [mp-spdz-fake]: klyshko-mp-spdz/README.md#additional-parameters +[o-sdk-logging]: https://sdk.operatorframework.io/docs/building-operators/golang/references/logging/ diff --git a/klyshko-operator/Dockerfile b/klyshko-operator/Dockerfile index 12f1faa..d36feb1 100644 --- a/klyshko-operator/Dockerfile +++ b/klyshko-operator/Dockerfile @@ -16,6 +16,7 @@ COPY main.go main.go COPY api/ api/ COPY castor/ castor/ COPY controllers/ controllers/ +COPY logging/ logging/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go diff --git a/klyshko-operator/castor/castor_client.go b/klyshko-operator/castor/castor_client.go index 27dc8c0..8cadb6b 100644 --- a/klyshko-operator/castor/castor_client.go +++ b/klyshko-operator/castor/castor_client.go @@ -11,6 +11,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/carbynestack/klyshko/logging" "github.com/google/uuid" "io" "io/ioutil" @@ -36,7 +37,7 @@ func NewClient(url string) *Client { func (c Client) ActivateTupleChunk(ctx context.Context, chunkID uuid.UUID) error { logger := log.FromContext(ctx).WithValues("TupleChunkId", chunkID) url := fmt.Sprintf("%s/intra-vcp/tuple-chunks/activate/%s", c.URL, chunkID) - logger.Info("Activating tuple chunk with castor URL", "URL", url) + logger.V(logging.DEBUG).Info("Activating tuple chunk with castor URL", "URL", url) req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, nil) if err != nil { return err @@ -58,7 +59,7 @@ func (c Client) ActivateTupleChunk(ctx context.Context, chunkID uuid.UUID) error logger.Error(err, "Failed to close response from castor") } }() - logger.Info("Response from castor", "Status", resp.Status) + logger.V(logging.DEBUG).Info("Response from castor", "Status", resp.Status) return nil } diff --git a/klyshko-operator/config/default/manager_auth_proxy_patch.yaml b/klyshko-operator/config/default/manager_auth_proxy_patch.yaml index 4e2232f..077701e 100644 --- a/klyshko-operator/config/default/manager_auth_proxy_patch.yaml +++ b/klyshko-operator/config/default/manager_auth_proxy_patch.yaml @@ -25,3 +25,4 @@ spec: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" + - "--zap-log-level=info" diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index 15914be..b7b9be6 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -13,6 +13,7 @@ import ( "fmt" klyshkov1alpha1 "github.com/carbynestack/klyshko/api/v1alpha1" "github.com/carbynestack/klyshko/castor" + "github.com/carbynestack/klyshko/logging" "github.com/go-logr/logr" "github.com/google/uuid" "go.etcd.io/etcd/api/v3/mvccpb" @@ -68,7 +69,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R req.NamespacedName, } logger := r.Logger.WithValues("Job.Key", jobKey) - logger.Info("Reconciling tuple generation jobs") + logger.V(logging.DEBUG).Info("Reconciling tuple generation jobs") // Cleanup if job has been deleted job := &klyshkov1alpha1.TupleGenerationJob{} @@ -80,13 +81,13 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R if err != nil { return ctrl.Result{}, fmt.Errorf("failed to delete roster for job %v: %w", req.Name, err) } - logger.Info("Roster deleted") + logger.V(logging.DEBUG).Info("Roster deleted") return ctrl.Result{}, nil } // Error reading the object - requeue the request. return ctrl.Result{}, fmt.Errorf("failed to read resource for job %v: %w", req.Name, err) } - logger.Info("Job exists already") + logger.V(logging.TRACE).Info("Job exists already") // Create roster if not existing (no etcd transaction needed as remote job creation is triggered by roster creation) resp, err := r.EtcdClient.Get(ctx, jobKey.ToEtcdKey()) @@ -96,7 +97,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R if resp.Count == 0 { playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) if playerID != 0 { - logger.Info("Roster not available, retrying later") + logger.V(logging.DEBUG).Info("Roster not available, retrying later") return ctrl.Result{}, nil } encoded, err := json.Marshal(job.Spec) @@ -107,9 +108,9 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create roster for job %v: %w", req.Name, err) } - logger.Info("Roster created") + logger.V(logging.DEBUG).Info("Roster created") } else { - logger.Info("Roster exists already") + logger.V(logging.TRACE).Info("Roster exists already") } // Create local task if not existing @@ -133,13 +134,13 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create local task for job %v: %w", req.Name, err) } - logger.Info("Local task created", "Task.Name", task.Name) + logger.V(logging.DEBUG).Info("Local task created", "Task.Name", task.Name) return ctrl.Result{Requeue: true}, nil } // Error reading resource, requeue return ctrl.Result{}, fmt.Errorf("failed to read task resource for job %v: %w", req.Name, err) } - logger.Info("Local task exists already", "Task.Name", task.Name) + logger.V(logging.TRACE).Info("Local task exists already", "Task.Name", task.Name) // Update job status based on owned task statuses; TODO That might not scale well in case we have many jobs tasks := &klyshkov1alpha1.TupleGenerationTaskList{} @@ -163,7 +164,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R ownedBy = append(ownedBy, t) } } - logger.Info("Collected statuses of owned tasks", "Tasks", ownedBy) + logger.V(logging.DEBUG).Info("Collected statuses of owned tasks", "Tasks", ownedBy) // Helper functions; TODO Consider moving this to state class allTerminated := func(tasks []klyshkov1alpha1.TupleGenerationTask) bool { @@ -205,9 +206,10 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R if err != nil { return ctrl.Result{}, fmt.Errorf("tuple chunk activation failed for job %v: %w", job.Name, err) } + logger.Info("Job done", "Job", job) } if state.IsValid() && state != job.Status.State { - logger.Info("State update", "from", job.Status.State, "to", state) + logger.V(logging.DEBUG).Info("State update", "from", job.Status.State, "to", state) job.Status.State = state job.Status.LastStateTransitionTime = metav1.Now() err = r.Status().Update(ctx, job) @@ -216,7 +218,7 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R } } - logger.Info("Desired state reached") + logger.V(logging.TRACE).Info("Desired state reached") return ctrl.Result{}, nil } @@ -266,7 +268,7 @@ func (r *TupleGenerationJobReconciler) handleWatchEvent(ctx context.Context, ev return } logger := r.Logger.WithValues("Key", key, "Value", string(ev.Kv.Value), "Type", ev.Type) - logger.Info("Processing roster event") + logger.V(logging.DEBUG).Info("Processing roster event") switch k := key.(type) { case RosterEntryKey: @@ -304,7 +306,7 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key logger.Error(err, "Failed to create job") return } - logger.Info("Job created") + logger.V(logging.DEBUG).Info("Job created") case mvccpb.DELETE: // Delete job iff exists found := &klyshkov1alpha1.TupleGenerationJob{} @@ -321,7 +323,7 @@ func (r *TupleGenerationJobReconciler) handleJobUpdate(ctx context.Context, key logger.Error(err, "Job deletion failed") return } - logger.Info("Job deleted") + logger.V(logging.DEBUG).Info("Job deleted") default: panic(fmt.Sprintf("Unexpected etcd event encounter: %v", ev)) } @@ -361,7 +363,7 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex logger.Error(err, "Failed to update proxy task") return } - logger.Info("Updated state", "State.New", status) + logger.V(logging.DEBUG).Info("Updated state", "State.New", status) } else { if !apierrors.IsNotFound(err) { logger.Error(err, "Failed to fetch task") @@ -384,7 +386,7 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex } return } - logger.Info("Proxy task created") + logger.V(logging.DEBUG).Info("Proxy task created") } case mvccpb.DELETE: // Delete task for job if exists @@ -406,7 +408,7 @@ func (r *TupleGenerationJobReconciler) handleRemoteTaskUpdate(ctx context.Contex logger.Error(err, "Proxy task deletion failed") return } - logger.Info("Proxy task deleted") + logger.V(logging.DEBUG).Info("Proxy task deleted") } } @@ -424,7 +426,7 @@ func (r *TupleGenerationJobReconciler) createJobIfNotExists(ctx context.Context, }, Spec: *jobSpec, } - logger.Info("Creating a new job") + logger.V(logging.DEBUG).Info("Creating a new job") return r.Create(ctx, job) } return err diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 8e67128..768cfdb 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "github.com/carbynestack/klyshko/castor" + "github.com/carbynestack/klyshko/logging" "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,7 +47,7 @@ type TupleGenerationSchedulerReconciler struct { // to bring the actual state closer to the desired one. func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) - logger.Info("Reconciling tuple generation schedulers") + logger.V(logging.DEBUG).Info("Reconciling tuple generation schedulers") // Fetch scheduler resource scheduler := &klyshkov1alpha1.TupleGenerationScheduler{} @@ -76,7 +77,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req // Stop if already at maximum concurrency level activeJobCount := len(activeJobs) if scheduler.Spec.Concurrency <= activeJobCount { - logger.Info("At maximum concurrency level", "Jobs.Active", activeJobCount, "Scheduler.Concurrency", scheduler.Spec.Concurrency) + logger.V(logging.DEBUG).Info("At maximum concurrency level - do nothing", "Jobs.Active", activeJobCount, "Scheduler.Concurrency", scheduler.Spec.Concurrency) return ctrl.Result{RequeueAfter: 60 * time.Second}, nil } @@ -89,7 +90,7 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req logger.Error(err, "Fetching telemetry data from Castor failed", "Castor.URL", r.CastorClient.URL) return ctrl.Result{RequeueAfter: 60 * time.Second}, err } - logger.Info("Tuple telemetry data fetched", "Metrics.Available", telemetry.TupleMetrics) + logger.V(logging.TRACE).Info("Tuple telemetry data fetched", "Metrics.Available", telemetry.TupleMetrics) for _, j := range activeJobs { for idx := range telemetry.TupleMetrics { if j.Spec.Type == telemetry.TupleMetrics[idx].TupleType { @@ -98,22 +99,22 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req } } } - logger.Info("With in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) + logger.V(logging.TRACE).Info("With in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) var belowThreshold []castor.TupleMetrics for _, m := range telemetry.TupleMetrics { if m.Available < scheduler.Spec.Threshold { belowThreshold = append(belowThreshold, m) } } - logger.Info("Filtered for eligible types", "Metrics.Eligible", belowThreshold) + logger.V(logging.TRACE).Info("Filtered for eligible types", "Metrics.Eligible", belowThreshold) if len(belowThreshold) == 0 { - logger.Info("Above threshold for all tuple types - do nothing") + logger.V(logging.DEBUG).Info("Above threshold for all tuple types - do nothing") return ctrl.Result{RequeueAfter: 60 * time.Second}, nil } sort.Slice(belowThreshold, func(i, j int) bool { return belowThreshold[i].Available < belowThreshold[j].Available }) - logger.Info("Sorted by priority", "Metrics.Sorted", belowThreshold) + logger.V(logging.DEBUG).Info("Sorted by priority", "Metrics.Sorted", belowThreshold) // Create job for first tuple type below threshold err = r.createJob(ctx, scheduler, belowThreshold[0].TupleType) @@ -176,7 +177,7 @@ func (r *TupleGenerationSchedulerReconciler) cleanupFinishedJobs(ctx context.Con logger.Error(err, "failed to fetch finished jobs") return err } - logger.Info("Deleting finished jobs", "jobs", finishedJobs) + logger.V(logging.DEBUG).Info("Deleting finished jobs", "jobs", finishedJobs) for _, j := range finishedJobs { err := r.Delete(ctx, &j) if err != nil { @@ -195,7 +196,7 @@ func (r *TupleGenerationSchedulerReconciler) getMatchingJobs(ctx context.Context if err != nil { return nil, err } - logger.Info("Considering potentially matching jobs", "jobs", allJobs) + logger.V(logging.TRACE).Info("Considering potentially matching jobs", "jobs", allJobs) var matchingJobs []klyshkov1alpha1.TupleGenerationJob for _, j := range allJobs.Items { if pred(j) { diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index 471f4cc..db998f9 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -11,6 +11,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/carbynestack/klyshko/logging" clientv3 "go.etcd.io/etcd/client/v3" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -45,7 +46,7 @@ type TupleGenerationTaskReconciler struct { // bring the actual state closer to the desired one. func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues("Task.Name", req.Name) - logger.Info("Reconciling tuple generation tasks") + logger.V(logging.DEBUG).Info("Reconciling tuple generation tasks") taskKey, err := r.taskKeyFromName(req.Namespace, req.Name) if err != nil { @@ -71,13 +72,13 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. if err != nil { return ctrl.Result{}, fmt.Errorf("failed to delete roster entry for task %v: %w", req.Name, err) } - logger.Info("Roster entry deleted") + logger.V(logging.DEBUG).Info("Roster entry deleted") return ctrl.Result{}, nil } // Error reading the object - requeue the request. return ctrl.Result{}, fmt.Errorf("failed to read resource for task %v: %w", req.Name, err) } - logger.Info("Task exists already") + logger.V(logging.TRACE).Info("Task exists already") // Create roster entry if not existing resp, err := r.EtcdClient.Get(ctx, taskKey.ToEtcdKey()) @@ -90,9 +91,9 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create roster entry for task %v: %w", req.Name, err) } - logger.Info("Roster entry created") + logger.V(logging.DEBUG).Info("Roster entry created") } else { - logger.Info("Roster entry exists already") + logger.V(logging.TRACE).Info("Roster entry exists already") } status, err := r.getStatus(ctx, *taskKey) if err != nil { @@ -170,7 +171,7 @@ func (r *TupleGenerationTaskReconciler) Reconcile(ctx context.Context, req ctrl. } } - logger.Info("Desired state reached") + logger.V(logging.DEBUG).Info("Desired state reached") return ctrl.Result{}, nil } @@ -244,7 +245,7 @@ func (r *TupleGenerationTaskReconciler) setStatus(ctx context.Context, taskKey R // setState updates the given status object with the given state and writes the status to etcd. func (r *TupleGenerationTaskReconciler) setState(ctx context.Context, taskKey RosterEntryKey, status *klyshkov1alpha1.TupleGenerationTaskStatus, state klyshkov1alpha1.TupleGenerationTaskState) error { logger := log.FromContext(ctx).WithValues("Task.Key", taskKey) - logger.Info("Task transitioning into new state", "from", status.State, "to", state) + logger.V(logging.DEBUG).Info("Task transitioning into new state", "from", status.State, "to", state) status.State = state return r.setStatus(ctx, taskKey, status) } @@ -264,7 +265,7 @@ func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key *Rost found := &v1.PersistentVolumeClaim{} err := r.Get(ctx, name, found) if err == nil { - logger.Info("Persistent volume claim already exists") + logger.V(logging.TRACE).Info("Persistent volume claim already exists") return nil } pvc := &v1.PersistentVolumeClaim{ @@ -283,7 +284,7 @@ func (r *TupleGenerationTaskReconciler) createPVC(ctx context.Context, key *Rost }, }, } - logger.Info("Creating persistent volume claim", "PVC", pvc) + logger.V(logging.DEBUG).Info("Creating persistent volume claim", "PVC", pvc) err = r.Create(ctx, pvc) if err != nil { return fmt.Errorf("persistent volume claim creation failed for task %v: %w", key, err) @@ -320,7 +321,7 @@ func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context } found, err := r.getProvisionerPod(ctx, key) if err == nil { - logger.Info("Provisioner pod already exists") + logger.V(logging.TRACE).Info("Provisioner pod already exists") return found, nil } pod := &v1.Pod{ @@ -366,7 +367,7 @@ func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context }, }, } - logger.Info("Creating provisioner pod", "Pod", pod) + logger.V(logging.DEBUG).Info("Creating provisioner pod", "Pod", pod) err = ctrl.SetControllerReference(task, pod, r.Scheme) if err != nil { return nil, fmt.Errorf("setting the owner reference for task %v failed: %w", name, err) @@ -398,7 +399,7 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, logger := log.FromContext(ctx).WithValues("Task.Key", key) found, err := r.getGeneratorPod(ctx, task) if err == nil { - logger.Info("Pod already exists") + logger.V(logging.TRACE).Info("Pod already exists") return found, nil } vcpCount, err := numberOfVCPs(ctx, &r.Client, job.Namespace) @@ -510,7 +511,7 @@ func (r *TupleGenerationTaskReconciler) createGeneratorPod(ctx context.Context, }, }, } - logger.Info("Creating generator pod", "Pod", pod) + logger.V(logging.DEBUG).Info("Creating generator pod", "Pod", pod) err = ctrl.SetControllerReference(task, pod, r.Scheme) if err != nil { return nil, fmt.Errorf("setting the owner reference for task %v failed: %w", task.Name, err) diff --git a/klyshko-operator/logging/levels.go b/klyshko-operator/logging/levels.go new file mode 100644 index 0000000..141a357 --- /dev/null +++ b/klyshko-operator/logging/levels.go @@ -0,0 +1,17 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package logging + +const ( + + // DEBUG is the logging level giving diagnostic information in a detailed manner. + DEBUG = 5 + + // TRACE is the logging level giving all the details about the behavior of the application. + TRACE = 10 +) diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index 264b420..ccb1737 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -51,6 +51,9 @@ var ( ) func main() { + + // Add the zap logger flag set to the CLI + // (see https://sdk.operatorframework.io/docs/building-operators/golang/references/logging/ for more information) opts := zap.Options{ Development: true, } From 32cee0b0b7f894b78ffc1c4e7d317a5049e1e783 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 16 Dec 2022 16:25:09 +0100 Subject: [PATCH 48/69] Add target folders to .gitignore Signed-off-by: Sven Trieflinger --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 579edad..4309f71 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ # Go /klyshko-operator/vendor/ + +# Java +**/target From d959197da4ba0dc09842f7e7a2a1be66dbc9c4be Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 20 Dec 2022 08:49:42 +0100 Subject: [PATCH 49/69] Refator code to use strategy pattern for scheduling logic Signed-off-by: Sven Trieflinger --- klyshko-operator/castor/castor_client.go | 11 +++ .../controllers/scheduling_strategy.go | 76 ++++++++++++++++ .../controllers/scheduling_strategy_test.go | 89 +++++++++++++++++++ .../tuplegenerationscheduler_controller.go | 44 +++------ 4 files changed, 187 insertions(+), 33 deletions(-) create mode 100644 klyshko-operator/controllers/scheduling_strategy.go create mode 100644 klyshko-operator/controllers/scheduling_strategy_test.go diff --git a/klyshko-operator/castor/castor_client.go b/klyshko-operator/castor/castor_client.go index 8cadb6b..72a1b4e 100644 --- a/klyshko-operator/castor/castor_client.go +++ b/klyshko-operator/castor/castor_client.go @@ -75,6 +75,17 @@ type Telemetry struct { TupleMetrics []TupleMetrics `json:"metrics"` } +// DeepCopy creates a deep copy of a telemetry struct. +func (t Telemetry) DeepCopy() Telemetry { + c := Telemetry{ + TupleMetrics: []TupleMetrics{}, + } + for _, m := range t.TupleMetrics { + c.TupleMetrics = append(c.TupleMetrics, m) + } + return c +} + // GetTelemetry fetches telemetry data from the Castor service. func (c Client) GetTelemetry(ctx context.Context) (Telemetry, error) { logger := log.FromContext(ctx) diff --git a/klyshko-operator/controllers/scheduling_strategy.go b/klyshko-operator/controllers/scheduling_strategy.go new file mode 100644 index 0000000..77e9490 --- /dev/null +++ b/klyshko-operator/controllers/scheduling_strategy.go @@ -0,0 +1,76 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package controllers + +import ( + "context" + "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/carbynestack/klyshko/castor" + "github.com/carbynestack/klyshko/logging" + "sigs.k8s.io/controller-runtime/pkg/log" + "sort" +) + +// SchedulingStrategy is used to decide for which tuple type a tuple generation job should be launched next. +type SchedulingStrategy interface { + + // Schedule returns the tuple type for which tuples should be generated next based on the number of tuples + // available in Castor and the currently running jobs. In case no tuples should be generated `nil` is returned. + // Arguments must not be altered by implementations. + Schedule(ctx context.Context, telemetry castor.Telemetry, activeJobs []v1alpha1.TupleGenerationJob) *string +} + +// LeastAvailableFirstStrategy selects the tuple type for which the least tuples are available or being generated and +// for which the number is below a given threshold. +type LeastAvailableFirstStrategy struct { + + // Threshold is the lower bound of available tuples above which tuple types are eligible for being selected by this + // strategy. + Threshold int +} + +// Schedule returns the tuple type selected by this strategy or `nil` if none is eligible. +func (s *LeastAvailableFirstStrategy) Schedule(ctx context.Context, telemetry castor.Telemetry, activeJobs []v1alpha1.TupleGenerationJob) *string { + logger := log.FromContext(ctx) + + // Compute aggregate of available and in-flight tuples + t := telemetry.DeepCopy() + for _, j := range activeJobs { + for idx := range t.TupleMetrics { + if j.Spec.Type == t.TupleMetrics[idx].TupleType { + t.TupleMetrics[idx].Available = t.TupleMetrics[idx].Available + j.Spec.Count + break + } + } + } + logger.V(logging.DEBUG).Info("With in-flight tuple generation jobs", "Metrics.WithInflight", t.TupleMetrics) + + // Filter for those that are below threshold + var belowThreshold []castor.TupleMetrics + for _, m := range t.TupleMetrics { + if m.Available < s.Threshold { + belowThreshold = append(belowThreshold, m) + } + } + logger.V(logging.TRACE).Info("Filtered for eligible types", "Metrics.Eligible", belowThreshold) + + // Terminate early if no tuple type is below threshold + if len(belowThreshold) == 0 { + logger.V(logging.DEBUG).Info("Above threshold for all tuple types") + return nil + } + + // Sort ascending by quantity + sort.Slice(belowThreshold, func(i, j int) bool { + return belowThreshold[i].Available < belowThreshold[j].Available + }) + logger.V(logging.DEBUG).Info("Sorted by priority", "Metrics.Sorted", belowThreshold) + + // Return tuple type with minimum number of tuples available or being generated + return &belowThreshold[0].TupleType +} diff --git a/klyshko-operator/controllers/scheduling_strategy_test.go b/klyshko-operator/controllers/scheduling_strategy_test.go new file mode 100644 index 0000000..2983944 --- /dev/null +++ b/klyshko-operator/controllers/scheduling_strategy_test.go @@ -0,0 +1,89 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ +package controllers + +import ( + "context" + "github.com/carbynestack/klyshko/api/v1alpha1" + "github.com/carbynestack/klyshko/castor" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + //+kubebuilder:scaffold:imports +) + +var _ = Context("LeastAvailableFirstStrategy", func() { + + // Test data + const TupleTypeA = "A" + const TupleTypeB = "B" + telemetry := castor.Telemetry{ + TupleMetrics: []castor.TupleMetrics{ + { + Available: 5000, + ConsumptionRate: 0, + TupleType: TupleTypeA, + }, + { + Available: 50000, + ConsumptionRate: 0, + TupleType: TupleTypeB, + }, + }, + } + someActiveJobs := []v1alpha1.TupleGenerationJob{ + {Spec: v1alpha1.TupleGenerationJobSpec{ + Type: TupleTypeA, + Count: 30000, + }}, + } + var noActiveJobs []v1alpha1.TupleGenerationJob + + Describe("Selecting a tuple type", func() { + + When("there is no tuple type below the threshold", func() { + It("returns nil", func() { + strategy := &LeastAvailableFirstStrategy{Threshold: 1000} + selected := strategy.Schedule(context.Background(), telemetry, noActiveJobs) + Expect(selected).To(BeNil()) + }) + }) + + When("there is a single tuple type below the threshold", func() { + It("returns that tuple type", func() { + strategy := &LeastAvailableFirstStrategy{Threshold: 10000} + selected := strategy.Schedule(context.Background(), telemetry, noActiveJobs) + Expect(*selected).To(Equal(TupleTypeA)) + }) + }) + + When("there are two tuple types below the threshold", func() { + It("returns the tuple type with less tuples available", func() { + strategy := &LeastAvailableFirstStrategy{Threshold: 100000} + selected := strategy.Schedule(context.Background(), telemetry, noActiveJobs) + Expect(*selected).To(Equal(TupleTypeA)) + }) + }) + + When("there is a tuple type with an active job above the threshold", func() { + It("returns nil", func() { + strategy := &LeastAvailableFirstStrategy{Threshold: 30000} + selected := strategy.Schedule(context.Background(), telemetry, someActiveJobs) + Expect(selected).To(BeNil()) + }) + }) + + When("there is a tuple type with an active job below the threshold", func() { + It("returns that tuple type", func() { + strategy := &LeastAvailableFirstStrategy{Threshold: 40000} + selected := strategy.Schedule(context.Background(), telemetry, someActiveJobs) + Expect(*selected).To(Equal(TupleTypeA)) + }) + }) + + }) + +}) diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 768cfdb..8293f10 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -15,7 +15,6 @@ import ( "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sort" "time" "k8s.io/apimachinery/pkg/runtime" @@ -78,46 +77,25 @@ func (r *TupleGenerationSchedulerReconciler) Reconcile(ctx context.Context, req activeJobCount := len(activeJobs) if scheduler.Spec.Concurrency <= activeJobCount { logger.V(logging.DEBUG).Info("At maximum concurrency level - do nothing", "Jobs.Active", activeJobCount, "Scheduler.Concurrency", scheduler.Spec.Concurrency) - return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + return ctrl.Result{RequeueAfter: PeriodicReconciliationDuration}, nil } - // Least available first strategy for selecting tuple types to create a job for - // 1. Compute available and in generation number of tuples per type - // 2. Filter out all above threshold - // 3. Sort ascending wrt to sum from step 1 + // Fetch telemetry data from Castor telemetry, err := r.CastorClient.GetTelemetry(ctx) if err != nil { logger.Error(err, "Fetching telemetry data from Castor failed", "Castor.URL", r.CastorClient.URL) - return ctrl.Result{RequeueAfter: 60 * time.Second}, err + return ctrl.Result{RequeueAfter: PeriodicReconciliationDuration}, err } - logger.V(logging.TRACE).Info("Tuple telemetry data fetched", "Metrics.Available", telemetry.TupleMetrics) - for _, j := range activeJobs { - for idx := range telemetry.TupleMetrics { - if j.Spec.Type == telemetry.TupleMetrics[idx].TupleType { - telemetry.TupleMetrics[idx].Available = telemetry.TupleMetrics[idx].Available + j.Spec.Count - break - } - } - } - logger.V(logging.TRACE).Info("With in-flight tuple generation jobs", "Metrics.WithInflight", telemetry.TupleMetrics) - var belowThreshold []castor.TupleMetrics - for _, m := range telemetry.TupleMetrics { - if m.Available < scheduler.Spec.Threshold { - belowThreshold = append(belowThreshold, m) - } - } - logger.V(logging.TRACE).Info("Filtered for eligible types", "Metrics.Eligible", belowThreshold) - if len(belowThreshold) == 0 { - logger.V(logging.DEBUG).Info("Above threshold for all tuple types - do nothing") - return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + + // Decide which tuple type to generate next based on (for now fixed) strategy + var strategy SchedulingStrategy = &LeastAvailableFirstStrategy{Threshold: scheduler.Spec.Threshold} + tupleType := strategy.Schedule(ctx, telemetry, activeJobs) + if tupleType == nil { + return ctrl.Result{RequeueAfter: PeriodicReconciliationDuration}, nil } - sort.Slice(belowThreshold, func(i, j int) bool { - return belowThreshold[i].Available < belowThreshold[j].Available - }) - logger.V(logging.DEBUG).Info("Sorted by priority", "Metrics.Sorted", belowThreshold) - // Create job for first tuple type below threshold - err = r.createJob(ctx, scheduler, belowThreshold[0].TupleType) + // Create job for selected tuple type + err = r.createJob(ctx, scheduler, *tupleType) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create tuple generation job: %w", err) } From d85d27d075b83e15f7cf8e151e905fabe3ba1264 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 20 Dec 2022 09:20:22 +0100 Subject: [PATCH 50/69] Improve package documentation Signed-off-by: Sven Trieflinger --- klyshko-operator/api/v1alpha1/groupversion_info.go | 1 - klyshko-operator/castor/doc.go | 9 +++++++++ klyshko-operator/logging/doc.go | 9 +++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 klyshko-operator/castor/doc.go create mode 100644 klyshko-operator/logging/doc.go diff --git a/klyshko-operator/api/v1alpha1/groupversion_info.go b/klyshko-operator/api/v1alpha1/groupversion_info.go index 869cb5b..522c1c7 100644 --- a/klyshko-operator/api/v1alpha1/groupversion_info.go +++ b/klyshko-operator/api/v1alpha1/groupversion_info.go @@ -5,7 +5,6 @@ see the NOTICE file and/or the repository https://github.com/carbynestack/klyshk SPDX-License-Identifier: Apache-2.0 */ -// Package v1alpha1 contains API Schema definitions for the klyshko v1alpha1 API group //+kubebuilder:object:generate=true //+groupName=klyshko.carbnyestack.io package v1alpha1 diff --git a/klyshko-operator/castor/doc.go b/klyshko-operator/castor/doc.go new file mode 100644 index 0000000..5fee815 --- /dev/null +++ b/klyshko-operator/castor/doc.go @@ -0,0 +1,9 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package castor contains functionality for interacting with the Castor service. +package castor diff --git a/klyshko-operator/logging/doc.go b/klyshko-operator/logging/doc.go new file mode 100644 index 0000000..1095327 --- /dev/null +++ b/klyshko-operator/logging/doc.go @@ -0,0 +1,9 @@ +/* +Copyright (c) 2022 - for information on the respective copyright owner +see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package logging contains logging related code. +package logging From d043c9f0e0ed6fde23c257a1f1520623d2733c85 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 20 Dec 2022 10:02:37 +0100 Subject: [PATCH 51/69] Add initial GHA build logic Signed-off-by: Sven Trieflinger --- .github/workflows/mp-spdz.yaml | 42 ++++++++++++++++++++++++++++++ .github/workflows/operator.yaml | 41 +++++++++++++++++++++++++++++ .github/workflows/provisioner.yaml | 41 +++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 .github/workflows/mp-spdz.yaml create mode 100644 .github/workflows/operator.yaml create mode 100644 .github/workflows/provisioner.yaml diff --git a/.github/workflows/mp-spdz.yaml b/.github/workflows/mp-spdz.yaml new file mode 100644 index 0000000..f548f47 --- /dev/null +++ b/.github/workflows/mp-spdz.yaml @@ -0,0 +1,42 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/kylshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Build and publish MP-SPDZ Integration +on: + workflow_dispatch: + push: +defaults: + run: + working-directory: klyshko-mp-spdz +env: + REGISTRY: ghcr.io + IMAGE_NAME: carbynestack/klyshko-mp-spdz + WORKING_DIRECTORY: klyshko-mp-spdz +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: ${{ env.WORKING_DIRECTORY }} + file: ${{ env.WORKING_DIRECTORY }}/Dockerfile.fake-offline + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/operator.yaml b/.github/workflows/operator.yaml new file mode 100644 index 0000000..1a9da00 --- /dev/null +++ b/.github/workflows/operator.yaml @@ -0,0 +1,41 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/kylshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Build and publish Operator +on: + workflow_dispatch: + push: +defaults: + run: + working-directory: klyshko-operator +env: + REGISTRY: ghcr.io + IMAGE_NAME: carbynestack/klyshko-operator + WORKING_DIRECTORY: klyshko-operator +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: ${{ env.WORKING_DIRECTORY }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/provisioner.yaml b/.github/workflows/provisioner.yaml new file mode 100644 index 0000000..a418148 --- /dev/null +++ b/.github/workflows/provisioner.yaml @@ -0,0 +1,41 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/kylshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Build and publish Provisioner +on: + workflow_dispatch: + push: +defaults: + run: + working-directory: klyshko-provisioner +env: + REGISTRY: ghcr.io + IMAGE_NAME: carbynestack/klyshko-provisioner + WORKING_DIRECTORY: klyshko-provisioner +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: ${{ env.WORKING_DIRECTORY }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From c42d6f66a66265bff1dc6f571271ab0c8be8f8ca Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 20 Dec 2022 14:49:53 +0100 Subject: [PATCH 52/69] Rename from klyshko-go to klyshko Signed-off-by: Sven Trieflinger --- klyshko-operator/Makefile | 4 ++-- klyshko-operator/PROJECT | 2 +- klyshko-operator/config/default/kustomization.yaml | 4 ++-- klyshko-operator/config/manifests/kustomization.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/klyshko-operator/Makefile b/klyshko-operator/Makefile index 5e44643..401c912 100644 --- a/klyshko-operator/Makefile +++ b/klyshko-operator/Makefile @@ -30,8 +30,8 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# carbnyestack.io/klyshko-go-bundle:$VERSION and carbnyestack.io/klyshko-go-catalog:$VERSION. -IMAGE_TAG_BASE ?= carbnyestack.io/klyshko-go +# carbnyestack.io/klyshko-operator-bundle:$VERSION and carbnyestack.io/klyshko-operator-catalog:$VERSION. +IMAGE_TAG_BASE ?= ghcr.io/carbynestack/klyshko/klyshko-operator # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) diff --git a/klyshko-operator/PROJECT b/klyshko-operator/PROJECT index 5faaab7..5d1ec87 100644 --- a/klyshko-operator/PROJECT +++ b/klyshko-operator/PROJECT @@ -4,7 +4,7 @@ layout: plugins: manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} -projectName: klyshko-go +projectName: klyshko repo: github.com/carbynestack/klyshko resources: - api: diff --git a/klyshko-operator/config/default/kustomization.yaml b/klyshko-operator/config/default/kustomization.yaml index 99d84d5..1e12e4f 100644 --- a/klyshko-operator/config/default/kustomization.yaml +++ b/klyshko-operator/config/default/kustomization.yaml @@ -1,12 +1,12 @@ # Adds namespace to all resources. -namespace: klyshko-go-system +namespace: klyshko-system # Value of this field is prepended to the # names of all resources, e.g. a deployment named # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. -namePrefix: klyshko-go- +namePrefix: klyshko- # Labels to add to all resources and selectors. #commonLabels: diff --git a/klyshko-operator/config/manifests/kustomization.yaml b/klyshko-operator/config/manifests/kustomization.yaml index 1dee91b..d63e606 100644 --- a/klyshko-operator/config/manifests/kustomization.yaml +++ b/klyshko-operator/config/manifests/kustomization.yaml @@ -1,7 +1,7 @@ # These resources constitute the fully configured set of manifests # used to generate the 'manifests/' directory in a bundle. resources: -- bases/klyshko-go.clusterserviceversion.yaml +- bases/klyshko.clusterserviceversion.yaml - ../default - ../samples - ../scorecard From 2f38f7f16bd48fe66552216e95f9a09dafe78932 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 20 Dec 2022 16:38:53 +0100 Subject: [PATCH 53/69] Ignore bundle-related files generated by Operator SDK Signed-off-by: Sven Trieflinger --- klyshko-operator/.gitignore | 12 +++- .../config/manager/kustomization.yaml | 4 +- .../bases/klyshko.clusterserviceversion.yaml | 63 +++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 klyshko-operator/config/manifests/bases/klyshko.clusterserviceversion.yaml diff --git a/klyshko-operator/.gitignore b/klyshko-operator/.gitignore index c0a7a54..32d968b 100644 --- a/klyshko-operator/.gitignore +++ b/klyshko-operator/.gitignore @@ -1,3 +1,9 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# # Binaries for programs and plugins *.exe @@ -15,7 +21,6 @@ testbin/* *.out # Kubernetes Generated files - skip generated files, except for vendored files - !vendor/**/zz_generated.* # editor and IDE paraphernalia @@ -23,3 +28,8 @@ testbin/* *.swp *.swo *~ + +# Files generated by Operator SDK +bundle/ +bundle.Dockerfile +bundle_tmp* diff --git a/klyshko-operator/config/manager/kustomization.yaml b/klyshko-operator/config/manager/kustomization.yaml index 352c591..5e793dd 100644 --- a/klyshko-operator/config/manager/kustomization.yaml +++ b/klyshko-operator/config/manager/kustomization.yaml @@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: carbynestack/klyshko-operator - newTag: v0.0.1 + newName: controller + newTag: latest diff --git a/klyshko-operator/config/manifests/bases/klyshko.clusterserviceversion.yaml b/klyshko-operator/config/manifests/bases/klyshko.clusterserviceversion.yaml new file mode 100644 index 0000000..bb6a1f5 --- /dev/null +++ b/klyshko-operator/config/manifests/bases/klyshko.clusterserviceversion.yaml @@ -0,0 +1,63 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: '[]' + capabilities: Basic Install + name: klyshko.v0.0.0 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - description: TupleGenerationJob is the Schema for the TupleGenerationJob API. + displayName: Tuple Generation Job + kind: TupleGenerationJob + name: tuplegenerationjobs.klyshko.carbnyestack.io + version: v1alpha1 + - description: TupleGenerationScheduler is the Schema for the TupleGenerationScheduler + API. + displayName: Tuple Generation Scheduler + kind: TupleGenerationScheduler + name: tuplegenerationschedulers.klyshko.carbnyestack.io + version: v1alpha1 + - description: TupleGenerationTask is the Schema for the TupleGenerationTask API. + displayName: Tuple Generation Task + kind: TupleGenerationTask + name: tuplegenerationtasks.klyshko.carbnyestack.io + version: v1alpha1 + description: Operator for the correlated randomness generation subsystem of Carbyne + Stack + displayName: Klyshko Operator + icon: + - base64data: "" + mediatype: "" + install: + spec: + deployments: null + strategy: "" + installModes: + - supported: false + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - Carbyne Stack + - Correlated Randomness + links: + - name: Klyshko + url: https://klyshko.domain + maintainers: + - email: sven.trieflinger@de.bosch.com + name: strieflin + - email: sebastian.becker@de.bosch.com + name: sbckr + maturity: alpha + provider: + name: Carbyne Stack community + url: https://github.com/carbynestack/klyshko + version: 0.0.0 From 09be11b1b9c8b20fb8cb586aaf99e15284a4affe Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 20 Dec 2022 16:43:10 +0100 Subject: [PATCH 54/69] Fix typo in copyright headers Signed-off-by: Sven Trieflinger --- .github/workflows/mp-spdz.yaml | 2 +- .github/workflows/operator.yaml | 2 +- .github/workflows/provisioner.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mp-spdz.yaml b/.github/workflows/mp-spdz.yaml index f548f47..39b739b 100644 --- a/.github/workflows/mp-spdz.yaml +++ b/.github/workflows/mp-spdz.yaml @@ -1,6 +1,6 @@ # # Copyright (c) 2022 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/kylshko. +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. # # SPDX-License-Identifier: Apache-2.0 # diff --git a/.github/workflows/operator.yaml b/.github/workflows/operator.yaml index 1a9da00..716dd4b 100644 --- a/.github/workflows/operator.yaml +++ b/.github/workflows/operator.yaml @@ -1,6 +1,6 @@ # # Copyright (c) 2022 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/kylshko. +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. # # SPDX-License-Identifier: Apache-2.0 # diff --git a/.github/workflows/provisioner.yaml b/.github/workflows/provisioner.yaml index a418148..02e210a 100644 --- a/.github/workflows/provisioner.yaml +++ b/.github/workflows/provisioner.yaml @@ -1,6 +1,6 @@ # # Copyright (c) 2022 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/kylshko. +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. # # SPDX-License-Identifier: Apache-2.0 # From b68ff6a72b591737f686ea891f6dd6bdabe67f52 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 20 Dec 2022 16:55:23 +0100 Subject: [PATCH 55/69] Add test step to operator build Signed-off-by: Sven Trieflinger --- .github/workflows/operator.yaml | 23 ++++++++++++++++++- klyshko-operator/Makefile | 2 +- .../config/manager/kustomization.yaml | 4 ++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/operator.yaml b/.github/workflows/operator.yaml index 716dd4b..9be5a7d 100644 --- a/.github/workflows/operator.yaml +++ b/.github/workflows/operator.yaml @@ -13,7 +13,7 @@ defaults: working-directory: klyshko-operator env: REGISTRY: ghcr.io - IMAGE_NAME: carbynestack/klyshko-operator + IMAGE_NAME: carbynestack/klyshko-operator-controller WORKING_DIRECTORY: klyshko-operator jobs: publish: @@ -21,6 +21,27 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.16' + - name: Setup Build Cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Build and Test + run: | + make test + - name: Publishing Coverage + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + name: codecov - name: Log in to container registry uses: docker/login-action@v2 with: diff --git a/klyshko-operator/Makefile b/klyshko-operator/Makefile index 401c912..8430d35 100644 --- a/klyshko-operator/Makefile +++ b/klyshko-operator/Makefile @@ -38,7 +38,7 @@ IMAGE_TAG_BASE ?= ghcr.io/carbynestack/klyshko/klyshko-operator BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG ?= klyshko-operator-controller:v$(VERSION) # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. diff --git a/klyshko-operator/config/manager/kustomization.yaml b/klyshko-operator/config/manager/kustomization.yaml index 5e793dd..68039e1 100644 --- a/klyshko-operator/config/manager/kustomization.yaml +++ b/klyshko-operator/config/manager/kustomization.yaml @@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: controller - newTag: latest + newName: klyshko-operator-controller + newTag: v0.0.1 From 526f40c1597f3d42c1cbcaf7d0ac02422102187f Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Wed, 21 Dec 2022 08:57:51 +0100 Subject: [PATCH 56/69] Add helm chart for klyshko-operator Signed-off-by: Sven Trieflinger --- .github/workflows/operator.yaml | 16 ++ .../charts/klyshko-operator/.helmignore | 33 +++ .../charts/klyshko-operator/Chart.yaml | 12 + .../charts/klyshko-operator/README.md | 85 +++++++ .../klyshko-operator/templates/crds.yaml | 238 ++++++++++++++++++ .../templates/deployment.yaml | 93 +++++++ .../klyshko-operator/templates/rbac.yaml | 233 +++++++++++++++++ .../klyshko-operator/templates/service.yaml | 21 ++ .../charts/klyshko-operator/values.yaml | 17 ++ 9 files changed, 748 insertions(+) create mode 100644 klyshko-operator/charts/klyshko-operator/.helmignore create mode 100644 klyshko-operator/charts/klyshko-operator/Chart.yaml create mode 100644 klyshko-operator/charts/klyshko-operator/README.md create mode 100644 klyshko-operator/charts/klyshko-operator/templates/crds.yaml create mode 100644 klyshko-operator/charts/klyshko-operator/templates/deployment.yaml create mode 100644 klyshko-operator/charts/klyshko-operator/templates/rbac.yaml create mode 100644 klyshko-operator/charts/klyshko-operator/templates/service.yaml create mode 100644 klyshko-operator/charts/klyshko-operator/values.yaml diff --git a/.github/workflows/operator.yaml b/.github/workflows/operator.yaml index 9be5a7d..a58952d 100644 --- a/.github/workflows/operator.yaml +++ b/.github/workflows/operator.yaml @@ -60,3 +60,19 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + - name: Configure Helm Chart + run: | + VERSION=$(sed -n 's/version: \(.*\)/\1/p' charts/klyshko-operator/Chart.yaml)-${{ steps.meta.outputs.version }} + echo "VERSION=$VERSION" >> $GITHUB_ENV + sed -i 's/version: \(.*\)/version: '"$VERSION"'/g' charts/klyshko-operator/Chart.yaml + sed -i 's/tag: latest/tag: ${{ steps.meta.outputs.version }}/g' charts/klyshko-operator/values.yaml + - name: Push Helm Chart + uses: appany/helm-oci-chart-releaser@v0.4.0 + with: + name: klyshko-operator + repository: carbynestack + tag: ${{ env.VERSION }} + path: klyshko-operator/charts/klyshko-operator + registry: ghcr.io + registry_username: ${{ github.actor }} + registry_password: ${{ secrets.GITHUB_TOKEN }} diff --git a/klyshko-operator/charts/klyshko-operator/.helmignore b/klyshko-operator/charts/klyshko-operator/.helmignore new file mode 100644 index 0000000..712ec3d --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/.helmignore @@ -0,0 +1,33 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store + +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ + +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ + +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/klyshko-operator/charts/klyshko-operator/Chart.yaml b/klyshko-operator/charts/klyshko-operator/Chart.yaml new file mode 100644 index 0000000..b5cdf11 --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/Chart.yaml @@ -0,0 +1,12 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v2 +name: klyshko-operator +description: A Helm chart for the Carbyne Stack Klyshko Operator +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/klyshko-operator/charts/klyshko-operator/README.md b/klyshko-operator/charts/klyshko-operator/README.md new file mode 100644 index 0000000..c81d562 --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/README.md @@ -0,0 +1,85 @@ +# Klyshko Operator + +Helm chart for the Klyshko operator. [Klyshko](../../README.md) is the +correlated randomness generation subsystem of Carbyne Stack. + +## TL;DR + +```bash +helm install klyshko-operator +``` + +## Introduction + +This chart deploys the +[Klyshko Operator](https://github.com/carbynestack/klyshko/klyshko-operator) on +a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) +package manager. + +> **Tip**: This chart is used in the `helmfile.d` based deployment specification +> available in the +> [`carbynestack/carbynestack`](https://github.com/carbynestack/carbynestack) +> repository. + +## Prerequisites + +- Kubernetes 1.18+ (may also work on earlier and later versions but has not been + tested) + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +helm install --name my-release klyshko-operator +``` + +Make sure that your current working directory is `/charts`. +The command deploys the Klyshko operator on the Kubernetes cluster in the +default configuration. The [configuration](#configuration) section lists the +parameters that can be configured to customize the deployment. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```bash +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and +deletes the release. + +## Configuration + +The following sections list the (main) configurable parameters of the +`klyshko-operator` chart and their default values. For the full list of +configuration parameters see [values.yaml](values.yaml). + +Specify each parameter using the `--set key=value[,key=value]` argument to +`helm install`. For example, + +```bash +helm install --name my-release --set controller.image.tag= klyshko-operator +``` + +The above command sets the Klyshko operator controller image version to ``. + +Alternatively, a YAML file that specifies the values for the parameters can be +provided while installing the chart. For example, + +```bash +helm install --name my-release -f values.yaml klyshko-operator +``` + +### Controller + +| Parameter | Description | Default | +| ----------------------------- | ------------------------------------------------ | ------------------------------------------ | +| `controller.image.registry` | Image registry used to pull the controller image | `ghcr.io` | +| `controller.image.repository` | Controller image name | `carbynestack/klyshko-operator-controller` | +| `controller.image.tag` | Controller image tag | `latest` | +| `controller.image.pullPolicy` | Controller image pull policy | `IfNotPresent` | +| `controller.imagePullSecrets` | Pull secrets used to fetch the controller image | `[]` | diff --git a/klyshko-operator/charts/klyshko-operator/templates/crds.yaml b/klyshko-operator/charts/klyshko-operator/templates/crds.yaml new file mode 100644 index 0000000..207a715 --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/templates/crds.yaml @@ -0,0 +1,238 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + name: tuplegenerationjobs.klyshko.carbnyestack.io +spec: + group: klyshko.carbnyestack.io + names: + kind: TupleGenerationJob + listKind: TupleGenerationJobList + plural: tuplegenerationjobs + shortNames: + - tgj + - tgjob + singular: tuplegenerationjob + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.type + name: Tuple Type + type: string + - jsonPath: .spec.count + name: Tuple Count + type: string + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TupleGenerationJob is the Schema for the TupleGenerationJob API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TupleGenerationJobSpec defines the desired state of a TupleGenerationJob. + properties: + count: + exclusiveMinimum: true + minimum: 0 + type: integer + generator: + description: GeneratorSpec is a description of a Correlated Randomness Generator. + properties: + image: + description: Container image name. + type: string + imagePullPolicy: + default: IfNotPresent + description: Image pull policy specifies under which circumstances the image is pulled from the registry. + enum: + - IfNotPresent + - Always + - Never + type: string + required: + - image + type: object + id: + type: string + type: + enum: + - BIT_GFP + - BIT_GF2N + - INPUT_MASK_GFP + - INPUT_MASK_GF2N + - INVERSE_TUPLE_GFP + - INVERSE_TUPLE_GF2N + - SQUARE_TUPLE_GFP + - SQUARE_TUPLE_GF2N + - MULTIPLICATION_TRIPLE_GFP + - MULTIPLICATION_TRIPLE_GF2N + type: string + required: + - count + - generator + - id + - type + type: object + status: + description: TupleGenerationJobStatus defines the observed state of a TupleGenerationJob. + properties: + lastStateTransitionTime: + format: date-time + type: string + state: + description: TupleGenerationJobState encodes the state of a TupleGenerationJob. + type: string + required: + - lastStateTransitionTime + - state + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + name: tuplegenerationschedulers.klyshko.carbnyestack.io +spec: + group: klyshko.carbnyestack.io + names: + kind: TupleGenerationScheduler + listKind: TupleGenerationSchedulerList + plural: tuplegenerationschedulers + singular: tuplegenerationscheduler + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: TupleGenerationScheduler is the Schema for the TupleGenerationScheduler API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TupleGenerationSchedulerSpec defines the desired state of a TupleGenerationScheduler. + properties: + concurrency: + default: 1 + minimum: 0 + type: integer + generator: + description: GeneratorSpec is a description of a Correlated Randomness Generator. + properties: + image: + description: Container image name. + type: string + imagePullPolicy: + default: IfNotPresent + description: Image pull policy specifies under which circumstances the image is pulled from the registry. + enum: + - IfNotPresent + - Always + - Never + type: string + required: + - image + type: object + threshold: + exclusiveMinimum: true + minimum: 0 + type: integer + ttlSecondsAfterFinished: + default: 600 + exclusiveMinimum: true + minimum: 0 + type: integer + required: + - generator + - threshold + - ttlSecondsAfterFinished + type: object + status: + description: TupleGenerationSchedulerStatus defines the observed state of a TupleGenerationScheduler. + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.1 + name: tuplegenerationtasks.klyshko.carbnyestack.io +spec: + group: klyshko.carbnyestack.io + names: + kind: TupleGenerationTask + listKind: TupleGenerationTaskList + plural: tuplegenerationtasks + shortNames: + - tgt + - tgtask + singular: tuplegenerationtask + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TupleGenerationTask is the Schema for the TupleGenerationTask API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TupleGenerationTaskSpec defines the desired state of a TupleGenerationTask. + type: object + status: + description: TupleGenerationTaskStatus defines the observed state of a TupleGenerationTask. + properties: + state: + description: TupleGenerationTaskState encodes the state of a TupleGenerationTask. + type: string + required: + - state + type: object + type: object + served: true + storage: true diff --git a/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml b/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml new file mode 100644 index 0000000..2e63257 --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml @@ -0,0 +1,93 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: controller-manager + name: klyshko-controller-manager + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + labels: + control-plane: controller-manager + spec: + {{- with .Values.controller.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + - --zap-log-level=info + command: + - /manager + image: "{{ .Values.controller.image.registry }}/{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + securityContext: + runAsNonRoot: true + serviceAccountName: klyshko-controller-manager + terminationGracePeriodSeconds: 10 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: klyshko-manager-config + namespace: {{ .Release.Namespace }} +data: + controller_manager_config.yaml: | + apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 + kind: ControllerManagerConfig + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: operator.klyshko.carbynestack.io diff --git a/klyshko-operator/charts/klyshko-operator/templates/rbac.yaml b/klyshko-operator/charts/klyshko-operator/templates/rbac.yaml new file mode 100644 index 0000000..e9868b1 --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/templates/rbac.yaml @@ -0,0 +1,233 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: ServiceAccount +metadata: + name: klyshko-controller-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: klyshko-leader-election-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: klyshko-leader-election-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: klyshko-leader-election-role +subjects: + - kind: ServiceAccount + name: klyshko-controller-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: klyshko-manager-role +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs/finalizers + verbs: + - update + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationjobs/status + verbs: + - get + - patch + - update + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers/finalizers + verbs: + - update + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationschedulers/status + verbs: + - get + - patch + - update + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks/finalizers + verbs: + - update + - apiGroups: + - klyshko.carbnyestack.io + resources: + - tuplegenerationtasks/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: klyshko-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: klyshko-manager-role +subjects: + - kind: ServiceAccount + name: klyshko-controller-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: klyshko-metrics-reader +rules: + - nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: klyshko-proxy-role +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: klyshko-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: klyshko-proxy-role +subjects: + - kind: ServiceAccount + name: klyshko-controller-manager + namespace: {{ .Release.Namespace }} diff --git a/klyshko-operator/charts/klyshko-operator/templates/service.yaml b/klyshko-operator/charts/klyshko-operator/templates/service.yaml new file mode 100644 index 0000000..81b3f13 --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/templates/service.yaml @@ -0,0 +1,21 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: klyshko-controller-manager-metrics-service + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/klyshko-operator/charts/klyshko-operator/values.yaml b/klyshko-operator/charts/klyshko-operator/values.yaml new file mode 100644 index 0000000..4198bcd --- /dev/null +++ b/klyshko-operator/charts/klyshko-operator/values.yaml @@ -0,0 +1,17 @@ +# +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# This file defines the default values for all variables used in the Klyshko Operator Helm Chart. + +controller: + image: + registry: ghcr.io + repository: carbynestack/klyshko-operator-controller + pullPolicy: IfNotPresent + tag: latest + imagePullSecrets: [] From 9650963c228eff36d050094540bed61bb37fb26b Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Wed, 21 Dec 2022 11:41:11 +0100 Subject: [PATCH 57/69] Add missing timestamp in job status on creation Signed-off-by: Sven Trieflinger --- .../controllers/tuplegenerationscheduler_controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 8293f10..5210c9f 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -125,7 +125,8 @@ func (r *TupleGenerationSchedulerReconciler) createJob(ctx context.Context, sche Generator: scheduler.Spec.Generator, }, Status: klyshkov1alpha1.TupleGenerationJobStatus{ - State: klyshkov1alpha1.JobPending, + State: klyshkov1alpha1.JobPending, + LastStateTransitionTime: metav1.Now(), }, } err := ctrl.SetControllerReference(scheduler, job, r.Scheme) From a7799edf6f63cc5c6f680600170625af69631dd6 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Wed, 21 Dec 2022 16:29:57 +0100 Subject: [PATCH 58/69] Fix non-working helm chart Signed-off-by: Sven Trieflinger --- .../klyshko-operator/templates/crds.yaml | 27 +++++++++++++++++++ .../klyshko-operator/templates/rbac.yaml | 1 + 2 files changed, 28 insertions(+) diff --git a/klyshko-operator/charts/klyshko-operator/templates/crds.yaml b/klyshko-operator/charts/klyshko-operator/templates/crds.yaml index 207a715..4291886 100644 --- a/klyshko-operator/charts/klyshko-operator/templates/crds.yaml +++ b/klyshko-operator/charts/klyshko-operator/templates/crds.yaml @@ -9,6 +9,7 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null name: tuplegenerationjobs.klyshko.carbnyestack.io spec: group: klyshko.carbnyestack.io @@ -109,12 +110,21 @@ spec: type: object served: true storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null name: tuplegenerationschedulers.klyshko.carbnyestack.io spec: group: klyshko.carbnyestack.io @@ -182,12 +192,21 @@ spec: type: object served: true storage: true + subresources: + status: { } +status: + acceptedNames: + kind: "" + plural: "" + conditions: [ ] + storedVersions: [ ] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.6.1 + creationTimestamp: null name: tuplegenerationtasks.klyshko.carbnyestack.io spec: group: klyshko.carbnyestack.io @@ -236,3 +255,11 @@ spec: type: object served: true storage: true + subresources: + status: { } +status: + acceptedNames: + kind: "" + plural: "" + conditions: [ ] + storedVersions: [ ] diff --git a/klyshko-operator/charts/klyshko-operator/templates/rbac.yaml b/klyshko-operator/charts/klyshko-operator/templates/rbac.yaml index e9868b1..99bf85b 100644 --- a/klyshko-operator/charts/klyshko-operator/templates/rbac.yaml +++ b/klyshko-operator/charts/klyshko-operator/templates/rbac.yaml @@ -65,6 +65,7 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + creationTimestamp: null name: klyshko-manager-role rules: - apiGroups: From fee0129f3a933e3c3eaa9ddc1e673906e0e1416a Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Wed, 21 Dec 2022 17:27:09 +0100 Subject: [PATCH 59/69] Make provisioner image configurable Signed-off-by: Sven Trieflinger --- klyshko-operator/charts/klyshko-operator/README.md | 8 ++++++++ .../charts/klyshko-operator/templates/deployment.yaml | 1 + klyshko-operator/charts/klyshko-operator/values.yaml | 6 ++++++ .../config/default/manager_auth_proxy_patch.yaml | 1 + klyshko-operator/controllers/controller_test.go | 7 ++++--- .../controllers/tuplegenerationtask_controller.go | 9 +++++---- klyshko-operator/main.go | 8 +++++--- 7 files changed, 30 insertions(+), 10 deletions(-) diff --git a/klyshko-operator/charts/klyshko-operator/README.md b/klyshko-operator/charts/klyshko-operator/README.md index c81d562..29f304c 100644 --- a/klyshko-operator/charts/klyshko-operator/README.md +++ b/klyshko-operator/charts/klyshko-operator/README.md @@ -83,3 +83,11 @@ helm install --name my-release -f values.yaml klyshko-operator | `controller.image.tag` | Controller image tag | `latest` | | `controller.image.pullPolicy` | Controller image pull policy | `IfNotPresent` | | `controller.imagePullSecrets` | Pull secrets used to fetch the controller image | `[]` | + +### Provisioner + +| Parameter | Description | Default | +| ------------------------------ | ------------------------------------------------- | ---------------------------------- | +| `provisioner.image.registry` | Image registry used to pull the provisioner image | `ghcr.io` | +| `provisioner.image.repository` | Provisioner image name | `carbynestack/klyshko-provisioner` | +| `provisioner.image.tag` | Provisioner image tag | `latest` | diff --git a/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml b/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml index 2e63257..4977c71 100644 --- a/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml +++ b/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml @@ -42,6 +42,7 @@ spec: - --metrics-bind-address=127.0.0.1:8080 - --leader-elect - --zap-log-level=info + - --provisioner-image={{ printf "%s/%s:%s" .Values.provisioner.image.registry .Values.provisioner.image.repository .Values.provisioner.image.tag }} command: - /manager image: "{{ .Values.controller.image.registry }}/{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" diff --git a/klyshko-operator/charts/klyshko-operator/values.yaml b/klyshko-operator/charts/klyshko-operator/values.yaml index 4198bcd..4c3cd26 100644 --- a/klyshko-operator/charts/klyshko-operator/values.yaml +++ b/klyshko-operator/charts/klyshko-operator/values.yaml @@ -15,3 +15,9 @@ controller: pullPolicy: IfNotPresent tag: latest imagePullSecrets: [] + +provisioner: + image: + registry: ghcr.io + repository: carbynestack/klyshko-provisioner + tag: latest diff --git a/klyshko-operator/config/default/manager_auth_proxy_patch.yaml b/klyshko-operator/config/default/manager_auth_proxy_patch.yaml index 077701e..4f34c7f 100644 --- a/klyshko-operator/config/default/manager_auth_proxy_patch.yaml +++ b/klyshko-operator/config/default/manager_auth_proxy_patch.yaml @@ -26,3 +26,4 @@ spec: - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" - "--zap-log-level=info" + - "--provisioner-image=carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT" diff --git a/klyshko-operator/controllers/controller_test.go b/klyshko-operator/controllers/controller_test.go index 41c233f..0e354e3 100644 --- a/klyshko-operator/controllers/controller_test.go +++ b/klyshko-operator/controllers/controller_test.go @@ -121,9 +121,10 @@ func (vcp *vcp) setupControllers(ctx context.Context, vcpID int, etcdClient *cli NewTupleGenerationJobReconciler( k8sManager.GetClient(), k8sManager.GetScheme(), etcdClient, castorClient), &TupleGenerationTaskReconciler{ // TODO Replace with constructors - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - EtcdClient: etcdClient, + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + EtcdClient: etcdClient, + ProvisionerImage: "carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT", }, } if vcpID == 0 { diff --git a/klyshko-operator/controllers/tuplegenerationtask_controller.go b/klyshko-operator/controllers/tuplegenerationtask_controller.go index db998f9..f50ad0d 100644 --- a/klyshko-operator/controllers/tuplegenerationtask_controller.go +++ b/klyshko-operator/controllers/tuplegenerationtask_controller.go @@ -32,8 +32,9 @@ import ( // TupleGenerationTaskReconciler reconciles a TupleGenerationTask object. type TupleGenerationTaskReconciler struct { client.Client - Scheme *runtime.Scheme - EtcdClient *clientv3.Client + Scheme *runtime.Scheme + EtcdClient *clientv3.Client + ProvisionerImage string } //+kubebuilder:rbac:groups=klyshko.carbnyestack.io,resources=tuplegenerationtasks,verbs=get;list;watch;create;update;patch;delete @@ -331,8 +332,8 @@ func (r *TupleGenerationTaskReconciler) createProvisionerPod(ctx context.Context }, Spec: v1.PodSpec{ Containers: []v1.Container{{ - Name: "generator", - Image: "carbynestack/klyshko-provisioner:1.0.0-SNAPSHOT", // TODO Read from config + Name: "provisioner", + Image: r.ProvisionerImage, Env: []v1.EnvVar{ { Name: "KII_JOB_ID", diff --git a/klyshko-operator/main.go b/klyshko-operator/main.go index ccb1737..c807a5f 100644 --- a/klyshko-operator/main.go +++ b/klyshko-operator/main.go @@ -48,6 +48,7 @@ var ( etcdEndpoint = flag.String("etcd-endpoint", "172.18.1.129:2379", "The address of the etcd service used for cross VCP coordination.") etcdDialTimeout = flag.Int("etcd-dial-timeout", 5, "The timeout (in seconds) for failing to establish a connection to the etcd service.") castorURL = flag.String("castor-url", "http://cs-castor.default.svc.cluster.local:10100", "The base url of the castor service used to upload generated tuples.") + provisionerImage = flag.String("provisioner-image", "ghcr.io/carbynestack/klyshko-provisioner:latest", "The name of the provisioner image.") ) func main() { @@ -96,9 +97,10 @@ func main() { } if err = (&controllers.TupleGenerationTaskReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EtcdClient: etcdClient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + EtcdClient: etcdClient, + ProvisionerImage: *provisionerImage, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TupleGenerationTask") os.Exit(1) From 8eee93ea504670ad6d7c1b87bf0ad0aa4b49dfef Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 27 Jan 2023 09:21:08 +0100 Subject: [PATCH 60/69] Update README with deploy instructions for helm Signed-off-by: Sven Trieflinger --- README.md | 20 +++++++++++++++++--- klyshko-operator/Makefile | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 320257f..df19ea4 100644 --- a/README.md +++ b/README.md @@ -63,13 +63,27 @@ To deploy Klyshko to your VC you have to perform the following steps: ### Install the operator -You can use the `make` tool to deploy the operator using +The Klyshko operator can be deployed either by building from source or by using +`helm`. Both variants are described below. Remember to perform the respective +steps on all VCPs of your VC. + +#### From Source + +You can use the `make` tool to build and deploy the operator using ```shell -make deploy IMG="carbynestack/klyshko-operator:v0.0.1" +cd klyshko-operator +make deploy IMG="carbynestack/klyshko-operator:v0.1.0" ``` -Remember to do this on all VCPs of your VC. +#### Using Helm + +You can deploy the Klyshko operator using `helm` as follows: + +```shell +HELM_EXPERIMENTAL_OCI=1 helm install klyshko-operator oci://ghcr.io/carbynestack/klyshko-operator \ + --version 0.1.0 +``` ### Provide the Configuration diff --git a/klyshko-operator/Makefile b/klyshko-operator/Makefile index 8430d35..1d06998 100644 --- a/klyshko-operator/Makefile +++ b/klyshko-operator/Makefile @@ -5,7 +5,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.0.1 +VERSION ?= 0.1.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") From 73cb3a2fdcebfa7ac8cd7a8090cc7a6a82b31a03 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 27 Jan 2023 11:59:54 +0100 Subject: [PATCH 61/69] Push image pull secrets to top-level configuration Signed-off-by: Sven Trieflinger --- klyshko-operator/charts/klyshko-operator/README.md | 7 ++++++- .../charts/klyshko-operator/templates/deployment.yaml | 2 +- klyshko-operator/charts/klyshko-operator/values.yaml | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/klyshko-operator/charts/klyshko-operator/README.md b/klyshko-operator/charts/klyshko-operator/README.md index 29f304c..253c7ca 100644 --- a/klyshko-operator/charts/klyshko-operator/README.md +++ b/klyshko-operator/charts/klyshko-operator/README.md @@ -74,6 +74,12 @@ provided while installing the chart. For example, helm install --name my-release -f values.yaml klyshko-operator ``` +### General + +| Parameter | Description | Default | +| ------------------ | --------------------------------------------- | ------- | +| `imagePullSecrets` | Pull secrets used to fetch the Klyshko images | `[]` | + ### Controller | Parameter | Description | Default | @@ -82,7 +88,6 @@ helm install --name my-release -f values.yaml klyshko-operator | `controller.image.repository` | Controller image name | `carbynestack/klyshko-operator-controller` | | `controller.image.tag` | Controller image tag | `latest` | | `controller.image.pullPolicy` | Controller image pull policy | `IfNotPresent` | -| `controller.imagePullSecrets` | Pull secrets used to fetch the controller image | `[]` | ### Provisioner diff --git a/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml b/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml index 4977c71..7225169 100644 --- a/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml +++ b/klyshko-operator/charts/klyshko-operator/templates/deployment.yaml @@ -21,7 +21,7 @@ spec: labels: control-plane: controller-manager spec: - {{- with .Values.controller.imagePullSecrets }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/klyshko-operator/charts/klyshko-operator/values.yaml b/klyshko-operator/charts/klyshko-operator/values.yaml index 4c3cd26..ac42b8f 100644 --- a/klyshko-operator/charts/klyshko-operator/values.yaml +++ b/klyshko-operator/charts/klyshko-operator/values.yaml @@ -8,13 +8,14 @@ # This file defines the default values for all variables used in the Klyshko Operator Helm Chart. +imagePullSecrets: [] + controller: image: registry: ghcr.io repository: carbynestack/klyshko-operator-controller pullPolicy: IfNotPresent tag: latest - imagePullSecrets: [] provisioner: image: From d26cf887b4e8f2ce3793ded8911b9bb5e1348ba6 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 27 Jan 2023 15:46:47 +0100 Subject: [PATCH 62/69] Update pre-commit configuration with new checks and bump versions Signed-off-by: Sven Trieflinger --- .pre-commit-config.yaml | 13 ++++++---- README.md | 2 +- .../github.com/evanphx/json-patch/LICENSE | 24 +++++++++---------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3523ba..5d8c5e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,11 +6,16 @@ # repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.3.0 hooks: + - id: check-byte-order-marker + - id: check-yaml + exclude: ^klyshko-operator/charts/klyshko-operator/templates/.*|klyshko-operator/config/manager/manager.yaml$ - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending - repo: https://github.com/tekwizely/pre-commit-golang - rev: v1.0.0-beta.5 + rev: v1.0.0-rc.1 hooks: - id: go-fmt args: [-w] # Update source files directly @@ -19,7 +24,7 @@ repos: hooks: - id: shellcheck - repo: https://github.com/executablebooks/mdformat - rev: 0.7.14 + rev: 0.7.16 hooks: - id: mdformat args: @@ -29,7 +34,7 @@ repos: - mdformat-gfm exclude: ^3RD-PARTY-LICENSES/.*$ - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.31.1 + rev: v0.33.0 hooks: - id: markdownlint args: diff --git a/README.md b/README.md index df19ea4..fbf1e63 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ make deploy IMG="carbynestack/klyshko-operator:v0.1.0" You can deploy the Klyshko operator using `helm` as follows: ```shell -HELM_EXPERIMENTAL_OCI=1 helm install klyshko-operator oci://ghcr.io/carbynestack/klyshko-operator \ +HELM_EXPERIMENTAL_OCI=1 helm install klyshko-operator oci://ghcr.io/carbynestack/klyshko-operator \ --version 0.1.0 ``` diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE b/klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE index df76d7d..b3e763e 100644 --- a/klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/evanphx/json-patch/LICENSE @@ -1,7 +1,7 @@ Copyright (c) 2014, Evan Phoenix All rights reserved. -Redistribution and use in source and binary forms, with or without +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this @@ -9,17 +9,17 @@ modification, are permitted provided that the following conditions are met: * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -* Neither the name of the Evan Phoenix nor the names of its contributors - may be used to endorse or promote products derived from this software +* Neither the name of the Evan Phoenix nor the names of its contributors + may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 16c14871f8caa925fddf406a964ee3089e98ddf7 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 27 Jan 2023 16:23:01 +0100 Subject: [PATCH 63/69] Address codacy findinds Signed-off-by: Sven Trieflinger --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fbf1e63..4ce8bdd 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ make deploy IMG="carbynestack/klyshko-operator:v0.1.0" You can deploy the Klyshko operator using `helm` as follows: ```shell -HELM_EXPERIMENTAL_OCI=1 helm install klyshko-operator oci://ghcr.io/carbynestack/klyshko-operator \ +HELM_EXPERIMENTAL_OCI=1 helm install klyshko-operator \ + oci://ghcr.io/carbynestack/klyshko-operator \ --version 0.1.0 ``` From d13c2ae6f3e7e3f2942a3adb60c4375d31453f78 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 27 Jan 2023 16:30:20 +0100 Subject: [PATCH 64/69] Add codecov badge Signed-off-by: Sven Trieflinger --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4ce8bdd..f503eea 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![stability-wip](https://img.shields.io/badge/stability-wip-lightgrey.svg)](https://github.com/mkenney/software-guides/blob/master/STABILITY-BADGES.md#work-in-progress) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/3a07fd83b67647138b8ea660d16cdc35)](https://www.codacy.com/gh/carbynestack/klyshko/dashboard?utm_source=github.com&utm_medium=referral&utm_content=carbynestack/klyshko&utm_campaign=Badge_Grade) +[![codecov](https://codecov.io/gh/carbynestack/klyshko/branch/master/graph/badge.svg?token=6hRb7xRW6C)](https://codecov.io/gh/carbynestack/klyshko) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) From 7bd3d386bb7daec4fe8a2b3a282db2541d0ffecf Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Tue, 7 Mar 2023 15:43:12 +0100 Subject: [PATCH 65/69] Address review comments Signed-off-by: Sven Trieflinger --- .github/workflows/auto-labeler.yaml | 23 ++++++++++++ .github/workflows/stale-issue-cleanup.yaml | 22 ++++++++++++ README.md | 35 +++++++++++++++---- klyshko-operator/PROJECT | 9 ----- .../api/v1alpha1/groupversion_info.go | 1 + .../tuplegenerationscheduler_types.go | 1 + .../api/v1alpha1/zz_generated.deepcopy.go | 1 + klyshko-operator/castor/castor_client_test.go | 8 ++--- .../charts/klyshko-operator/README.md | 2 +- .../klyshko-operator/templates/crds.yaml | 3 ++ ...nyestack.io_tuplegenerationschedulers.yaml | 3 ++ .../tuplegenerationjob_controller.go | 26 ++++++++------ .../tuplegenerationscheduler_controller.go | 6 ++++ 13 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/auto-labeler.yaml create mode 100644 .github/workflows/stale-issue-cleanup.yaml diff --git a/.github/workflows/auto-labeler.yaml b/.github/workflows/auto-labeler.yaml new file mode 100644 index 0000000..3984b3c --- /dev/null +++ b/.github/workflows/auto-labeler.yaml @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: 'Auto Labeler' + +on: + + # Allow workflow to be triggered manually + workflow_dispatch: + + issues: + types: + - reopened + - opened + +jobs: + + # re-usable workflow @see https://docs.github.com/en/actions/using-workflows/reusing-workflows + auto-labeling: + uses: carbynestack/.github/.github/workflows/auto-labeler.yml@master diff --git a/.github/workflows/stale-issue-cleanup.yaml b/.github/workflows/stale-issue-cleanup.yaml new file mode 100644 index 0000000..02fcb51 --- /dev/null +++ b/.github/workflows/stale-issue-cleanup.yaml @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: 'Stale Issue Cleanup' + +on: + + # Allow workflow to be triggered manually + workflow_dispatch: + + # Trigger at 1:00 AM each day + schedule: + - cron: '0 1 * * *' + +jobs: + + # re-usable workflow @see https://docs.github.com/en/actions/using-workflows/reusing-workflows + stale-issue-cleanup: + uses: carbynestack/.github/.github/workflows/stale-actions.yml@master diff --git a/README.md b/README.md index f503eea..7974a3e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Carbyne Stack Klyshko Correlated Randomness Generation -[![stability-wip](https://img.shields.io/badge/stability-wip-lightgrey.svg)](https://github.com/mkenney/software-guides/blob/master/STABILITY-BADGES.md#work-in-progress) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/3a07fd83b67647138b8ea660d16cdc35)](https://www.codacy.com/gh/carbynestack/klyshko/dashboard?utm_source=github.com&utm_medium=referral&utm_content=carbynestack/klyshko&utm_campaign=Badge_Grade) [![codecov](https://codecov.io/gh/carbynestack/klyshko/branch/master/graph/badge.svg?token=6hRb7xRW6C)](https://codecov.io/gh/carbynestack/klyshko) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) @@ -46,7 +45,7 @@ Klyshko consists of three main components: generated. - A *Task* (kind: `TupleGenerationTask`) represents a local or remote execution of a CRG. A task exposes the state of the invocation on a single - VCP. On the job level task states are aggregated into a job state. Remote + VCP. On the job level, task states are aggregated into a job state. Remote tasks are proxied locally to make their state available to the job controller. The task controller makes use of the [Klyshko Integration Interface (KII)](#klyshko-integration-interface-kii) to @@ -62,6 +61,26 @@ to orchestrate actions across VCPs. To deploy Klyshko to your VC you have to perform the following steps: +### Provide VCP Configuration + +> **NOTE**: This is a workaround until the Carbyne Stack Operator (see +> [CSEP-0053](https://github.com/carbynestack/carbynestack/pull/54)) is +> available. + +Klyshko needs to know the overall number of VCPs in the VC and the zero-based +index of the local VCP. This done by creating a configuration map with the +following content: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: +name: cs-vcp-config +data: +playerCount: <> +playerId: <> +``` + ### Install the operator The Klyshko operator can be deployed either by building from source or by using @@ -236,7 +255,12 @@ These are made available to CRGs by the Klyshko runtime as files in folder The `deploy.sh` scripts in the `hack` folders (top-level and within modules) can be used to (re-)deploy Klyshko to a 2-party Carbyne Stack VC setup as described in the [tutorials](https://carbynestack.io/getting-started) on the Carbyne Stack -website. +website. To trigger (re-)deployment, the top-level script must be called from +the project root folder using + +```shell +./hack/deploy.sh +``` ### Logging @@ -274,9 +298,8 @@ We use the following logging level convention in the Klyshko code basis. ## License -Carbyne Stack *Klyshko Correlated Randomness Generation Subsystem* is -open-sourced under the Apache License 2.0. See the [LICENSE](LICENSE) file for -details. +Carbyne Stack *Klyshko Correlated Randomness Generation Service* is open-sourced +under the Apache License 2.0. See the [LICENSE](LICENSE) file for details. ### 3rd Party Licenses diff --git a/klyshko-operator/PROJECT b/klyshko-operator/PROJECT index 5d1ec87..c262785 100644 --- a/klyshko-operator/PROJECT +++ b/klyshko-operator/PROJECT @@ -7,15 +7,6 @@ plugins: projectName: klyshko repo: github.com/carbynestack/klyshko resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: carbnyestack.io - group: klyshko - kind: Job - path: github.com/carbynestack/klyshko/api/v1alpha1 - version: v1alpha1 - api: crdVersion: v1 namespaced: true diff --git a/klyshko-operator/api/v1alpha1/groupversion_info.go b/klyshko-operator/api/v1alpha1/groupversion_info.go index 522c1c7..487de63 100644 --- a/klyshko-operator/api/v1alpha1/groupversion_info.go +++ b/klyshko-operator/api/v1alpha1/groupversion_info.go @@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0 //+kubebuilder:object:generate=true //+groupName=klyshko.carbnyestack.io + package v1alpha1 import ( diff --git a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go index 61cb34d..55f7ab7 100644 --- a/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go +++ b/klyshko-operator/api/v1alpha1/tuplegenerationscheduler_types.go @@ -37,6 +37,7 @@ type TupleGenerationSchedulerStatus struct { } //+kubebuilder:object:root=true +//+kubebuilder:resource:shortName=tgs;tgscheduler //+kubebuilder:subresource:status // TupleGenerationScheduler is the Schema for the TupleGenerationScheduler API. diff --git a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go index 13138fb..2088891 100644 --- a/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/klyshko-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/klyshko-operator/castor/castor_client_test.go b/klyshko-operator/castor/castor_client_test.go index 9f17a2d..c1df62b 100644 --- a/klyshko-operator/castor/castor_client_test.go +++ b/klyshko-operator/castor/castor_client_test.go @@ -27,7 +27,7 @@ var _ = Describe("Fetching telemetry", func() { httpmock.DeactivateAndReset() }) - When("when Castor services responds with expected JSON data", func() { + When("when Castor service responds with expected JSON data", func() { expectedTelemetry := Telemetry{TupleMetrics: []TupleMetrics{ { @@ -55,7 +55,7 @@ var _ = Describe("Fetching telemetry", func() { }) }) - When("when Castor services responds with non-success status code", func() { + When("when Castor service responds with non-success status code", func() { BeforeEach(func() { httpmock.Activate() httpmock.RegisterResponder( @@ -89,7 +89,7 @@ var _ = Describe("Activating a tuple chunk", func() { httpmock.DeactivateAndReset() }) - When("when Castor services responds with success status code", func() { + When("when Castor service responds with success status code", func() { BeforeEach(func() { httpmock.Activate() httpmock.RegisterResponder( @@ -106,7 +106,7 @@ var _ = Describe("Activating a tuple chunk", func() { }) }) - When("when Castor services responds with bon-success status code", func() { + When("when Castor service responds with non-success status code", func() { BeforeEach(func() { httpmock.Activate() httpmock.RegisterResponder( diff --git a/klyshko-operator/charts/klyshko-operator/README.md b/klyshko-operator/charts/klyshko-operator/README.md index 253c7ca..fd7dfb5 100644 --- a/klyshko-operator/charts/klyshko-operator/README.md +++ b/klyshko-operator/charts/klyshko-operator/README.md @@ -54,7 +54,7 @@ deletes the release. ## Configuration -The following sections list the (main) configurable parameters of the +The following sections list the (main) configuration parameters of the `klyshko-operator` chart and their default values. For the full list of configuration parameters see [values.yaml](values.yaml). diff --git a/klyshko-operator/charts/klyshko-operator/templates/crds.yaml b/klyshko-operator/charts/klyshko-operator/templates/crds.yaml index 4291886..ea16d97 100644 --- a/klyshko-operator/charts/klyshko-operator/templates/crds.yaml +++ b/klyshko-operator/charts/klyshko-operator/templates/crds.yaml @@ -132,6 +132,9 @@ spec: kind: TupleGenerationScheduler listKind: TupleGenerationSchedulerList plural: tuplegenerationschedulers + shortNames: + - tgs + - tgscheduler singular: tuplegenerationscheduler scope: Namespaced versions: diff --git a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml index 8effedb..28bfe97 100644 --- a/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml +++ b/klyshko-operator/config/crd/bases/klyshko.carbnyestack.io_tuplegenerationschedulers.yaml @@ -13,6 +13,9 @@ spec: kind: TupleGenerationScheduler listKind: TupleGenerationSchedulerList plural: tuplegenerationschedulers + shortNames: + - tgs + - tgscheduler singular: tuplegenerationscheduler scope: Namespaced versions: diff --git a/klyshko-operator/controllers/tuplegenerationjob_controller.go b/klyshko-operator/controllers/tuplegenerationjob_controller.go index b7b9be6..a452986 100644 --- a/klyshko-operator/controllers/tuplegenerationjob_controller.go +++ b/klyshko-operator/controllers/tuplegenerationjob_controller.go @@ -71,17 +71,26 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R logger := r.Logger.WithValues("Job.Key", jobKey) logger.V(logging.DEBUG).Info("Reconciling tuple generation jobs") + // Get local player index to be used throughout the reconciliation loop + playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) + if err != nil { + return ctrl.Result{RequeueAfter: 60 * time.Second}, + fmt.Errorf("can't read playerId from VCP configuration for job %v: %w", req.Name, err) + } + // Cleanup if job has been deleted job := &klyshkov1alpha1.TupleGenerationJob{} - err := r.Get(ctx, req.NamespacedName, job) + err = r.Get(ctx, req.NamespacedName, job) if err != nil { if apierrors.IsNotFound(err) { - // Job resource not available -> has been deleted - _, err := r.EtcdClient.Delete(ctx, jobKey.ToEtcdKey()) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to delete roster for job %v: %w", req.Name, err) + // Job resource not available -> has been deleted, delete etcd entry, iff we are the master + if playerID == 0 { + _, err := r.EtcdClient.Delete(ctx, jobKey.ToEtcdKey()) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to delete roster for job %v: %w", req.Name, err) + } + logger.V(logging.DEBUG).Info("Roster deleted") } - logger.V(logging.DEBUG).Info("Roster deleted") return ctrl.Result{}, nil } // Error reading the object - requeue the request. @@ -95,7 +104,6 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, fmt.Errorf("failed to read resource for roster with key %v for task %v: %w", jobKey, req.Name, err) } if resp.Count == 0 { - playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) if playerID != 0 { logger.V(logging.DEBUG).Info("Roster not available, retrying later") return ctrl.Result{}, nil @@ -114,10 +122,6 @@ func (r *TupleGenerationJobReconciler) Reconcile(ctx context.Context, req ctrl.R } // Create local task if not existing - playerID, err := localPlayerID(ctx, &r.Client, req.Namespace) - if err != nil { - return ctrl.Result{RequeueAfter: 60 * time.Second}, fmt.Errorf("can't read playerId from VCP configuration for job %v: %w", req.Name, err) - } task := &klyshkov1alpha1.TupleGenerationTask{} err = r.Get(ctx, types.NamespacedName{ Namespace: job.Namespace, diff --git a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go index 5210c9f..9e8a9e2 100644 --- a/klyshko-operator/controllers/tuplegenerationscheduler_controller.go +++ b/klyshko-operator/controllers/tuplegenerationscheduler_controller.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "math/rand" "time" "k8s.io/apimachinery/pkg/runtime" @@ -157,6 +158,11 @@ func (r *TupleGenerationSchedulerReconciler) cleanupFinishedJobs(ctx context.Con return err } logger.V(logging.DEBUG).Info("Deleting finished jobs", "jobs", finishedJobs) + // Shuffling jobs to ensure that finished jobs do not accumulate while we try to delete + // the same finished job over and over again + rand.Shuffle(len(finishedJobs), func(i, j int) { + finishedJobs[i], finishedJobs[j] = finishedJobs[j], finishedJobs[i] + }) for _, j := range finishedJobs { err := r.Delete(ctx, &j) if err != nil { From e63bcc8d30e2ae01438a0ccb5f4793031fd6f643 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 10 Mar 2023 15:15:32 +0100 Subject: [PATCH 66/69] chore: add commitlint pre-commit hook Signed-off-by: Sven Trieflinger --- .pre-commit-config.yaml | 8 ++++++++ commitlint.config.js | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 commitlint.config.js diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d8c5e6..f2ba26e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,14 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: mixed-line-ending + - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook + rev: v9.4.0 + hooks: + - id: commitlint + stages: + - commit-msg + additional_dependencies: + - '@commitlint/config-conventional' - repo: https://github.com/tekwizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000..7773fa1 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2022 - for information on the respective copyright owner + * see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. + * + * SPDX-License-Identifier: Apache-2.0 + */ +module.exports = { + extends: [ + "@commitlint/config-conventional" + ], + rules: { + 'scope-empty': [0, 'never'], + "scope-enum": [ + 2, + "always", + [ + "mp-spdz", + "operator", + "provisioner" + ] + ] + } +} From 7ea2274ba6a5019f59ef6b2c369ee09b7f2f8b9a Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Fri, 10 Mar 2023 18:15:52 +0100 Subject: [PATCH 67/69] chore: introduce release process Signed-off-by: Sven Trieflinger --- .github/workflows/auto-labeler.yaml | 23 -- .github/workflows/mp-spdz.publish.yml | 64 ++++++ .github/workflows/mp-spdz.yaml | 42 ---- .github/workflows/operator-chart.publish.yml | 38 ++++ .../workflows/operator.build-and-test.yaml | 58 +++++ .github/workflows/operator.publish.yml | 64 ++++++ .github/workflows/operator.yaml | 78 ------- .github/workflows/provisioner.publish.yml | 64 ++++++ .github/workflows/provisioner.yaml | 41 ---- .github/workflows/release.yml | 20 ++ .github/workflows/stale-issue-cleanup.yaml | 22 -- .release-please-manifest.json | 6 + README.md | 1 + commitlint.config.js | 3 +- klyshko-mp-spdz/3RD-PARTY-LICENSES/sbom.json | 2 + klyshko-mp-spdz/Dockerfile.fake-offline | 13 +- klyshko-mp-spdz/version.txt | 1 + klyshko-operator/Dockerfile | 21 +- .../github.com/carbynestack/cli/LICENSE | 202 ++++++++++++++++++ .../github.com/carbynestack/cli/NOTICE.md | 31 +++ .../3RD-PARTY-LICENSES/sbom.json | 6 + klyshko-provisioner/Dockerfile | 13 +- klyshko-provisioner/version.txt | 1 + release-please-config.json | 24 +++ 24 files changed, 626 insertions(+), 212 deletions(-) delete mode 100644 .github/workflows/auto-labeler.yaml create mode 100644 .github/workflows/mp-spdz.publish.yml delete mode 100644 .github/workflows/mp-spdz.yaml create mode 100644 .github/workflows/operator-chart.publish.yml create mode 100644 .github/workflows/operator.build-and-test.yaml create mode 100644 .github/workflows/operator.publish.yml delete mode 100644 .github/workflows/operator.yaml create mode 100644 .github/workflows/provisioner.publish.yml delete mode 100644 .github/workflows/provisioner.yaml create mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/stale-issue-cleanup.yaml create mode 100644 .release-please-manifest.json create mode 100644 klyshko-mp-spdz/3RD-PARTY-LICENSES/sbom.json create mode 100644 klyshko-mp-spdz/version.txt create mode 100644 klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/LICENSE create mode 100644 klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/NOTICE.md create mode 100644 klyshko-provisioner/3RD-PARTY-LICENSES/sbom.json create mode 100644 klyshko-provisioner/version.txt create mode 100644 release-please-config.json diff --git a/.github/workflows/auto-labeler.yaml b/.github/workflows/auto-labeler.yaml deleted file mode 100644 index 3984b3c..0000000 --- a/.github/workflows/auto-labeler.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. -# -# SPDX-License-Identifier: Apache-2.0 -# -name: 'Auto Labeler' - -on: - - # Allow workflow to be triggered manually - workflow_dispatch: - - issues: - types: - - reopened - - opened - -jobs: - - # re-usable workflow @see https://docs.github.com/en/actions/using-workflows/reusing-workflows - auto-labeling: - uses: carbynestack/.github/.github/workflows/auto-labeler.yml@master diff --git a/.github/workflows/mp-spdz.publish.yml b/.github/workflows/mp-spdz.publish.yml new file mode 100644 index 0000000..aa55a33 --- /dev/null +++ b/.github/workflows/mp-spdz.publish.yml @@ -0,0 +1,64 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Publish MP-SPDZ +on: + push: + tags: + - "mp-spdz-v[0-9]+.[0-9]+.[0-9]+" +defaults: + run: + working-directory: klyshko-mp-spdz +env: + REGISTRY: ghcr.io + IMAGE_NAME: carbynestack/klyshko-mp-spdz + WORKING_DIRECTORY: klyshko-mp-spdz +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Retrieve license obligation resources + run: | + cd 3RD-PARTY-LICENSES + find . -maxdepth 1 -type d -not -path . | zip -r@ 3rd-party-copyrights + find . -iname origin.src | \ + awk '{ \ + split($0,b,"/"); \ + system("xargs < " $0 " curl --create-dirs -Lo ./sources/" b[2] ".zip " $2)}' && \ + find -regex './sources$' | awk '{system("zip -jr ./3rd-party-sources.zip " $0)}' + mkdir -p ../license-obligations && mv `find . -regex "^./3rd-party-.*.zip$"` ../license-obligations/ + - name: Update Release with license obligations resources + uses: ncipollo/release-action@v1 + with: + allowUpdates: true + artifacts: ${{ env.WORKING_DIRECTORY }}/license-obligations/* + artifactErrorsFailBuild: true + makeLatest: true + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=match,pattern=mp-spdz-v(\d+.\d+.\d+),group=1 + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: ${{ env.WORKING_DIRECTORY }} + file: ${{ env.WORKING_DIRECTORY }}/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/mp-spdz.yaml b/.github/workflows/mp-spdz.yaml deleted file mode 100644 index 39b739b..0000000 --- a/.github/workflows/mp-spdz.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2022 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. -# -# SPDX-License-Identifier: Apache-2.0 -# -name: Build and publish MP-SPDZ Integration -on: - workflow_dispatch: - push: -defaults: - run: - working-directory: klyshko-mp-spdz -env: - REGISTRY: ghcr.io - IMAGE_NAME: carbynestack/klyshko-mp-spdz - WORKING_DIRECTORY: klyshko-mp-spdz -jobs: - publish: - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Log in to container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - context: ${{ env.WORKING_DIRECTORY }} - file: ${{ env.WORKING_DIRECTORY }}/Dockerfile.fake-offline - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/operator-chart.publish.yml b/.github/workflows/operator-chart.publish.yml new file mode 100644 index 0000000..f1628ff --- /dev/null +++ b/.github/workflows/operator-chart.publish.yml @@ -0,0 +1,38 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Publish Operator Chart +on: + push: + tags: + - "operator-chart-v[0-9]+.[0-9]+.[0-9]+" +env: + REGISTRY: ghcr.io + WORKING_DIRECTORY: klyshko-operator +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Get tag + run: echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push Helm Chart + uses: appany/helm-oci-chart-releaser@v0.4.0 + with: + name: klyshko-operator + repository: carbynestack + tag: ${{ env.TAG }} + path: ${{ env.WORKING_DIRECTORY }}/charts/klyshko-operator + registry: ${{ env.REGISTRY }} + registry_username: ${{ github.actor }} + registry_password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/operator.build-and-test.yaml b/.github/workflows/operator.build-and-test.yaml new file mode 100644 index 0000000..b79afce --- /dev/null +++ b/.github/workflows/operator.build-and-test.yaml @@ -0,0 +1,58 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Build and test Operator +on: + workflow_dispatch: + push: + branches: + - 'master' + pull_request: + branches: + - 'master' +jobs: + changes: + runs-on: ubuntu-22.04 + permissions: + pull-requests: read + outputs: + operator: ${{ steps.filter.outputs.operator }} + steps: + - name: Check whether Operator codebase (w/o chart) is affected + uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + operator: + - 'klyshko-operator/!(charts/**)/**' + test: + runs-on: ubuntu-22.04 + needs: changes + if: ${{ needs.changes.outputs.operator == 'true' }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.16' + - name: Setup Build Cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Build and Test + run: | + make test + - name: Publishing Coverage + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + name: codecov diff --git a/.github/workflows/operator.publish.yml b/.github/workflows/operator.publish.yml new file mode 100644 index 0000000..731e729 --- /dev/null +++ b/.github/workflows/operator.publish.yml @@ -0,0 +1,64 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Publish Operator +on: + push: + tags: + - "operator-v[0-9]+.[0-9]+.[0-9]+" +defaults: + run: + working-directory: klyshko-operator +env: + REGISTRY: ghcr.io + IMAGE_NAME: carbynestack/klyshko-operator + WORKING_DIRECTORY: klyshko-operator +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Retrieve license obligation resources + run: | + cd 3RD-PARTY-LICENSES + find . -maxdepth 1 -type d -not -path . | zip -r@ 3rd-party-copyrights + find . -iname origin.src | \ + awk '{ \ + split($0,b,"/"); \ + system("xargs < " $0 " curl --create-dirs -Lo ./sources/" b[2] ".zip " $2)}' && \ + find -regex './sources$' | awk '{system("zip -jr ./3rd-party-sources.zip " $0)}' + mkdir -p ../license-obligations && mv `find . -regex "^./3rd-party-.*.zip$"` ../license-obligations/ + - name: Update Release with license obligations resources + uses: ncipollo/release-action@v1 + with: + allowUpdates: true + artifacts: ${{ env.WORKING_DIRECTORY }}/license-obligations/* + artifactErrorsFailBuild: true + makeLatest: true + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=match,pattern=operator-v(\d+.\d+.\d+),group=1 + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: ${{ env.WORKING_DIRECTORY }} + file: ${{ env.WORKING_DIRECTORY }}/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/operator.yaml b/.github/workflows/operator.yaml deleted file mode 100644 index a58952d..0000000 --- a/.github/workflows/operator.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2022 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. -# -# SPDX-License-Identifier: Apache-2.0 -# -name: Build and publish Operator -on: - workflow_dispatch: - push: -defaults: - run: - working-directory: klyshko-operator -env: - REGISTRY: ghcr.io - IMAGE_NAME: carbynestack/klyshko-operator-controller - WORKING_DIRECTORY: klyshko-operator -jobs: - publish: - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: '1.16' - - name: Setup Build Cache - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Build and Test - run: | - make test - - name: Publishing Coverage - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - name: codecov - - name: Log in to container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - context: ${{ env.WORKING_DIRECTORY }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - name: Configure Helm Chart - run: | - VERSION=$(sed -n 's/version: \(.*\)/\1/p' charts/klyshko-operator/Chart.yaml)-${{ steps.meta.outputs.version }} - echo "VERSION=$VERSION" >> $GITHUB_ENV - sed -i 's/version: \(.*\)/version: '"$VERSION"'/g' charts/klyshko-operator/Chart.yaml - sed -i 's/tag: latest/tag: ${{ steps.meta.outputs.version }}/g' charts/klyshko-operator/values.yaml - - name: Push Helm Chart - uses: appany/helm-oci-chart-releaser@v0.4.0 - with: - name: klyshko-operator - repository: carbynestack - tag: ${{ env.VERSION }} - path: klyshko-operator/charts/klyshko-operator - registry: ghcr.io - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/provisioner.publish.yml b/.github/workflows/provisioner.publish.yml new file mode 100644 index 0000000..80f7739 --- /dev/null +++ b/.github/workflows/provisioner.publish.yml @@ -0,0 +1,64 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Publish Provisioner +on: + push: + tags: + - "provisioner-v[0-9]+.[0-9]+.[0-9]+" +defaults: + run: + working-directory: klyshko-provisioner +env: + REGISTRY: ghcr.io + IMAGE_NAME: carbynestack/klyshko-provisioner + WORKING_DIRECTORY: klyshko-provisioner +jobs: + publish: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Retrieve license obligation resources + run: | + cd 3RD-PARTY-LICENSES + find . -maxdepth 1 -type d -not -path . | zip -r@ 3rd-party-copyrights + find . -iname origin.src | \ + awk '{ \ + split($0,b,"/"); \ + system("xargs < " $0 " curl --create-dirs -Lo ./sources/" b[2] ".zip " $2)}' && \ + find -regex './sources$' | awk '{system("zip -jr ./3rd-party-sources.zip " $0)}' + mkdir -p ../license-obligations && mv `find . -regex "^./3rd-party-.*.zip$"` ../license-obligations/ + - name: Update Release with license obligations resources + uses: ncipollo/release-action@v1 + with: + allowUpdates: true + artifacts: ${{ env.WORKING_DIRECTORY }}/license-obligations/* + artifactErrorsFailBuild: true + makeLatest: true + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=match,pattern=provisioner-v(\d+.\d+.\d+),group=1 + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: ${{ env.WORKING_DIRECTORY }} + file: ${{ env.WORKING_DIRECTORY }}/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/provisioner.yaml b/.github/workflows/provisioner.yaml deleted file mode 100644 index 02e210a..0000000 --- a/.github/workflows/provisioner.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2022 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. -# -# SPDX-License-Identifier: Apache-2.0 -# -name: Build and publish Provisioner -on: - workflow_dispatch: - push: -defaults: - run: - working-directory: klyshko-provisioner -env: - REGISTRY: ghcr.io - IMAGE_NAME: carbynestack/klyshko-provisioner - WORKING_DIRECTORY: klyshko-provisioner -jobs: - publish: - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Log in to container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - context: ${{ env.WORKING_DIRECTORY }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9d0f481 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. +# +# SPDX-License-Identifier: Apache-2.0 +# +name: Release +on: + push: + branches: + - master +jobs: + release-please: + runs-on: ubuntu-22.04 + steps: + - name: Release Please + uses: google-github-actions/release-please-action@v3 + with: + command: manifest + token: ${{ secrets.CS_MINION_PAT }} diff --git a/.github/workflows/stale-issue-cleanup.yaml b/.github/workflows/stale-issue-cleanup.yaml deleted file mode 100644 index 02fcb51..0000000 --- a/.github/workflows/stale-issue-cleanup.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2023 - for information on the respective copyright owner -# see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. -# -# SPDX-License-Identifier: Apache-2.0 -# -name: 'Stale Issue Cleanup' - -on: - - # Allow workflow to be triggered manually - workflow_dispatch: - - # Trigger at 1:00 AM each day - schedule: - - cron: '0 1 * * *' - -jobs: - - # re-usable workflow @see https://docs.github.com/en/actions/using-workflows/reusing-workflows - stale-issue-cleanup: - uses: carbynestack/.github/.github/workflows/stale-actions.yml@master diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..e90033b --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,6 @@ +{ + "klyshko-mp-spdz": "0.0.1", + "klyshko-operator": "0.0.1", + "klyshko-operator/charts/klyshko-operator": "0.0.1", + "klyshko-provisioner": "0.0.1" +} diff --git a/README.md b/README.md index 7974a3e..743e20b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Codacy Badge](https://app.codacy.com/project/badge/Grade/3a07fd83b67647138b8ea660d16cdc35)](https://www.codacy.com/gh/carbynestack/klyshko/dashboard?utm_source=github.com&utm_medium=referral&utm_content=carbynestack/klyshko&utm_campaign=Badge_Grade) [![codecov](https://codecov.io/gh/carbynestack/klyshko/branch/master/graph/badge.svg?token=6hRb7xRW6C)](https://codecov.io/gh/carbynestack/klyshko) +[![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-%23FE5196?logo=conventionalcommits&logoColor=white)](https://conventionalcommits.org) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) diff --git a/commitlint.config.js b/commitlint.config.js index 7773fa1..da4f9ce 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 - for information on the respective copyright owner + * Copyright (c) 2023 - for information on the respective copyright owner * see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. * * SPDX-License-Identifier: Apache-2.0 @@ -16,6 +16,7 @@ module.exports = { [ "mp-spdz", "operator", + "operator-chart", "provisioner" ] ] diff --git a/klyshko-mp-spdz/3RD-PARTY-LICENSES/sbom.json b/klyshko-mp-spdz/3RD-PARTY-LICENSES/sbom.json new file mode 100644 index 0000000..0d4f101 --- /dev/null +++ b/klyshko-mp-spdz/3RD-PARTY-LICENSES/sbom.json @@ -0,0 +1,2 @@ +[ +] diff --git a/klyshko-mp-spdz/Dockerfile.fake-offline b/klyshko-mp-spdz/Dockerfile.fake-offline index ee5ea25..d8f1800 100644 --- a/klyshko-mp-spdz/Dockerfile.fake-offline +++ b/klyshko-mp-spdz/Dockerfile.fake-offline @@ -1,13 +1,24 @@ # -# Copyright (c) 2022 - for information on the respective copyright owner +# Copyright (c) 2023 - for information on the respective copyright owner # see the NOTICE file and/or the repository https://github.com/carbynestack/klyshko. # # SPDX-License-Identifier: Apache-2.0 # FROM ghcr.io/carbynestack/spdz:5350e66 +ARG RELEASE_PAGE="https://github.com/carbynestack/klyshko/releases" + # Copy resources +COPY 3RD-PARTY-LICENSES /3RD-PARTY-LICENSES COPY kii-run.sh . +RUN printf "\n## Klyshko MP-SPDZ\n\ +General information about third-party software components and their licenses, \ +which are distributed with Klyshko MP-SPDZ, can be found in the \ +[SBOM](./sbom.json). Further details are available in the subfolder for the \ +respective component or can be downloaded from the \ +[Klyshko Release Page](%s).\n" "${RELEASE_PAGE}"\ + >> /3RD-PARTY-LICENSES/disclosure.md + # Run the kii-run script ENTRYPOINT ["/bin/bash", "-c", "./kii-run.sh"] diff --git a/klyshko-mp-spdz/version.txt b/klyshko-mp-spdz/version.txt new file mode 100644 index 0000000..8acdd82 --- /dev/null +++ b/klyshko-mp-spdz/version.txt @@ -0,0 +1 @@ +0.0.1 diff --git a/klyshko-operator/Dockerfile b/klyshko-operator/Dockerfile index d36feb1..6c24cc7 100644 --- a/klyshko-operator/Dockerfile +++ b/klyshko-operator/Dockerfile @@ -3,6 +3,8 @@ # Build the manager binary FROM golang:1.16 as builder +ARG RELEASE_PAGE="https://github.com/carbynestack/klyshko/releases" + WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod @@ -23,8 +25,21 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go # Use compliant base image from carbynestack/base-images FROM ghcr.io/carbynestack/ubuntu:20.04-20210827-nonroot -WORKDIR / -COPY --from=builder /workspace/manager . -USER 65532:65532 + +# Copy resources +COPY --from=builder /workspace/manager / +COPY 3RD-PARTY-LICENSES /3RD-PARTY-LICENSES + +USER root:root +RUN chmod -R ugo+rXw /3RD-PARTY-LICENSES/disclosure.md +RUN printf "\n## Klyshko Operator\n\ +General information about third-party software components and their licenses, \ +which are distributed with Klyshko Operator, can be found in the \ +[SBOM](./sbom.json). Further details are available in the subfolder for the \ +respective component or can be downloaded from the \ +[Klyshko Release Page](%s).\n" "${RELEASE_PAGE}"\ + >> /3RD-PARTY-LICENSES/disclosure.md + +USER cs:cs ENTRYPOINT ["/manager"] diff --git a/klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/LICENSE b/klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/NOTICE.md b/klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/NOTICE.md new file mode 100644 index 0000000..dfb4c73 --- /dev/null +++ b/klyshko-provisioner/3RD-PARTY-LICENSES/github.com/carbynestack/cli/NOTICE.md @@ -0,0 +1,31 @@ +# Copyright Holders and Authors + +For instructions how to maintain this file see the +[NOTICE](https://github.com/carbynestack/carbynestack/blob/master/NOTICE.md) +file in the Carbyne Stack +[umbrella repository](https://github.com/carbynestack/carbynestack). + +## Contributors + +> **NOTE**: Please keep the following list of contributors sorted. + +### Bosch.IO GmbH + +- Suschke Volker +- Modica Hanna + +### Honda Research Institute Europe GmbH + +- Graf Johannes (synyx) [info@honda-ri.de](mailto:info@honda-ri.de) +- Klenk Timo (synyx) [info@honda-ri.de](mailto:info@honda-ri.de) +- Scherer Petra (synyx) [info@honda-ri.de](mailto:info@honda-ri.de) + +### Robert Bosch LLC + +- Jain Shalabh + +### Robert Bosch GmbH + +- Becker Sebastian +- Raskin Vadim +- Trieflinger Sven diff --git a/klyshko-provisioner/3RD-PARTY-LICENSES/sbom.json b/klyshko-provisioner/3RD-PARTY-LICENSES/sbom.json new file mode 100644 index 0000000..13de67b --- /dev/null +++ b/klyshko-provisioner/3RD-PARTY-LICENSES/sbom.json @@ -0,0 +1,6 @@ +[ + { + "project": "github.com/carbynestack/cli", + "license": "Apache-2.0" + } +] diff --git a/klyshko-provisioner/Dockerfile b/klyshko-provisioner/Dockerfile index fa70753..c2fe5ba 100644 --- a/klyshko-provisioner/Dockerfile +++ b/klyshko-provisioner/Dockerfile @@ -6,11 +6,22 @@ # FROM ghcr.io/carbynestack/openjdk:jre8-20210827 +ARG RELEASE_PAGE="https://github.com/carbynestack/klyshko/releases" + # Download Carbyne Stack CLI Jar at specified version ARG CLI_VERSION=0.2-SNAPSHOT-2336890983-14-a4260ab -ADD https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-$CLI_VERSION-jar-with-dependencies.jar cs.jar +ADD https://github.com/carbynestack/cli/releases/download/$CLI_VERSION/cli-${CLI_VERSION}-jar-with-dependencies.jar cs.jar # Copy resources +COPY 3RD-PARTY-LICENSES /3RD-PARTY-LICENSES COPY kii-provisioner-run.sh . +RUN printf "\n## Klyshko Provisioner\n\ +General information about third-party software components and their licenses, \ +which are distributed with Klyshko Provisioner, can be found in the \ +[SBOM](./sbom.json). Further details are available in the subfolder for the \ +respective component or can be downloaded from the \ +[Klyshko Release Page](%s).\n" "${RELEASE_PAGE}"\ + >> /3RD-PARTY-LICENSES/disclosure.md + ENTRYPOINT ["/bin/bash", "-c", "./kii-provisioner-run.sh"] diff --git a/klyshko-provisioner/version.txt b/klyshko-provisioner/version.txt new file mode 100644 index 0000000..8acdd82 --- /dev/null +++ b/klyshko-provisioner/version.txt @@ -0,0 +1 @@ +0.0.1 diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..5833827 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,24 @@ +{ + "bump-minor-pre-major": true, + "separate-pull-requests": true, + "pull-request-title-pattern": "chore: release ${component} ${version}", + "pull-request-header": ":package: Staging a new release", + "packages": { + "klyshko-mp-spdz": { + "package-name": "mp-spdz", + "release-type": "simple" + }, + "klyshko-operator": { + "package-name": "operator", + "release-type": "go" + }, + "klyshko-operator/charts/klyshko-operator": { + "package-name": "operator-chart", + "release-type": "helm" + }, + "klyshko-provisioner": { + "package-name": "provisioner", + "release-type": "simple" + } + } +} From 52fd89b818ab7d783ee24ee41acbddd06b33c0fb Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Sat, 11 Mar 2023 10:14:50 +0100 Subject: [PATCH 68/69] chore: add prettier hook to pre-commit Signed-off-by: Sven Trieflinger --- .../workflows/operator.build-and-test.yaml | 1 + .pre-commit-config.yaml | 5 ++++ commitlint.config.js | 25 +++++++------------ 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/operator.build-and-test.yaml b/.github/workflows/operator.build-and-test.yaml index b79afce..5b7c510 100644 --- a/.github/workflows/operator.build-and-test.yaml +++ b/.github/workflows/operator.build-and-test.yaml @@ -50,6 +50,7 @@ jobs: ${{ runner.os }}-go- - name: Build and Test run: | + cd klyshko-operator make test - name: Publishing Coverage uses: codecov/codecov-action@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f2ba26e..b94deb6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,11 @@ repos: rev: v0.7.2 hooks: - id: shellcheck + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + types_or: [javascript] - repo: https://github.com/executablebooks/mdformat rev: 0.7.16 hooks: diff --git a/commitlint.config.js b/commitlint.config.js index da4f9ce..2218b09 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -5,20 +5,13 @@ * SPDX-License-Identifier: Apache-2.0 */ module.exports = { - extends: [ - "@commitlint/config-conventional" + extends: ["@commitlint/config-conventional"], + rules: { + "scope-empty": [0, "never"], + "scope-enum": [ + 2, + "always", + ["mp-spdz", "operator", "operator-chart", "provisioner"], ], - rules: { - 'scope-empty': [0, 'never'], - "scope-enum": [ - 2, - "always", - [ - "mp-spdz", - "operator", - "operator-chart", - "provisioner" - ] - ] - } -} + }, +}; From dcbe01ff3112b07624b469c10a9c039b93a361b1 Mon Sep 17 00:00:00 2001 From: Sven Trieflinger Date: Mon, 13 Mar 2023 11:31:17 +0100 Subject: [PATCH 69/69] chore(operator): add source link for MPL2.0-versioned dependency Signed-off-by: Sven Trieflinger --- .../github.com/hashicorp/golang-lru/origin.src | 1 + 1 file changed, 1 insertion(+) create mode 100644 klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/origin.src diff --git a/klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/origin.src b/klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/origin.src new file mode 100644 index 0000000..c4b613e --- /dev/null +++ b/klyshko-operator/3RD-PARTY-LICENSES/github.com/hashicorp/golang-lru/origin.src @@ -0,0 +1 @@ +https://github.com/hashicorp/golang-lru/archive/refs/tags/v0.5.4.zip