Merge branch 'k8up-io:master' into enhancement/add_imagePullSecret

.gitignore

@@ -42,3 +42,11 @@ e2e/debug
 # Charts
 .cr-release-packages/
 .cr-index/
+
+# Vagrant
+.vagrant/
+
+# Container volumes mount
+.config/
+.kube/
+.npm/

Makefile

@@ -27,11 +27,11 @@ include Makefile.restic-integration.mk envtest/integration.mk
 # E2E tests
 -include e2e/Makefile
 
-go_build ?= go build -o $(BIN_FILENAME) $(K8UP_MAIN_GO)
+go_build ?= $(GO_EXEC) build -o $(BIN_FILENAME) $(K8UP_MAIN_GO)
 
 .PHONY: test
 test: ## Run tests
-	go test ./... -coverprofile cover.out
+	$(GO_EXEC) test ./... -coverprofile cover.out
 
 .PHONY: build
 build: generate fmt vet $(BIN_FILENAME) docs-update-usage ## Build manager binary
@@ -41,7 +41,7 @@ run: export BACKUP_ENABLE_LEADER_ELECTION = $(ENABLE_LEADER_ELECTION)
 run: export K8UP_DEBUG = true
 run: export BACKUP_OPERATOR_NAMESPACE = default
 run: fmt vet ## Run against the configured Kubernetes cluster in ~/.kube/config. Use ARGS to pass arguments to the command, e.g. `make run ARGS="--help"`
-	go run $(K8UP_MAIN_GO) $(ARGS) $(CMD) $(CMD_ARGS)
+	$(GO_EXEC) run $(K8UP_MAIN_GO) $(ARGS) $(CMD) $(CMD_ARGS)
 
 .PHONY: run-operator
 run-operator: CMD := operator
@@ -80,21 +80,21 @@ deploy: kind-load-image install ## Deploy controller in the configured Kubernete
 .PHONY: generate
 generate: ## Generate manifests e.g. CRD, RBAC etc.
 	# Generate code
-	go run sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile=".github/boilerplate.go.txt" paths="./..."
+	$(GO_EXEC) run sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile=".github/boilerplate.go.txt" paths="./..."
 	# Generate CRDs
-	go run sigs.k8s.io/controller-tools/cmd/controller-gen rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=$(CRD_ROOT_DIR)/v1 crd:crdVersions=v1
+	$(GO_EXEC) run sigs.k8s.io/controller-tools/cmd/controller-gen rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=$(CRD_ROOT_DIR)/v1 crd:crdVersions=v1
 
 .PHONY: crd
 crd: generate ## Generate CRD to file
 	@yq $(CRD_ROOT_DIR)/v1/*.yaml > $(CRD_FILE)
 
 .PHONY: fmt
 fmt: ## Run go fmt against code
-	go fmt ./...
+	$(GO_EXEC) fmt ./...
 
 .PHONY: vet
 vet: ## Run go vet against code
-	go vet ./...
+	$(GO_EXEC) vet ./...
 
 .PHONY: lint
 lint: fmt vet golangci-lint ## Invokes all linting targets

Makefile.restic-integration.vars.mk

@@ -13,7 +13,7 @@ restore_dir ?= $(integrationtest_dir)/restore
 
 stats_url ?= http://localhost:8091
 
-restic_version ?= $(shell go mod edit -json | jq -r '.Require[] | select(.Path == "github.com/restic/restic").Version' | sed "s/v//")
+restic_version ?= $(shell $(GO_EXEC) mod edit -json | jq -r '.Require[] | select(.Path == "github.com/restic/restic").Version' | sed "s/v//")
 restic_path ?= $(go_bin)/restic
 restic_pid ?= $(integrationtest_dir)/restic.pid
 restic_url ?= https://github.com/restic/restic/releases/download/v$(restic_version)/restic_$(restic_version)_$(os)_$(arch).bz2

Makefile.vars.mk

@@ -1,5 +1,6 @@
 IMG_TAG ?= latest
 
+GO_EXEC ?= go
 K8UP_MAIN_GO ?= cmd/k8up/main.go
 K8UP_GOOS ?= linux
 K8UP_GOARCH ?= amd64

api/v1/backend.go

@@ -25,6 +25,9 @@ type (
 		Swift   *SwiftSpec             `json:"swift,omitempty"`
 		B2      *B2Spec                `json:"b2,omitempty"`
 		Rest    *RestServerSpec        `json:"rest,omitempty"`
+
+		TLSOptions   *TLSOptions           `json:"tlsOptions,omitempty"`
+		VolumeMounts *[]corev1.VolumeMount `json:"volumeMounts,omitempty"`
 	}
 
 	// +k8s:deepcopy-gen=false
@@ -279,3 +282,9 @@ func (in *RestServerSpec) String() string {
 	protocol, url, _ := strings.Cut(in.URL, "://")
 	return fmt.Sprintf("rest:%s://%s:%s@%s", protocol, "$(USER)", "$(PASSWORD)", url)
 }
+
+type TLSOptions struct {
+	CACert     string `json:"caCert,omitempty"`
+	ClientCert string `json:"clientCert,omitempty"`
+	ClientKey  string `json:"clientKey,omitempty"`
+}

api/v1/restore_types.go

@@ -35,8 +35,10 @@ type RestoreSpec struct {
 // RestoreMethod contains how and where the restore should happen
 // all the settings are mutual exclusive.
 type RestoreMethod struct {
-	S3     *S3Spec        `json:"s3,omitempty"`
-	Folder *FolderRestore `json:"folder,omitempty"`
+	S3           *S3Spec               `json:"s3,omitempty"`
+	Folder       *FolderRestore        `json:"folder,omitempty"`
+	TLSOptions   *TLSOptions           `json:"tlsOptions,omitempty"`
+	VolumeMounts *[]corev1.VolumeMount `json:"volumeMounts,omitempty"`
 }
 
 type FolderRestore struct {

api/v1/runnable_types.go

@@ -15,11 +15,34 @@ type RunnableSpec struct {
 	// PodSecurityContext describes the security context with which this action shall be executed.
 	PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
 
+	// Volumes List of volumes that can be mounted by containers belonging to the pod.
+	Volumes *[]RunnableVolumeSpec `json:"volumes,omitempty"`
+
 	// ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be continuously active before the system tries to terminate it.
 	// Value must be positive integer if given.
 	ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
 }
 
+type RunnableVolumeSpec struct {
+	// name of the volume.
+	// Must be a DNS_LABEL and unique within the pod.
+	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+	Name string `json:"name"`
+
+	// persistentVolumeClaimVolumeSource represents a reference to a
+	// PersistentVolumeClaim in the same namespace.
+	// More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims
+	// +optional
+	PersistentVolumeClaim *corev1.PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty"`
+	// secret represents a secret that should populate this volume.
+	// More info: https://kubernetes.io/docs/concepts/storage/volumes#secret
+	// +optional
+	Secret *corev1.SecretVolumeSource `json:"secret,omitempty"`
+	// configMap represents a configMap that should populate this volume
+	// +optional
+	ConfigMap *corev1.ConfigMapVolumeSource `json:"configMap,omitempty"`
+}
+
 // AppendEnvFromToContainer will add EnvFromSource from the given RunnableSpec to the Container
 func (in *RunnableSpec) AppendEnvFromToContainer(containerSpec *corev1.Container) {
 	if in.Backend != nil {

charts/k8up/README.md

@@ -69,6 +69,8 @@ Document your changes in values.yaml and let `make docs:helm` generate this sect
 | metrics.prometheusRule.jobFailedRulesFor | list | `["archive","backup","check","prune","restore"]` | Create default rules for the given job types. Valid values are "archive", "backup", "check", "prune", and "restore". |
 | metrics.prometheusRule.namespace | string | `""` | If the object should be installed in a different namespace than operator |
 | metrics.service.annotations | object | `{}` | Annotations to add to the service |
+| metrics.service.ipFamilies | list | `[]` | Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. Can be IPv4 and/or IPv6. |
+| metrics.service.ipFamilyPolicy | string | `""` | Set the ip family policy to configure dual-stack see [Configure dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services) |
 | metrics.service.nodePort | int | `0` | Service node port of the metrics endpoint, requires `metrics.service.type=NodePort` |
 | metrics.service.port | int | `8080` |  |
 | metrics.service.type | string | `"ClusterIP"` |  |

charts/k8up/templates/service.yaml

@@ -10,6 +10,12 @@ metadata:
   {{- end }}
 spec:
   type: {{ .Values.metrics.service.type }}
+  {{- if .Values.metrics.service.ipFamilyPolicy }}
+  ipFamilyPolicy: {{ .Values.metrics.service.ipFamilyPolicy }}
+  {{- end }}
+  {{- if .Values.metrics.service.ipFamilies }}
+  ipFamilies: {{ .Values.metrics.service.ipFamilies | toYaml | nindent 2 }}
+  {{- end }}
   ports:
     - name: http
       port: {{ .Values.metrics.service.port }}

charts/k8up/values.yaml

@@ -94,6 +94,10 @@ metrics:
   service:
     port: 8080
     type: ClusterIP
+    # -- Set the ip family policy to configure dual-stack see [Configure dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services)
+    ipFamilyPolicy: ""
+    # -- Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. Can be IPv4 and/or IPv6.
+    ipFamilies: []
     # -- Service node port of the metrics endpoint, requires `metrics.service.type=NodePort`
     nodePort: 0
     # -- Annotations to add to the service

clean.sh

@@ -9,7 +9,12 @@ pidfile_exists() {
 }
 
 pid_alive() {
-  xargs ps -p >/dev/null < "${1}"
+  if ps --help 2>&1 | grep -q BusyBox; then
+    xargs ps p >/dev/null < "${1}"
+  else
+    xargs ps -p >/dev/null < "${1}"
+  fi
+
   return $?
 }
 

cmd/operator/main.go

@@ -85,6 +85,8 @@ var (
 			&cli.BoolFlag{Destination: &cfg.Config.SkipWithoutAnnotation, Name: "skip-pvcs-without-annotation", EnvVars: []string{"BACKUP_SKIP_WITHOUT_ANNOTATION"}, Value: false, DefaultText: "disabled", Usage: "skip selecting PVCs that don't have the BACKUP_ANNOTATION"},
 			&cli.StringFlag{Destination: &cfg.Config.BackupCheckSchedule, Name: "checkschedule", EnvVars: []string{"BACKUP_CHECKSCHEDULE"}, Value: "0 0 * * 0", Usage: "the default check schedule"},
 			&cli.StringFlag{Destination: &cfg.Config.OperatorNamespace, Name: "operator-namespace", EnvVars: []string{"BACKUP_OPERATOR_NAMESPACE"}, Required: true, Usage: "set the namespace in which the K8up operator itself runs"},
+
+			&cli.StringFlag{Destination: &cfg.Config.PodVarDir, Name: "vardir", EnvVars: []string{"VAR_DIR"}, Value: "/k8up", Usage: "the var data dir for read/write k8up data or temp file in the backup pod"},
 		},
 	}
 )

cmd/restic/integration_test.go

@@ -131,7 +131,7 @@ func initTest(t *testing.T) *testEnvironment {
 
 func connectToS3Server(t *testing.T, ctx context.Context) *s3.Client {
 	repo := getS3Repo()
-	s3client := s3.New(repo, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"))
+	s3client := s3.New(repo, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), s3.Cert{})
 
 	err := s3client.Connect(ctx)
 	require.NoErrorf(t, err, "Unable to connect to S3 repo '%s'", repo)
@@ -211,7 +211,7 @@ func testBackup(t *testing.T) *testEnvironment {
 }
 
 func testCheckS3Restore(t *testing.T, ctx context.Context) {
-	s3c := s3.New(os.Getenv("RESTORE_S3ENDPOINT"), os.Getenv("RESTORE_ACCESSKEYID"), os.Getenv("RESTORE_SECRETACCESSKEY"))
+	s3c := s3.New(os.Getenv("RESTORE_S3ENDPOINT"), os.Getenv("RESTORE_ACCESSKEYID"), os.Getenv("RESTORE_SECRETACCESSKEY"), s3.Cert{})
 	err := s3c.Connect(ctx)
 	require.NoError(t, err)
 	files, err := s3c.ListObjects(ctx)