From 29768f8d2b8f7dd42122b0e9baed5ccabe6189fe Mon Sep 17 00:00:00 2001 From: Kaihang Zhang Date: Tue, 18 Apr 2023 16:52:33 +0800 Subject: [PATCH] feat: Support VM HA --- .github/workflows/build.yml | 2 + .github/workflows/release.yml | 30 +- build/lockspace-attacher/Dockerfile | 21 + build/lockspace-detector/Dockerfile | 21 + build/lockspace-initializer/Dockerfile | 21 + build/virt-controller/Dockerfile | 6 + build/virt-prerunner/Dockerfile | 18 +- .../virt-prerunner/cloud-hypervisor-finish.sh | 10 - build/virt-prerunner/cloud-hypervisor-run.sh | 3 - build/virt-prerunner/cloud-hypervisor-type | 1 - build/virt-prerunner/virt-prerunner-run.sh | 3 - build/virt-prerunner/virt-prerunner-type | 1 - build/virt-prerunner/virt-prerunner-up | 1 - cmd/lockspace-attacher/main.go | 100 ++++ cmd/lockspace-detector/main.go | 483 +++++++++++++++ cmd/lockspace-initializer/main.go | 39 ++ cmd/virt-controller/main.go | 12 + cmd/virt-prerunner/main.go | 33 +- deploy/crd/virt.virtink.smartx.com_locks.yaml | 60 ++ .../virt.virtink.smartx.com_lockspaces.yaml | 74 +++ ...rt.virtink.smartx.com_virtualmachines.yaml | 6 + deploy/kustomization.yaml | 4 + deploy/lockspace-attacher/kustomization.yaml | 4 + deploy/lockspace-attacher/role.yaml | 13 + deploy/lockspace-attacher/rolebinding.yaml | 12 + deploy/lockspace-attacher/sa.yaml | 5 + deploy/lockspace-detector/kustomization.yaml | 4 + deploy/lockspace-detector/role.yaml | 84 +++ deploy/lockspace-detector/rolebinding.yaml | 12 + deploy/lockspace-detector/sa.yaml | 5 + deploy/virt-controller/role.yaml | 39 ++ docs/images/vm_ha_architecture.jpg | Bin 0 -> 85474 bytes docs/vm_ha.md | 172 ++++++ go.mod | 49 +- go.sum | 72 ++- hack/Dockerfile | 5 +- hack/generate.sh | 2 + pkg/apis/virt/v1alpha1/register.go | 4 + pkg/apis/virt/v1alpha1/types.go | 96 +++ .../virt/v1alpha1/zz_generated.deepcopy.go | 196 +++++++ pkg/controller/lockspace_controller.go | 549 ++++++++++++++++++ pkg/controller/vm_controller.go | 102 ++++ pkg/controller/vm_webhook.go | 10 + pkg/controller/vm_webhook_test.go | 18 + .../typed/virt/v1alpha1/fake/fake_lock.go | 126 ++++ .../virt/v1alpha1/fake/fake_lockspace.go | 117 ++++ .../virt/v1alpha1/fake/fake_virt_client.go | 8 + .../virt/v1alpha1/generated_expansion.go | 4 + .../versioned/typed/virt/v1alpha1/lock.go | 179 ++++++ .../typed/virt/v1alpha1/lockspace.go | 168 ++++++ .../typed/virt/v1alpha1/virt_client.go | 10 + .../informers/externalversions/generic.go | 4 + .../virt/v1alpha1/interface.go | 14 + .../externalversions/virt/v1alpha1/lock.go | 74 +++ .../virt/v1alpha1/lockspace.go | 73 +++ .../virt/v1alpha1/expansion_generated.go | 12 + pkg/generated/listers/virt/v1alpha1/lock.go | 83 +++ .../listers/virt/v1alpha1/lockspace.go | 52 ++ pkg/sanlock/sanlock.go | 297 ++++++++++ samples/ubuntu-ha.yaml | 47 ++ skaffold.yaml | 15 + 61 files changed, 3589 insertions(+), 96 deletions(-) create mode 100644 build/lockspace-attacher/Dockerfile create mode 100644 build/lockspace-detector/Dockerfile create mode 100644 build/lockspace-initializer/Dockerfile delete mode 100755 build/virt-prerunner/cloud-hypervisor-finish.sh delete mode 100755 build/virt-prerunner/cloud-hypervisor-run.sh delete mode 100644 build/virt-prerunner/cloud-hypervisor-type delete mode 100755 build/virt-prerunner/virt-prerunner-run.sh delete mode 100644 build/virt-prerunner/virt-prerunner-type delete mode 100755 build/virt-prerunner/virt-prerunner-up create mode 100644 cmd/lockspace-attacher/main.go create mode 100644 cmd/lockspace-detector/main.go create mode 100644 cmd/lockspace-initializer/main.go create mode 100644 deploy/crd/virt.virtink.smartx.com_locks.yaml create mode 100644 deploy/crd/virt.virtink.smartx.com_lockspaces.yaml create mode 100644 deploy/lockspace-attacher/kustomization.yaml create mode 100644 deploy/lockspace-attacher/role.yaml create mode 100644 deploy/lockspace-attacher/rolebinding.yaml create mode 100644 deploy/lockspace-attacher/sa.yaml create mode 100644 deploy/lockspace-detector/kustomization.yaml create mode 100644 deploy/lockspace-detector/role.yaml create mode 100644 deploy/lockspace-detector/rolebinding.yaml create mode 100644 deploy/lockspace-detector/sa.yaml create mode 100644 docs/images/vm_ha_architecture.jpg create mode 100644 docs/vm_ha.md create mode 100644 pkg/controller/lockspace_controller.go create mode 100644 pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lock.go create mode 100644 pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lockspace.go create mode 100644 pkg/generated/clientset/versioned/typed/virt/v1alpha1/lock.go create mode 100644 pkg/generated/clientset/versioned/typed/virt/v1alpha1/lockspace.go create mode 100644 pkg/generated/informers/externalversions/virt/v1alpha1/lock.go create mode 100644 pkg/generated/informers/externalversions/virt/v1alpha1/lockspace.go create mode 100644 pkg/generated/listers/virt/v1alpha1/lock.go create mode 100644 pkg/generated/listers/virt/v1alpha1/lockspace.go create mode 100644 pkg/sanlock/sanlock.go create mode 100644 samples/ubuntu-ha.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6e513b..0f7a52e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,8 @@ jobs: with: go-version: 1.19.3 + - run: sudo apt-get update && sudo apt-get install libaio-dev libsanlock-dev + - run: make test - uses: codecov/codecov-action@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f7c166..e3cc83c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,30 @@ jobs: username: smartxrocks password: ${{ secrets.DOCKERHUB_PUSH_TOKEN }} + - id: build_lockspace_initializer + uses: docker/build-push-action@v2 + with: + file: build/lockspace-initializer/Dockerfile + tags: smartxworks/lockspace-initializer:${{ steps.get_version.outputs.version }} + platforms: linux/amd64,linux/arm64 + push: true + + - id: build_lockspace_attacher + uses: docker/build-push-action@v2 + with: + file: build/lockspace-attacher/Dockerfile + tags: smartxworks/lockspace-attacher:${{ steps.get_version.outputs.version }} + platforms: linux/amd64,linux/arm64 + push: true + + - id: build_lockspace_detector + uses: docker/build-push-action@v2 + with: + file: build/lockspace-detector/Dockerfile + tags: smartxworks/lockspace-detector:${{ steps.get_version.outputs.version }} + platforms: linux/amd64,linux/arm64 + push: true + - id: build_virt_prerunner uses: docker/build-push-action@v2 with: @@ -34,7 +58,11 @@ jobs: - uses: docker/build-push-action@v2 with: file: build/virt-controller/Dockerfile - build-args: PRERUNNER_IMAGE=smartxworks/virt-prerunner:${{ steps.get_version.outputs.version }}@${{ steps.build_virt_prerunner.outputs.digest }} + build-args: | + PRERUNNER_IMAGE=smartxworks/virt-prerunner:${{ steps.get_version.outputs.version }}@${{ steps.build_virt_prerunner.outputs.digest }} + INITIALIZER_IMAGE=smartxworks/lockspace-initializer:${{ steps.get_version.outputs.version }}@${{ steps.build_lockspace_initializer.outputs.digest }} + ATTACHER_IMAGE=smartxworks/lockspace-attacher:${{ steps.get_version.outputs.version }}@${{ steps.build_lockspace_attacher.outputs.digest }} + DETECTOR_IMAGE=smartxworks/lockspace-detector:${{ steps.get_version.outputs.version }}@${{ steps.build_lockspace_detector.outputs.digest }} tags: smartxworks/virt-controller:${{ steps.get_version.outputs.version }} platforms: linux/amd64,linux/arm64 push: true diff --git a/build/lockspace-attacher/Dockerfile b/build/lockspace-attacher/Dockerfile new file mode 100644 index 0000000..7d2d11f --- /dev/null +++ b/build/lockspace-attacher/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.19-alpine AS builder + +RUN apk add --no-cache gcc musl-dev libaio-dev +RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ + +WORKDIR /workspace + +COPY go.mod go.mod +COPY go.sum go.sum +RUN go mod download + +COPY cmd/ cmd/ +COPY pkg/ pkg/ +RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/lockspace-attacher/main.go + +FROM alpine + +RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ + +COPY --from=builder /workspace/main /usr/bin/lockspace-attacher +ENTRYPOINT ["lockspace-attacher"] diff --git a/build/lockspace-detector/Dockerfile b/build/lockspace-detector/Dockerfile new file mode 100644 index 0000000..0600ab8 --- /dev/null +++ b/build/lockspace-detector/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.19-alpine AS builder + +RUN apk add --no-cache gcc musl-dev libaio-dev +RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ + +WORKDIR /workspace + +COPY go.mod go.mod +COPY go.sum go.sum +RUN go mod download + +COPY cmd/ cmd/ +COPY pkg/ pkg/ +RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/lockspace-detector/main.go + +FROM alpine + +RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ + +COPY --from=builder /workspace/main /usr/bin/lockspace-detector +ENTRYPOINT ["lockspace-detector"] diff --git a/build/lockspace-initializer/Dockerfile b/build/lockspace-initializer/Dockerfile new file mode 100644 index 0000000..9160593 --- /dev/null +++ b/build/lockspace-initializer/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.19-alpine AS builder + +RUN apk add --no-cache gcc musl-dev libaio-dev +RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ + +WORKDIR /workspace + +COPY go.mod go.mod +COPY go.sum go.sum +RUN go mod download + +COPY cmd/ cmd/ +COPY pkg/ pkg/ +RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/lockspace-initializer/main.go + +FROM alpine + +RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ + +COPY --from=builder /workspace/main /usr/bin/lockspace-initializer +ENTRYPOINT ["lockspace-initializer"] diff --git a/build/virt-controller/Dockerfile b/build/virt-controller/Dockerfile index e781cf6..f183624 100644 --- a/build/virt-controller/Dockerfile +++ b/build/virt-controller/Dockerfile @@ -14,6 +14,12 @@ FROM alpine ARG PRERUNNER_IMAGE ENV PRERUNNER_IMAGE=$PRERUNNER_IMAGE +ARG DETECTOR_IMAGE +ENV DETECTOR_IMAGE=$DETECTOR_IMAGE +ARG ATTACHER_IMAGE +ENV ATTACHER_IMAGE=$ATTACHER_IMAGE +ARG INITIALIZER_IMAGE +ENV INITIALIZER_IMAGE=$INITIALIZER_IMAGE COPY --from=builder /workspace/main /usr/bin/virt-controller ENTRYPOINT ["virt-controller"] diff --git a/build/virt-prerunner/Dockerfile b/build/virt-prerunner/Dockerfile index daaa947..df11a58 100644 --- a/build/virt-prerunner/Dockerfile +++ b/build/virt-prerunner/Dockerfile @@ -1,6 +1,7 @@ FROM golang:1.19-alpine AS builder -RUN apk add --no-cache gcc musl-dev +RUN apk add --no-cache gcc musl-dev libaio-dev +RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ WORKDIR /workspace @@ -14,7 +15,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/virt-prerunn FROM alpine -RUN apk add --no-cache curl screen dnsmasq cdrkit iptables iproute2 qemu-virtiofsd dpkg util-linux s6-overlay nmap-ncat +RUN apk add --no-cache curl screen dnsmasq cdrkit iptables iproute2 qemu-virtiofsd dpkg util-linux tini nmap-ncat +RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ RUN set -eux; \ mkdir /var/lib/cloud-hypervisor; \ @@ -34,19 +36,9 @@ RUN set -eux; \ chmod +x /usr/bin/cloud-hypervisor; \ chmod +x /usr/bin/ch-remote -COPY build/virt-prerunner/cloud-hypervisor-type /etc/s6-overlay/s6-rc.d/cloud-hypervisor/type -COPY build/virt-prerunner/cloud-hypervisor-run.sh /etc/s6-overlay/s6-rc.d/cloud-hypervisor/run -COPY build/virt-prerunner/cloud-hypervisor-finish.sh /etc/s6-overlay/s6-rc.d/cloud-hypervisor/finish -RUN touch /etc/s6-overlay/s6-rc.d/user/contents.d/cloud-hypervisor - COPY --from=builder /workspace/main /usr/bin/virt-prerunner -COPY build/virt-prerunner/virt-prerunner-type /etc/s6-overlay/s6-rc.d/virt-prerunner/type -COPY build/virt-prerunner/virt-prerunner-up /etc/s6-overlay/s6-rc.d/virt-prerunner/up -COPY build/virt-prerunner/virt-prerunner-run.sh /etc/s6-overlay/scripts/virt-prerunner-run.sh -RUN touch /etc/s6-overlay/s6-rc.d/user/contents.d/virt-prerunner -ENV S6_BEHAVIOUR_IF_STAGE2_FAILS=2 -ENTRYPOINT ["/init"] +ENTRYPOINT ["tini", "-s", "-g", "--", "/usr/bin/virt-prerunner"] COPY build/virt-prerunner/iptables-wrapper /sbin/iptables-wrapper RUN update-alternatives --install /sbin/iptables iptables /sbin/iptables-wrapper 100 diff --git a/build/virt-prerunner/cloud-hypervisor-finish.sh b/build/virt-prerunner/cloud-hypervisor-finish.sh deleted file mode 100755 index 6de07ad..0000000 --- a/build/virt-prerunner/cloud-hypervisor-finish.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -if test "$1" -eq 256 ; then - e=$((128 + $2)) -else - e="$1" -fi - -echo "$e" > /run/s6-linux-init-container-results/exitcode -/run/s6/basedir/bin/halt diff --git a/build/virt-prerunner/cloud-hypervisor-run.sh b/build/virt-prerunner/cloud-hypervisor-run.sh deleted file mode 100755 index b3d1c2c..0000000 --- a/build/virt-prerunner/cloud-hypervisor-run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/execlineb -P - -cloud-hypervisor --api-socket /var/run/virtink/ch.sock diff --git a/build/virt-prerunner/cloud-hypervisor-type b/build/virt-prerunner/cloud-hypervisor-type deleted file mode 100644 index 5883cff..0000000 --- a/build/virt-prerunner/cloud-hypervisor-type +++ /dev/null @@ -1 +0,0 @@ -longrun diff --git a/build/virt-prerunner/virt-prerunner-run.sh b/build/virt-prerunner/virt-prerunner-run.sh deleted file mode 100755 index b5f70c7..0000000 --- a/build/virt-prerunner/virt-prerunner-run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/with-contenv sh - -/usr/bin/virt-prerunner diff --git a/build/virt-prerunner/virt-prerunner-type b/build/virt-prerunner/virt-prerunner-type deleted file mode 100644 index bdd22a1..0000000 --- a/build/virt-prerunner/virt-prerunner-type +++ /dev/null @@ -1 +0,0 @@ -oneshot diff --git a/build/virt-prerunner/virt-prerunner-up b/build/virt-prerunner/virt-prerunner-up deleted file mode 100755 index 103a5e6..0000000 --- a/build/virt-prerunner/virt-prerunner-up +++ /dev/null @@ -1 +0,0 @@ -/etc/s6-overlay/scripts/virt-prerunner-run.sh diff --git a/cmd/lockspace-attacher/main.go b/cmd/lockspace-attacher/main.go new file mode 100644 index 0000000..1a0505c --- /dev/null +++ b/cmd/lockspace-attacher/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "syscall" + + "github.com/namsral/flag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/smartxworks/virtink/pkg/sanlock" +) + +func main() { + var lockspaceName string + var nodeName string + flag.StringVar(&lockspaceName, "lockspace-name", "", "") + flag.StringVar(&nodeName, "node-name", "", "") + flag.Parse() + + cfg, err := clientcmd.BuildConfigFromFlags("", "") + if err != nil { + log.Fatalf("failed to build kubeconfig: %s", err) + } + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Fatalf("failed to create Kubernetes client: %s", err) + } + + hostID, err := getHostID(kubeClient, nodeName) + if err != nil { + log.Fatalf("failed to get host ID: %s", err) + + } + + leaseFilePath := filepath.Join("/var/lib/sanlock", lockspaceName, "leases") + if err := sanlock.AcquireDeltaLease(lockspaceName, leaseFilePath, hostID); err != nil { + log.Fatalf("failed to acquire delta lease: %s", err) + } + log.Println("succeeded to acquire delta lease") + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + close(stop) + <-c + os.Exit(1) + }() + + <-stop + if err := sanlock.ReleaseDeltaLease(lockspaceName, leaseFilePath, hostID); err != nil { + if err != sanlock.ENOENT { + log.Fatalf("failed to release delta lease: %s", err) + } + } + if err := umountLeaseVolume(lockspaceName); err != nil { + log.Fatalf("failed to umont lease volume: %s", err) + } +} + +//+kubebuilder:rbac:groups="",resources=nodes,verbs=get + +func getHostID(client *kubernetes.Clientset, nodeName string) (uint64, error) { + node, err := client.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return 0, fmt.Errorf("get node: %s", err) + } + + if id, exist := node.Annotations["virtink.smartx.com/sanlock-host-id"]; exist { + if hostID, err := strconv.ParseUint(id, 10, 64); err != nil { + return 0, fmt.Errorf("parse uint: %s", err) + } else { + return hostID, nil + } + } + return 0, fmt.Errorf("sanlock host %s ID not found", nodeName) +} + +func umountLeaseVolume(lockspace string) error { + leaseFileDir := filepath.Join("/var/lib/sanlock", lockspace) + output, err := exec.Command("sh", "-c", fmt.Sprintf("mount | grep '%s'", leaseFileDir)).CombinedOutput() + if err != nil && string(output) == "" { + return nil + } + + if _, err := exec.Command("umount", leaseFileDir).CombinedOutput(); err != nil { + return fmt.Errorf("unmount volume: %s", err) + } + return nil +} diff --git a/cmd/lockspace-detector/main.go b/cmd/lockspace-detector/main.go new file mode 100644 index 0000000..3eed20e --- /dev/null +++ b/cmd/lockspace-detector/main.go @@ -0,0 +1,483 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "reflect" + "strconv" + "sync" + "time" + + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + "github.com/smartxworks/virtink/pkg/sanlock" +) + +const ( + LockProtectionFinalizer = "virtink.smartx.com/lock-protection" +) + +var ( + scheme = runtime.NewScheme() + + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(virtv1alpha1.AddToScheme(scheme)) +} + +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=lockspaces,verbs=get;list;watch +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=locks,verbs=get;list;watch;update +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=locks/status,verbs=get;update +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=virtualmachines,verbs=get;list;watch +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=virtualmachines/status,verbs=get;update +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;delete +//+kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch + +func main() { + opts := zap.Options{ + Development: true, + TimeEncoder: zapcore.ISO8601TimeEncoder, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + lockspace := os.Getenv("LOCKSPACE_NAME") + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + LeaderElection: true, + LeaderElectionID: fmt.Sprintf("%s-detector.virtink.smartx.com", lockspace), + }) + if err != nil { + setupLog.Error(err, "unable to create manager") + os.Exit(1) + } + + ioTimeout, err := strconv.Atoi(os.Getenv("IO_TIMEOUT")) + if err != nil { + setupLog.Error(err, "failed to get io_timeout_seconds") + } + if err = (&Detector{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("lockspace-detector"), + lockspace: lockspace, + ioTimeout: time.Duration(ioTimeout) * time.Second, + freeStateDetector: make(map[string]chan struct{}), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Detector") + os.Exit(1) + } + + if err = (&LockReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("lock-controller"), + lockspace: lockspace, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Lock") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +type Detector struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + lockspace string + ioTimeout time.Duration + + mutex sync.Mutex + freeStateDetector map[string]chan struct{} +} + +func (d *Detector) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + ls, err := d.getLockspaceOrNil(ctx, d.lockspace) + if err != nil { + return ctrl.Result{}, fmt.Errorf("get Lockspace: %s", err) + } + if ls == nil { + return ctrl.Result{}, nil + } + + var node corev1.Node + if err := d.Get(ctx, req.NamespacedName, &node); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if err := d.detectNode(ctx, &node); err != nil { + reconcileErr := reconcileError{} + if errors.As(err, &reconcileErr) { + return reconcileErr.Result, nil + } + return ctrl.Result{}, fmt.Errorf("reconcile Node: %s", err) + } + return ctrl.Result{RequeueAfter: 2 * d.ioTimeout}, nil +} + +func (d *Detector) getLockspaceOrNil(ctx context.Context, lockspace string) (*virtv1alpha1.Lockspace, error) { + lsKey := types.NamespacedName{ + Name: lockspace, + } + var ls virtv1alpha1.Lockspace + if err := d.Client.Get(ctx, lsKey, &ls); err != nil { + return nil, client.IgnoreNotFound(err) + } + + return &ls, nil +} + +func (d *Detector) detectNode(ctx context.Context, node *corev1.Node) error { + if node == nil || node.DeletionTimestamp != nil { + d.stopFreeStateDetector(node.Name) + return nil + } + + id, exist := node.Annotations["virtink.smartx.com/sanlock-host-id"] + if !exist { + return fmt.Errorf("sanlock host %s ID not found", node.Name) + } + hostID, err := strconv.ParseUint(id, 10, 64) + if err != nil { + return fmt.Errorf("parse uint: %s", err) + } + + hostStatus, err := sanlock.GetHostStatus(d.lockspace, hostID) + if err != nil { + panic(fmt.Errorf("get Sanlock host status: %s", err)) + } + + switch hostStatus { + case sanlock.HostStatusLive: + delete(node.Labels, fmt.Sprintf("virtink.smartx.com/dead-lockspace-%s", d.lockspace)) + if err := d.Client.Update(ctx, node); err != nil { + return fmt.Errorf("update Node labels: %s", err) + } + d.stopFreeStateDetector(node.Name) + ctrl.LoggerFrom(ctx).Info("node is alive", "Node", node.Name) + case sanlock.HostStatusDead: + node.Labels[fmt.Sprintf("virtink.smartx.com/dead-lockspace-%s", d.lockspace)] = "" + if err := d.Client.Update(ctx, node); err != nil { + return fmt.Errorf("update Node labels: %s", err) + } + ctrl.LoggerFrom(ctx).Info("node is dead", "Node", node.Name) + + if err := d.updateVMOnDeadNode(ctx, node.Name); err != nil { + return fmt.Errorf("update VM on dead Node: %s", err) + } + case sanlock.HostStatusFree: + if _, exist := node.Labels[fmt.Sprintf("virtink.smartx.com/dead-lockspace-%s", d.lockspace)]; !exist { + go d.startFreeStateDetector(ctx, node) + } + case sanlock.HostStatusFail: + node.Labels[fmt.Sprintf("virtink.smartx.com/dead-lockspace-%s", d.lockspace)] = "" + if err := d.Client.Update(ctx, node); err != nil { + return fmt.Errorf("update Node labels: %s", err) + } + ctrl.LoggerFrom(ctx).Info("node is failed", "Node", node.Name) + + return reconcileError{ctrl.Result{RequeueAfter: d.ioTimeout}} + default: + // ignore + } + + return nil +} + +func (d *Detector) updateVMOnDeadNode(ctx context.Context, node string) error { + var vmList virtv1alpha1.VirtualMachineList + vmSelector := client.MatchingFields{".status.nodeName": node} + if err := d.Client.List(ctx, &vmList, vmSelector); err != nil { + return fmt.Errorf("list VM: %s", err) + } + for _, vm := range vmList.Items { + if vm.Status.Phase != virtv1alpha1.VirtualMachineScheduling && vm.Status.Phase != virtv1alpha1.VirtualMachineScheduled && vm.Status.Phase != virtv1alpha1.VirtualMachineRunning { + continue + } + for _, l := range vm.Spec.Locks { + var lock virtv1alpha1.Lock + lockKey := types.NamespacedName{ + Namespace: vm.Namespace, + Name: l, + } + if err := d.Client.Get(ctx, lockKey, &lock); err != nil { + return fmt.Errorf("get Lock: %s", err) + } + if lock.Spec.LockspaceName == d.lockspace { + vm.Status.Phase = virtv1alpha1.VirtualMachineFailed + if vm.Spec.EnableHA { + vm.Status = virtv1alpha1.VirtualMachineStatus{ + Phase: virtv1alpha1.VirtualMachinePending, + } + } + if err := d.Client.Status().Update(ctx, &vm); err != nil { + return fmt.Errorf("update VM status: %s", err) + } + break + } + } + } + + return nil +} + +func (d *Detector) startFreeStateDetector(ctx context.Context, node *corev1.Node) { + if _, exist := d.freeStateDetector[node.Name]; !exist { + d.mutex.Lock() + stop := make(chan struct{}) + d.freeStateDetector[node.Name] = stop + d.mutex.Unlock() + + timeout := time.NewTicker(8*d.ioTimeout + sanlock.WatchdogFireTimeoutDefaultSeconds*time.Second) + defer timeout.Stop() + + select { + case <-stop: + // no-operation + case <-timeout.C: + nodeKey := types.NamespacedName{ + Name: node.Name, + } + if err := d.Client.Get(ctx, nodeKey, node); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to get Node in free state detector", "Node", node.Name) + } else { + node.Labels[fmt.Sprintf("virtink.smartx.com/dead-lockspace-%s", d.lockspace)] = "" + if err := d.Client.Update(ctx, node); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to update Node in free state detector", "Node", node.Name) + } + ctrl.LoggerFrom(ctx).Info("node is dead, detected by free state detector", "Node", node.Name) + + if err := d.updateVMOnDeadNode(ctx, node.Name); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to update VM on dead Node", "Node", node.Name) + } + } + } + + d.mutex.Lock() + delete(d.freeStateDetector, node.Name) + d.mutex.Unlock() + } +} + +func (d *Detector) stopFreeStateDetector(node string) { + if ch, exist := d.freeStateDetector[node]; exist { + close(ch) + } +} + +func (d *Detector) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &virtv1alpha1.VirtualMachine{}, ".status.nodeName", func(obj client.Object) []string { + vm := obj.(*virtv1alpha1.VirtualMachine) + return []string{vm.Status.NodeName} + }); err != nil { + return fmt.Errorf("index VM by Node name: %s", err) + } + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Node{}). + Complete(d) +} + +type LockReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + lockspace string +} + +func (r *LockReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var lock virtv1alpha1.Lock + if err := r.Get(ctx, req.NamespacedName, &lock); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if lock.Spec.LockspaceName != r.lockspace { + return ctrl.Result{}, nil + } + + status := lock.Status.DeepCopy() + if err := r.reconcile(ctx, &lock); err != nil { + reconcileErr := reconcileError{} + if errors.As(err, &reconcileErr) { + return reconcileErr.Result, nil + } + r.Recorder.Eventf(&lock, corev1.EventTypeWarning, "FailedReconcile", "Failed to reconcile Lock: %s", err) + return ctrl.Result{}, err + } + + if !reflect.DeepEqual(lock.Status, status) { + if err := r.Status().Update(ctx, &lock); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("update Lock status: %s", err) + } + } + + return ctrl.Result{}, nil +} + +func (r *LockReconciler) reconcile(ctx context.Context, lock *virtv1alpha1.Lock) error { + if lock.DeletionTimestamp != nil { + if err := r.cleanupLock(ctx, lock); err != nil { + return fmt.Errorf("cleanup Lock: %s", err) + } + return nil + } + + if !controllerutil.ContainsFinalizer(lock, LockProtectionFinalizer) { + controllerutil.AddFinalizer(lock, LockProtectionFinalizer) + return r.Client.Update(ctx, lock) + } + + if !lock.Status.Ready { + if err := r.reconcileNotReadyLock(ctx, lock); err != nil { + return fmt.Errorf("reconcile not ready Lock: %s", err) + } + lock.Status.Ready = true + } + + return nil +} + +func (r *LockReconciler) cleanupLock(ctx context.Context, lock *virtv1alpha1.Lock) error { + var vmList virtv1alpha1.VirtualMachineList + vmSelector := client.MatchingFields{".spec.locks": lock.Name} + if err := r.Client.List(ctx, &vmList, vmSelector); err != nil { + return fmt.Errorf("list VM: %s", err) + } + isVMPodFound := false + for _, vm := range vmList.Items { + pod, err := r.getVMPodOrNil(ctx, &vm) + if err != nil { + return fmt.Errorf("get VM Pod: %s", err) + } + if pod != nil { + isVMPodFound = true + // TODO: shutdown the VMM + if err := r.Client.Delete(ctx, pod); err != nil { + return fmt.Errorf("delete VM Pod: %s", err) + } + } + } + if isVMPodFound { + return reconcileError{ctrl.Result{RequeueAfter: time.Second}} + } + + leaseFilePath := filepath.Join("/var/lib/sanlock", lock.Spec.LockspaceName, "leases") + _, err := sanlock.SearchResource(lock.Spec.LockspaceName, leaseFilePath, lock.Name) + if err != nil { + if err == sanlock.ENOENT { + // no-operation + } else { + return fmt.Errorf("search Sanlock resource: %s", err) + } + } else { + if err := sanlock.DeleteResource(lock.Spec.LockspaceName, leaseFilePath, lock.Name); err != nil { + return fmt.Errorf("delete Sanlock resource: %s", err) + } + } + + if controllerutil.ContainsFinalizer(lock, LockProtectionFinalizer) { + controllerutil.RemoveFinalizer(lock, LockProtectionFinalizer) + if err := r.Client.Update(ctx, lock); err != nil { + return fmt.Errorf("update Lock finalizer: %s", err) + } + } + + return nil +} + +func (r *LockReconciler) getVMPodOrNil(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*corev1.Pod, error) { + var pod corev1.Pod + podKey := types.NamespacedName{ + Namespace: vm.Namespace, + Name: vm.Status.VMPodName, + } + if err := r.Client.Get(ctx, podKey, &pod); err != nil { + return nil, client.IgnoreNotFound(err) + } + + if !metav1.IsControlledBy(&pod, vm) { + return nil, fmt.Errorf("pod %q is not controlled by VM %q", namespacedName(&pod), namespacedName(vm)) + } + return &pod, nil +} + +func (r *LockReconciler) reconcileNotReadyLock(ctx context.Context, lock *virtv1alpha1.Lock) error { + leaseFilePath := filepath.Join("/var/lib/sanlock", lock.Spec.LockspaceName, "leases") + offset, err := sanlock.SearchResource(lock.Spec.LockspaceName, leaseFilePath, lock.Name) + if err != nil { + if err == sanlock.ENOENT { + offset, err = sanlock.CreateResource(lock.Spec.LockspaceName, leaseFilePath, lock.Name) + if err != nil { + return fmt.Errorf("create Sanlock resource: %s", err) + } + } else { + return fmt.Errorf("search Sanlock resource: %s", err) + } + } + lock.Status.Offset = offset + return nil +} + +func (r *LockReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &virtv1alpha1.VirtualMachine{}, ".spec.locks", func(obj client.Object) []string { + vm := obj.(*virtv1alpha1.VirtualMachine) + return vm.Spec.Locks + }); err != nil { + return fmt.Errorf("index VM by Lock: %s", err) + } + + return ctrl.NewControllerManagedBy(mgr). + For(&virtv1alpha1.Lock{}). + Complete(r) +} + +func namespacedName(obj metav1.ObjectMetaAccessor) types.NamespacedName { + meta := obj.GetObjectMeta() + return types.NamespacedName{ + Namespace: meta.GetNamespace(), + Name: meta.GetName(), + } +} + +type reconcileError struct { + ctrl.Result +} + +func (rerr reconcileError) Error() string { + return fmt.Sprintf("requeue: %v, requeueAfter: %s", rerr.Requeue, rerr.RequeueAfter) +} diff --git a/cmd/lockspace-initializer/main.go b/cmd/lockspace-initializer/main.go new file mode 100644 index 0000000..36eed3b --- /dev/null +++ b/cmd/lockspace-initializer/main.go @@ -0,0 +1,39 @@ +package main + +import ( + "log" + "os" + "os/exec" + "path/filepath" + + "github.com/namsral/flag" + + "github.com/smartxworks/virtink/pkg/sanlock" +) + +func main() { + var lockspaceName string + var ioTimeoutSeconds int + flag.StringVar(&lockspaceName, "lockspace-name", "", "") + flag.IntVar(&ioTimeoutSeconds, "io-timeout-seconds", 10, "") + flag.Parse() + + leaseFilePath := filepath.Join("/var/lib/sanlock", lockspaceName, "leases") + if _, err := os.Stat(leaseFilePath); err != nil { + if os.IsNotExist(err) { + if _, err := exec.Command("touch", leaseFilePath).CombinedOutput(); err != nil { + log.Fatalf("create lease file: %s", err) + } + } else { + log.Fatalf("check lease file status: %s", err) + } + } + + if err := sanlock.WriteLockspaceWithIOTimeout(lockspaceName, leaseFilePath, uint32(ioTimeoutSeconds)); err != nil { + log.Fatalf("create Sanlock Lockspace: %s", err) + } + + if err := sanlock.FormatRIndex(lockspaceName, leaseFilePath); err != nil { + log.Fatalf("format Sanlock RIndex: %s", err) + } +} diff --git a/cmd/virt-controller/main.go b/cmd/virt-controller/main.go index 5615b78..c34d0d1 100644 --- a/cmd/virt-controller/main.go +++ b/cmd/virt-controller/main.go @@ -84,6 +84,18 @@ func main() { os.Exit(1) } + if err = (&controller.LockspaceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("lockspace-controller"), + DetectorImageName: os.Getenv("DETECTOR_IMAGE"), + AttacherImageName: os.Getenv("ATTACHER_IMAGE"), + InitializerImageName: os.Getenv("INITIALIZER_IMAGE"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Lockspace") + os.Exit(1) + } + mgr.GetWebhookServer().Register("/mutate-v1alpha1-virtualmachine", &webhook.Admission{Handler: &controller.VMMutator{}}) mgr.GetWebhookServer().Register("/validate-v1alpha1-virtualmachine", &webhook.Admission{Handler: &controller.VMValidator{}}) mgr.GetWebhookServer().Register("/validate-v1alpha1-virtualmachinemigration", &webhook.Admission{Handler: &controller.VMMValidator{Client: mgr.GetClient()}}) diff --git a/cmd/virt-prerunner/main.go b/cmd/virt-prerunner/main.go index d11248c..ee48b57 100644 --- a/cmd/virt-prerunner/main.go +++ b/cmd/virt-prerunner/main.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strings" + "syscall" "text/template" "github.com/docker/libnetwork/resolvconf" @@ -26,6 +27,7 @@ import ( virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" "github.com/smartxworks/virtink/pkg/cloudhypervisor" "github.com/smartxworks/virtink/pkg/cpuset" + "github.com/smartxworks/virtink/pkg/sanlock" ) func main() { @@ -45,26 +47,35 @@ func main() { log.Fatalf("Failed to unmarshal VM: %s", err) } + if len(vm.Spec.Locks) > 0 { + resources := os.Getenv("LOCKSPACE_RESOURCE") + if err := sanlock.AcquireResourceLease(strings.Split(resources, " "), vm.Name); err != nil { + log.Fatalf("Failed to acquire lock: %s", err) + } + } + vmConfig, err := buildVMConfig(context.Background(), &vm) if err != nil { log.Fatalf("Failed to build VM config: %s", err) } - if receiveMigration { - return - } + if !receiveMigration { + vmConfigFile, err := os.Create("/var/run/virtink/vm-config.json") + if err != nil { + log.Fatalf("Failed to create VM config file: %s", err) + } - vmConfigFile, err := os.Create("/var/run/virtink/vm-config.json") - if err != nil { - log.Fatalf("Failed to create VM config file: %s", err) - } + if err := json.NewEncoder(vmConfigFile).Encode(vmConfig); err != nil { + log.Fatalf("Failed to write VM config to file: %s", err) + } + vmConfigFile.Close() - if err := json.NewEncoder(vmConfigFile).Encode(vmConfig); err != nil { - log.Fatalf("Failed to write VM config to file: %s", err) + log.Println("Succeeded to setup") } - vmConfigFile.Close() - log.Println("Succeeded to setup") + if err := syscall.Exec("/usr/bin/cloud-hypervisor", []string{"cloud-hypervisor", "--api-socket", "/var/run/virtink/ch.sock"}, nil); err != nil { + log.Fatalf("failed to start cloud-hypervisor: %s", err) + } } func buildVMConfig(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*cloudhypervisor.VmConfig, error) { diff --git a/deploy/crd/virt.virtink.smartx.com_locks.yaml b/deploy/crd/virt.virtink.smartx.com_locks.yaml new file mode 100644 index 0000000..3771d95 --- /dev/null +++ b/deploy/crd/virt.virtink.smartx.com_locks.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: locks.virt.virtink.smartx.com +spec: + group: virt.virtink.smartx.com + names: + kind: Lock + listKind: LockList + plural: locks + singular: lock + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.lockspaceName + name: Lockspace + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + lockspaceName: + type: string + required: + - lockspaceName + type: object + status: + properties: + offset: + format: int64 + type: integer + ready: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/crd/virt.virtink.smartx.com_lockspaces.yaml b/deploy/crd/virt.virtink.smartx.com_lockspaces.yaml new file mode 100644 index 0000000..5f9f695 --- /dev/null +++ b/deploy/crd/virt.virtink.smartx.com_lockspaces.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: lockspaces.virt.virtink.smartx.com +spec: + group: virt.virtink.smartx.com + names: + kind: Lockspace + listKind: LockspaceList + plural: lockspaces + singular: lockspace + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.storageClassName + name: StorageClass + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + ioTimeoutSeconds: + default: 10 + format: int32 + maximum: 15 + minimum: 5 + type: integer + maxLocks: + default: 1000 + description: The maximum number of Lock that can be held in a Lockspace. + maximum: 16384 + minimum: 1 + type: integer + storageClassName: + type: string + volumeMode: + default: Filesystem + description: PersistentVolumeMode describes how a volume is intended + to be consumed, either Block or Filesystem. + type: string + required: + - storageClassName + type: object + status: + properties: + ready: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml index 9d1117f..453ca42 100644 --- a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml +++ b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml @@ -861,6 +861,8 @@ spec: type: array type: object type: object + enableHA: + type: boolean instance: properties: cpu: @@ -1093,6 +1095,10 @@ spec: format: int32 type: integer type: object + locks: + items: + type: string + type: array networks: items: properties: diff --git a/deploy/kustomization.yaml b/deploy/kustomization.yaml index c9775d4..ae0e609 100644 --- a/deploy/kustomization.yaml +++ b/deploy/kustomization.yaml @@ -1,6 +1,10 @@ resources: - crd/virt.virtink.smartx.com_virtualmachines.yaml - crd/virt.virtink.smartx.com_virtualmachinemigrations.yaml + - crd/virt.virtink.smartx.com_locks.yaml + - crd/virt.virtink.smartx.com_lockspaces.yaml - namespace.yaml - virt-controller - virt-daemon + - lockspace-attacher + - lockspace-detector diff --git a/deploy/lockspace-attacher/kustomization.yaml b/deploy/lockspace-attacher/kustomization.yaml new file mode 100644 index 0000000..5059ee0 --- /dev/null +++ b/deploy/lockspace-attacher/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - rolebinding.yaml + - role.yaml + - sa.yaml diff --git a/deploy/lockspace-attacher/role.yaml b/deploy/lockspace-attacher/role.yaml new file mode 100644 index 0000000..7e9764f --- /dev/null +++ b/deploy/lockspace-attacher/role.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: lockspace-attacher +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get diff --git a/deploy/lockspace-attacher/rolebinding.yaml b/deploy/lockspace-attacher/rolebinding.yaml new file mode 100644 index 0000000..85fb628 --- /dev/null +++ b/deploy/lockspace-attacher/rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: lockspace-attacher +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: lockspace-attacher +subjects: + - kind: ServiceAccount + name: lockspace-attacher + namespace: virtink-system diff --git a/deploy/lockspace-attacher/sa.yaml b/deploy/lockspace-attacher/sa.yaml new file mode 100644 index 0000000..d2808e7 --- /dev/null +++ b/deploy/lockspace-attacher/sa.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: lockspace-attacher + namespace: virtink-system diff --git a/deploy/lockspace-detector/kustomization.yaml b/deploy/lockspace-detector/kustomization.yaml new file mode 100644 index 0000000..5059ee0 --- /dev/null +++ b/deploy/lockspace-detector/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - rolebinding.yaml + - role.yaml + - sa.yaml diff --git a/deploy/lockspace-detector/role.yaml b/deploy/lockspace-detector/role.yaml new file mode 100644 index 0000000..29d0138 --- /dev/null +++ b/deploy/lockspace-detector/role.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: lockspace-detector +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - virt.virtink.smartx.com + resources: + - locks + verbs: + - get + - list + - update + - watch +- apiGroups: + - virt.virtink.smartx.com + resources: + - locks/status + verbs: + - get + - update +- apiGroups: + - virt.virtink.smartx.com + resources: + - lockspaces + verbs: + - get + - list + - watch +- apiGroups: + - virt.virtink.smartx.com + resources: + - virtualmachines + verbs: + - get + - list + - watch +- apiGroups: + - virt.virtink.smartx.com + resources: + - virtualmachines/status + verbs: + - get + - update diff --git a/deploy/lockspace-detector/rolebinding.yaml b/deploy/lockspace-detector/rolebinding.yaml new file mode 100644 index 0000000..6f03bb6 --- /dev/null +++ b/deploy/lockspace-detector/rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: lockspace-detector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: lockspace-detector +subjects: + - kind: ServiceAccount + name: lockspace-detector + namespace: virtink-system diff --git a/deploy/lockspace-detector/sa.yaml b/deploy/lockspace-detector/sa.yaml new file mode 100644 index 0000000..a03b00f --- /dev/null +++ b/deploy/lockspace-detector/sa.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: lockspace-detector + namespace: virtink-system diff --git a/deploy/virt-controller/role.yaml b/deploy/virt-controller/role.yaml index a776f1a..6110821 100644 --- a/deploy/virt-controller/role.yaml +++ b/deploy/virt-controller/role.yaml @@ -30,6 +30,8 @@ rules: resources: - persistentvolumeclaims verbs: + - create + - delete - get - list - watch @@ -45,6 +47,17 @@ rules: - patch - update - watch +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - cdi.kubevirt.io resources: @@ -73,6 +86,32 @@ rules: - get - list - watch +- apiGroups: + - virt.virtink.smartx.com + resources: + - locks + verbs: + - delete + - get + - list + - update + - watch +- apiGroups: + - virt.virtink.smartx.com + resources: + - lockspaces + verbs: + - get + - list + - update + - watch +- apiGroups: + - virt.virtink.smartx.com + resources: + - lockspaces/status + verbs: + - get + - update - apiGroups: - virt.virtink.smartx.com resources: diff --git a/docs/images/vm_ha_architecture.jpg b/docs/images/vm_ha_architecture.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6dec486a049dcc812f495d8f5041eac8d41603ae GIT binary patch literal 85474 zcmeFZ1y~*1mM*$*hd_Wp(BK+^2VFpd2X_zd?rs4Rf)gMRg1d&`7Tkg?+}+(>Z?R9G zdv@4;&)$7s_j}#nJLD@?s8uyG6r%&LK;q5h8N6i zY-}XdTzoHCc%QSfvD`NTg@lBJih_!VhK9%TjPM!DfB5IF5rl~d-4D|U14Rac#)N{w zgt}`5kpTUKgZk|U^lv{<&@ixY@Cb-V$SA-C<&QwnP%tpiurP3Nu&}_@p1}7YSWGyq zXUu}|kL7d`$ZWA$d?Qm4$%RUqaO4N}DOmOF{E(1wpWxv?rKF;!p`~ME=iq$F#r^8F zu!yLbxP*eDlCp}bn!3J$p^>qPshPclqm#3XtDFD3fWV;lAA+NzKgGnxeU48^OV7y6 z%FfBn`(9dBUQt<9UDMps+ScCD+4Xa9Xn16FYL zUM?sQ%-_TUe*aCd2XbKoazVqw!oVWj%LN7P3>+|+uyD_q;jske5Oi%HldS75`+o^1xy|cCI}2VyQE3; zLH^hA|F1C!7O$&R#H=lawy9cY-W1KrHDl>MeJ9_*EJAvT8gd7c`RM~6`Kjoc6GR?y z>4+_JTB{ryen*wEpi)0S= ztygsl4?gmRG{RXI9=OQ#SQgg&ACE9m!%Jrl6c81rwDVSv3I>J${t>AeZ-blhk@ysG zI&bd#!tsws{7Zy?&BDJF>0jpjdw22wtw&-fB@OY9Fv*!$(r}3(uzLq0kqZVz1=FNJ z|Jjkw@Ou4=3tfkE7UI&$4V{E8(^?M9@}!;TA@hw1G@8Ulfy36vOxa#6Tx7SW;ctfd zXR;Z4g2`>v3^Z?=(a}VU??9ci z>R7Q+><|0Z;Fb_SZy1%7`tlmRinA9#yN5+d^}uv%hAxbSCk|0i6e621n{nytpWu_H zV4oNhQw=GLGhFvZLC5|SfuCiuhe-2(f~r5g;#*HP_jkV;IfB=yBs`K%+MOJRv9-k} zJw76Q?#;c5HHvXAWMy<1-NUc{q=O4xfa@AgE5VvmB}Ol7fGQaFh^ki4E}MEqM@B(_ zAlW3S^XPj0I?=dinqz5gU$TjjfjQ-{P7kRrbKfk_ zbFmw`x`tm4yXwBv!^pX!yy65teo~BS_<~ih~ zJI|J3lB$Z-9y^5^z>Ow9-3{+%Zf=*oAlY$*X!Ds%(3?;VvyVklI9;ePu==h9gC?HD zwl8fjE|N3jAD)HI#i@%ITUsuwu2u(fS_SVyfqjddcwF@R%eb)E)zEris|XX5T(e$N zu$ujFsBuf?yKZvCXDmH;o#xkcNI_51_)K~S(qgkcs9k&e@?>q3u5h)`(mYGtBTAIb zj9)7XmH8O{JEP^GWG^O_u1&t>Gi3*IurmU}WLm=qtf|(>G`h6U<1~cOvREcCIl+nv zlu?{2B7`i7{^z4cF@He!w&$;nHZCzL4Wb=rIn0M$24tOt>LJKRXd_#}qUvuu8@7 zqfb%n*31#qpOIctBOJi6+Zu*uxQq%v|2&&=SiKQBS?v6+)Fu3`}k(e6ojyDy1EF3_0uT4+XK$B z1!3JBYZk>^%caS-)WRt;vob#9tpNw?T&8gsp@JZTt&fF{Qm8)6ZiEd1@kB4k0VTJC zBNMI;jj%-gal_}(Bt_pa{&G7m{m+lF-!DiHD)}}Y7dfkzek~VcJRS|3=`Cn*v~3rK zAwoJS^dvTSE$a*|6QU3-N!tzK_TNhY)eB?GLjC6-{hL}5B=G_io~9fT=xJZb-!B6}-uzX_bQ^g7_%c@3N*{|Te+Rafc9ZU~obZc3BmlIv zV?9X8pF@@=zlyUp(qsjpZl`l;8U;kKh52)78Xsi#1Q~G$TZEB^eyySDYI{i-GE`F^ z@HXvBy+>pG#rrPQLF}zV{Z1RmLb1{$cjW158abifj4f=5>MpHf@%HUya1sznZtq`E zkx+qY)@aKtxt0&!#P*FkDJM+gU+@iSxa}@YXR4 zY>|pZX|GN0R5`rP2%F(ZE3)&-NEk})%?xoWA~87j-VnNusdz#;~_Hk9O_S!q6Gxc#&d@5$kHPTrAT z-2NzQFb=XXU+$=2K}JhQ6n*2S_(>L4VM@>;)rvGs34v71*XMe}rmWbtL4H>kjQ%TdrRl&8u|2N zf9W5BZrku@qZ8+1Y9tPDj=&E`kR@fuFj_92RkG{cWD1^8^ZhrY3(!;8LB{&sm} zQZdzan^BXXT5O{#gRfJEd2rI06Q@^L+rHen&FL}0z_VI3k9B{w7`d=s6E^dvz-nR+ zCnx>hx#g`uwxM2zw%Y3`QC(UOLSp-*&&OFBxELhyw*|3>`!0Pl>h&vGH&;)b&q{iq zw>1)oK=m=pe(Po1t6ZwM1KG?7=ZcVFiH%2GB}0vb$qYX=+rgdHt7dimOu`9v&t6e! zH6d{kM%xNAjcAOdS&)SGvas*!m1q$+9ny)N3qP^F60Vk5@D*Q{`&j}Ne*x=;{CPXd zv~IP%pW`S5)sZArjafvsM3})X(OzO8UEhwj<4stMj&jVDC3xcwlswO563iG+=@CPh z#}QAM3q$0lR7LWlov4LN^Tu;y=P4K5fCdo@tE6w@yzyvJ48D-G3U*dP3GBJ~`^Y|W z(nMoW68Q22y>mI1W@IA0d)eU1_Bg*FZwWOuC>1}2E<{(LLz-KTn{d$bq;exW%O~5w zn19h&PfmgvD!!AB=dE5cE&8AVw48_}7l_aa$Duu>DVSEbKe88|%?MRa68&A6IXLfU zug>Lgu}QF)-2qq)?T3hrLK_FsT$Rm$U#r0$MP-o)-EHfIm+*y~?VvBA?nhYJj@%A) z{(Smu?=6`_43pq@yJ8bzSZYZ6iMCAD&F{dhjy9}~dktX6@mpAlnezGxp782@LEoExY0|=s&(p=Y ziP&a>BQHpYM1({{3u{uKN*2f`MmTVy;;?sPdMmht6C@{$GpUCyCk(nja0pSg5vPj= zf1=Mxb1Q!8l>AK2g32XV<-U{v^873IDx&1oAEL2}*oYw)nY?i^9D_e``mWD;+rpED z^@avo;`hkX3fq-=IBp}mM)%c}Cx@O?Oy<`X!Ox|KoxAV2j7VO9e4t=pm&a5z??Arb zQ*yG^C&r?oe9b*1caCbd&wi8Y`urNw*MNgljO zlY)Hu(U@4&0 z<+-|qjX*qZ=GFv4;^k-JgEQQ}B-h!?*H^g*#5sXBtG2?D`(jlkYdXVA8ry9nL{oE4 z6)N&JOg9_yJzG`<3CXB=RJX}@lWfemG7mE&{gw+fYOaQ2pU5Aw?tUtQf&qD9hg%H> zGMpQQ>n=4EDt@|PM4b4berCbD0TU33jUcarFjSJ09@*XD18amfsfO_su##V1o%H^ ze^eF5RLNTI}DsKkKi6eLcy%(`)+hRbEo#_ z)SU#Oi|D-+<>oV@t>+SF^3K=bdf@T-E!JBp9~`e{!t}wk){&(lZbPwZUt#bXMK$-Q z!+81`Ki8DjM5GjrXrhv|2lxr#yqOTj1ED@XImr($J}o|XwLZTCDL-Cb7qw>li1Xt_ zD-7B{$v*tHoWzR6oq2mE#QLIU>1A%f4{~2l?>y2qA{%BY4QE+Q!`8QJ1g@7x*->@! z;TZU@gCNF~yk`2>R@cKb6Z7L|2F9>qu? zkPBoe5M;Ofi1rQIRT{-bM3gvwpE*17Y3bq4jO*p6X~ZAX{HOX8wRm$!j_tPcu7f4^ zgt{>Vrd`XIivg961h|@_iA&btH-lfC;hH7O3713c_-stuR4VH)y_FHV`a_K)pV~EY z?m_23?BKX|iAfh zOwPP~gh)Uojx49!!Ke>KS}+ah&yJjKrb$eR>oz?9b-)XQ>7ssx3?8`UwQHn15XDi8 z&M9*F9cVyO3+|yONRa7jt=GBLG*@i4G9 zR1yLQj8DBwr=&QVmOUm8Z+Bg`Hnc>WKfcfYHX0Bz7)r$_7~4L$Dm0`)H4yVw+z{@& zP?GIQraKAldJ@k8aek&rta?CF{;6ds;~gkM;pa!wC39!$!$L6P`qT2H$|4tkjVlV3 z$&z=Tu>;*1;#e`7B{24ciHS}r=rzAXXE2C|elVEM;tyOuS(w|Ny82lAkpOYon{ot` z=n<{}c3FlKGP(>t1ZN$*A8+b?v{mFiAAEKP3K{W6*f@2*17UbjpLVNS@rLsEnbZxV zt#p>qDr+8-p5YA3B7pBeNU?EuApV-QTf}1T^Ino)+ZYoR1_8_AuO|l{-sFJ&ua^Fs zj+5>AnsEH8SlX!8f@f6TdR#`jC0gLk7=k%%)3)HAymzx<<{b!9RDB2Xqo2EF6%A@w z=30bYA$Qz?GL*~v|241wP3QGrHu*>2WoDZ2HLK)pT!r)w3~Tltnael}vk06fJSGACI9Ee zzqT}HVcoj<@JnS=&ykaiCjWvdf%t#t+o6H(Kx5v45Yc^!y<8Zc`3c^Pg+jl)43q~W zuhTU+PNU*+@NJ01e3s)R|b$29qzgr$XW_5J8IQjtoPTBm(ONo33 z+zOLaXGvtQMKXS^dj}Gn!2qJgA@4&akOQ^~_!Nxi-S7PB4%EapcO4`8hf8p9dRvv9 zN;Ufyo&`J=Cp=byp}1B0@atV~yfMtwQx(TfT#9yZ{Mo;LO|1-;M;C~;uYGX`GA1RS z52Jzhh8LtP_VoZ;vwVtW`Z&(aN z^~(ukgZ{na7w}SCuCZ-C^ydyk$_@ZE{WU04E{4j%a~e*!a*V?M(yN@3IlU*J^d()U z=u-H83c`R~&KG_7D3+P`LZE9_5vs$3RnZ1MCqgf^+cGFh9PLF>zmZJSN3LdSkXRX* zexompO=@oi$^6NpSn;_UyY|}XQ>x8^As-dQSKg%>25Ms_k-?A%*x%5h)XR;iHntWg zCao}^z1Gc$zT%D08HaK7Ke-`LYIdHog>VZZA#L{l3XIV=ZkE_1mxT2BC>#tDTdbuP{XKqpipi!fj(V< zHx?UJ_+&4mVZLUwRgY{Fb`GTFer)Wr=I-Q;?^J|mjS@S7ki@8j9qKz$g+-pU1{!c^#L-S7mwB-SJByDtX(R6tf?BR zsk)@~kzB>nLUw|Z_T~i;RADT?sTstbcjP36%u#axTn7n|DrnTm(WXC9khHM9_!Kj~ zj^L5X1+b42wC^NWp^2cK;r`YrD6WU1274Pi@z(YB6X=qT{T~lOrW$X`2&vS2**0Z3P zK8rihj>BDzD}t)6XMZD?D%aK{;9M3AEIdX zw`f2jNd&nGaAdk+OsK?`D)AS`w`zR`^1?5`KtS^T_FTXL<0e1nP;u!NY_yXAT3^o@ zmnsqJ!n0aVCB4N@eV0+xA;7bO1i2rRH@~Pj>hKA7c3srs8QzVF z97j7Eg<K>U;5I;+jwdA3&665b6S4SZ}N`(1>fnnny92v{Z^2Ub)q5oqrd zHMo!#>%u2U;A_)w^Wj4x{NndIAeJVp;hswL?VbKf3KDHZ+zSx{JiBz8vktc4-&pC4*m}1FAr8^~IeYI zI!~T+IzLvSERpjh8&@(~(}0(R_}k5Yy?sgHnd}&J>j*?7@O7 z7jmUAUqfr7Uaq~&o7)dJOFrh>NTYxcL8I|{c!eD=536W)h>#J$=JoyGQM zqefMgYHA!9BGiVR|qd-pzb@-u-Ju z-g94=ZbdV4GoHVeK`USPYoFL@E-VXHmzocRA@@B$%NEUs)I8b>z>-$=dx??g&Hd*S zU7zrTjG^F37CnJerEJ9@)dpwC3%L;A(XzFG=cG|j@Kxlq4+_Blv_E-rVi^S{8x>=i z7tvZvkhi%_ke}(7iOKD`RB?8U1xtBNsq(jK z%zQ)G2)+qhUy7Bnq;#|J4dtzSR^(S*5PhJ28HFi>CH-=Msyz7OfnE;b|AmYCf&A_s zyZS{Rw^&33^Rf~FpXhxGdf#se8hu=?l%&Ez z=x#E6t!K7|`o?J~ow}%%GjEh8JuXY@T83nZem%$$NP+$^01wR(i1Ghaq<^=!f3zL} zA~+lf;FW%_QR%;`?{8_@pY{pl@*v@{al;MB`BAdysDo?H{AqYVEta0Q)2qwMnd2?_ zOHy0k077Gm-CtGtx8(BA^}&Z5nf}k`9<)wVX>okM@6sSV5KP_w2n{>_qa7}txJ&80t|+gU!7-}@AG|0HSf~md z($A=5Zg1T7)!40G7iAy+57&hufahf{sFR)oxQJ5mZLs=1dO>|14MFoBryIKi2{YOy z6g8mxAEdcjF&6J)NkLO08JitNc${%PgK9iW!H`U=YX)OYZh>)+UDGOu8TI@nYFIog z1|>bNJMCb6BbZgZ7x>4}{XCR^5WhP&@j3#VgUw$U6FbM1S}`A4&Lqkrq{?Zs-+{uI zRu3pg!&4?rqr1(mv6ppp@QAm1v>Js{(Oufbm#A?h)kb+m!J*fWfF-Lec$eS)iFeuQ*6HEqh#h5fa_aw0ts>@wdBWbe?10B_vJYjeMS#_}_N z4=Ky^&1jy=_ZGqzrs%dFiGBW?1=+VS0>qhs%_4wZ??99oBKHGw2f7U{1JbrD$OVlX z=Zzmw@p^2T>G1EPIlo7ERKQCOz^GE*H!^k`e2)no%mJf{bq5-!+X9A_9q{4zWfKr! z(d~LQ>b;Ev0B`A5TPa0ekXs4j(mT-Ge2g0c=lj*5*+V0F`v;nzhgdV8^gwVsk{)cH z4QSIdKk+h=BUcgsp2< z%HtnThmz&Qu+|$h-?6ZL46(6mB#iWP*-EN6fl&_LmwM^Ke8PlN`M!)u;&#VzYFf(D zoY#lc3x0VX6SAfqyJu9GIvOLV0;sxe+L?*S6~0<^k$KFhxefjFA%K1%`d*e9_fEL zWx#5c77U_Kt^**~pme|*?tuCckx3On&6~CBPkaQoO)7VwOOC<&wU4zmi1^7!=hi80 z!!eBJ6Te$hH8mIVSLM+n2iV1EAvA-pG9}hqN%56oVBPx>a^$q7=5-!xn(Cm|QR?a% zFqM1C?P5*h8y@zP!1*$Q;X)BS;!U zDJV|>@_X6mCW~MzY+Hlzp-wQb3x+=~Q^wdCrVDL0IR`&Ipz&u$wP45^zUCpfA_)qU zBLE(wYf0tf>D+M7i;_+AhPA#~@VAolc>z727}k;ra%S`ON6m#LUaY&4FO$K)mZes2%=|5pLe&s{)G0~Vkn56sX+J}_l}1+PZ{FEL#}O?i*2NCCHv-_LZr z&LIp?!t|{O5GnK9nBaf79FXwW(jYU$W1f-oWZPmKnQuQLV&Cpw)HTs_Ee*liE1W;R z6<-T%*v2ux1Es}&cT9dda|be!$zJC;JtQz@hPXOLV^wVi=Ly>JB>G?wCi=2O1=l_{ zrT;!#xX}Gow4=r;OG!Ez_p7}**IZyvfUWN_eD%P>9$v9uxNksa&;)O^5qo%9qUYy> zWNItt?Irwn*Isdee&q}SuU*}P?gn$$k)qcWvj9+@yaNF9aM9j}!p0}To}H)PcEuN& zkmodK*fVCI4A#BMWURAQ!_U%i-9auwWGaiwIEo<+bIEF10Ap@Ea-nkR;Vz#IE^Tw7 zsAAq#Q$)>wWcLWwz6<2AJ$Ei%2lyZGB*`rb6_6zcgBPd{g02BUz0Vo$IR~1{HvWYj z-Raqb?4Y7+IjYbsyd_IQZIn<+dwn+%mdK?lqjs;D@I{iq1jq8siqr)(vi5GxZ#BWI z@PaEJpeuU`NOohWk8`{qwhJT-U)o8IZtmHP@7A|s>08)WLFGaAgV<3ik$$2w66#0R zf3wqeKw#w7RK>UpJJ=F8X0nd#5D+3~G46w?n{9L20AXph?Mp za4l6%te`0MlV66xBNwvRa<*S;v)8Vj?`t($8%~jHfYFk=uhrnWuNqs_xq{R3UeCG@ zvw;ed>MJm?^g(X?3c$yfGXRV7>w2{gO+vV?l0@3Sv%}!gz`?WS9(Y7xG8qML-2we| z=?3elH3@#i;Ufmqj*47Bc#ma%MB|}1@&>mGn(YCy>8kZM-!mpl!Dm)QR@d+1W?qCz zTt%=>ktf893u}EStE8wIEZ;$C8>`v^eKg1nX({zR+)YHGzSQ-FkC=UfOY;jfw?COeKdpl^HF$ZQeFFq7wIWQXD#}2 zIv5jec@rI`uGo5s{O_enqSE3kBW?=r9?NQScv{{=pOK^K5^0YT=kf4iG` zwoJaNZg|L@tv+=IRv2B+e9b$Mu4$CFiB@6X+>RPT+e}7|kMuCMOhMz*8s%ybUmOn`dIXEK3+J#NC{1+`1REMvHZ7ST9(y%wBmyYns-asqe;PF9jGG$AfIOaEOYvWUY?d@Lv(BH_AQ(G=?pl`ya1G{-ERBF zBLBu3>B5wW&8OPwme5m~%R}nZZNT0EE`!Lqn8tAtfld2lE4QCCFkxd3d}iHkri;nw zqUQEIQz(*CC?8OT9MG!qHDSt|DD0U1C9~Ad^OXY+|CG`q05~rW}FrZP?`K8Hx zk{0dfLu@y6E>UDccAswGw{EH-$xVS#4=TwUNjA9V!qjB*ce z{7q!L?vlX)&9Q50H|jI?@M|n>?}I$>ulMcfUU%=It2uG2(=1U zL!E!V9~}drn+|14=4e6?#Zt*yg3r5*b36v2-(Sv$>tVn;<+Gx0<3TJUT zflEk|NGA5+e0U7VB6(PaJ#T~4Ze~v0I(25soQd2s3E2FeD2nP2rWcRfx|tV*Se5tU z;FdE7OPM*G)m%rK$z+>eT1*BE%N3NuI)>^i_w}oVEBN~=K|bO-eh=%Au1-4KAQrS( z(PMu1(y}dF^Pt^cX_QY=Xw2F=q1jV}{#KY7@eWG5ztMequd+XvM>8c;)4Z(yrpL1O@ zLM7${$w|YOGL5qW@?W94HI<`zu&WFccNySgezzDWNVR$4$tn~gG(dSk+~9PJKbjKN zxAmsMZ^lqz6&g7fhHlgCl)TWYqj;*@(-D@rE%qC+*~PSQBVBS@JPL#;1$ASn%Ty;*Nqr1Qd#ggGC}w0Oo$s}{}_z2!^1HEK$Y zkR{&LFLEn@(FLN=<2u}dupl0%b{wJ3I5E4tA0vJ)MMh{TMx}(^fs*JRFK$q9?wO~} zY6@oeQgE{5lQzBT>z7ttO7x`1Y7wjZC^s}GHBju}9`h9_+qYQS4)Na$(CXmY@nM!X zT>yq28F6o>xqpmv3o$-W)g;-;E zPu8ybwr(+h&fDB2VRX02;eu_8meC+|cF*eUQ}`zu<(G1O;#-p?I}S`d+ql^+Pnb13 z$Oz(#tcg+t>eZO3dtB!Aw}e96KG2--At$VV7wK%FYxD#53Mx|>o6pBwpL~RH2olM(Y@t32yjIp?y#or_Q~5-bYlMdoAe&2773c zOD+GS;bMn%tA$$M5cH_mFjEcrs7rXfm`Q16R+h%rM5bqCPhL=lbPn$DD1D@Y_jFc@cKYsAF zasnHJ0AnW%1tCbhaE4R1a>RnHzJft7f)GKz0P!WMf4?7Mk#TE9|Bs4jO*OM`ks#G_ zB8FP=#KiWWDF@RAn7tM*TIC%5i`*p<)7G9@N+{35Z71r`Vf68HqckhAqso8cx@}HK zF4BietlajRx{-(IQ$T+RkmZh&k#TF#U*vJ`J}ybYJ_ZV{Nw$cPlnt#Q7el=_%w& zfw`$;ToE9$W|8*jeSM#JUV_wVYKDu~*efaAO(|(qAx=l7QAemb55`9=la{uV_@87h{a@{$;7EbWS zuItth#mX?(L=(P;YC;$$yc|)kA7R+7Be&1jkg;{)A6q38L9OKo!Ad{(n~&hynvL?) zlM|Ig_^?e;{v%;t%4ihC-or^-z65JP2rY$`P2`j(V^jMmu|<@Q8v(=T>|sYu+ZxnA!1*eYn9 z$Bv?unp>K?=DaB>_|BN9dtGi_BTj)XN{a58|3s#)e*g4~YsOcy!_|EmRt+|r*kGG& zHA%4WmVXp;aJFqWM0b%$WJ^}cP`@5ioG?LBOvOf^AmnjUx}W<~E^?a{QOj&I+{w%k zmDunf_)tycu#ap65mI3>gUeNuu)mu#V!l$BG8_DPgqQuLC&G$P^)e~Lsn>E;UYv)F z_!3UBj`BHMu1b)UawqNajO|mBUbUbldQ%P#(wR;IvCDfM`C#F&88I++7a3f0KD*jB%kyr~^2Nqk1g=dFUvrc<@&z*6O+kV?5L%1obJ=CMnRSNr zo=X5u(ctAOmsR5JVJ|uLTj0Gx@FSoQ$FuxX%$2p;%B%**w!S@c zqRgzaN^SDtGFvF+@4@(QYQK~8%(V09TW_gbZ9I*2Qj(hqgL648x6L7PR!f$OH9Zdh zrYfEz9iqG{X~7|3JDjWMMhG*+USDK#8h4iPblqnCGPD4fk%F6V3MWkLa}lL9;>;8S z&qy~K_Gm@T4bvf|qFd6(#E@V!?oO5vJcmxMV)7bp%e6p#D#YOKB!O*1um`g3m2A0IQ85{ z*D#xpl^2H#_oC1HHc(cKbsp3v@8UO0pIcZYTqn*2nV3Hu&ZqL)71ytFlVf1htNt={ zi>!6sq-ZTkv{lod`PDCHQ*~a!x3hCYTVm^`WYLnNxew7pFiT?ti7~+}j*9uYR!k;& zH_tc-f!r73wpj;*)|;VTpIB%TyRVIjR#@JZsJS*8FnvrK>f6H=T2~~L$;4pW?7+8b6b`6y0kn&hxA#XHX zYM?pbtmFOVjo|v{-!GTm>^oKGsEe;mPi`FbqU<1;7g+qXapCnHt@tB3^N^PYlA&Mg z8nocBp_?YyYcI5 z>3$M-Ml2ytx2rl#Yh+=Y2oP}v82nda2VZAK^Ld>mEz`v5y0N{~9Mzymi^L$>Nh95O zT7;XNJP87unkjEpL1GYcK5&fdwD!v+dUs0^E$!y{pBrnYB?3+9BBVscNow}dk^cV5 zOLf%)b%IN1((dnK2@HZo-U*jA>>m0%xpchh;B(2|PL?}ltx2o9ooZKTV}Dm_j>|b` z8bhAchlh;kd=O)ixIf~RLUk=dwo5@}v9NUE@@ zsjx5GZ4ar+ik!#3kJkJHmQKskgtb+67-hJKc$TmJO!=z{Sq@hoU!Ibku;bN{YhjEa zh1NqElhJneDob7=mv=|4@qIhle9SwdrAO$m%F?P$PKb;Z4un^}hK`isa`+*sB@ykB zI`$28cD#ZN)+7WGV2$|YtBq@@BDIxit0oyG$$b9W*TLWl&Xy<@e)fC4B8vMZkp#$x z#$eoFa6!E;+`pfY=P!{oZ0G50`!LXZ6Uf2ky z@xu>=3XqMPjo>lack}NL0503|j*sj1i`;PR2BJnvjz+HnLCgMostHN?-Zz!-;hCbA zr?EcM-He@jky$(GP+}C(NxIXvNU3HJf=woeHM7E-k;JuN{nEYlNo=w?jcMxpw^kFC*W`lzKPC zWk~GlRbtvjDEaR5*8Oz+OJuL!@W>>8VLLbP;0Z^AZw+;WyRjzPs;fg86wd@>mF2}Z zI=R*mf<42E7kRcMwUp^mIKNdH>Y5eIy0vx$O-m2j_Aw9DKNku334na_$2Zz{AW>qP zVJ28kB%Iy0V7{gVpgu8&V!tkU>~?gttab`^RIkL{d?KLATx!1M8Eg*EhiP=g&W^z# zDSv-%^=#X$sTng4l|iecg0l-%UXLXN@&=snl+DZ05_jHWxDa)VCyzBMpBvhR2q`#tQQr*m_|dD~n;a;b!)kp~p&c>~7yhn#e?{&7#=-&M)<31U|He%JeM1CJugl)8@F(Euu|S1SX2h7J=GeV{ ztI1p;0mYM-YG}qP^Kt2PU@Uwka4Ex&3415Ybk^zik~gi!Zm0`l8(}hQ%3CDL2mEpA zT9k#urF=u#pGhlyS0kuwtMZ@)&e#tk+h!PaqkB`~};zm;=|i|3JpYZQ(`^t z0AaWfMCVICX!Q=9K0J?f%=7$~S)OiLeMIKyqCu(~5!#ZoqkemPJ$bry_ITGaHJ$j2 zG0slgqDC6T<|AS8&57m?tiuxZQNe@<(ZWT6WxQw)Z5B)cv&P=0}-fK1SaqetVpfu=re2}QV7jF`)kpd~v~ThFXwG#;5usrahIJj^?(+An_S z0v+t4S_dIa^P$Fy{kvkdC2v?HkyHW>UmDmTE)6TG*G2i)7q8WN3hk@MrSCHZ>Ir?! zcvVT7V`MbphiB1s!3DFsSZJ1n-G7pwYM)~|(91CsaQIWryJkvknlH@kYgQ%rn)0&k z>+nHlz}L|FMh572j19Jo8v#UST0HIQRJpEbh4rmXYp_#gqld4feY*}PKEYqp=~UM5 z&dm)!p=v7>ZBggQhpXdE@Q{ved1UbuFOWHa-Y=afKz+!emWa5^;&@ASr*Jq56G7fB zdegJlsA&leYfFQhuYRc8jl$(C780VU<}#~g<5hO<8cwYmWRkL*emwK%iJ&mHvf^80`G+#wESy`R#C=_YcxaUHH{ZS@@mb; zDm?FD#$XZ*65#XA*)L7GKjEHM9+nLvy?wr4ixD3Y2)uWV56#W`n!Xs_8^R|8y!PWs z^0ia=xr#P0@JVjTx99*RutTDf>_Gfe3&PG%kie%sxk*q$Syh6|$vjQ#90NtoQ+gIA zZLb7gz!{#C&_dQyrWzPSYK7Hg6F9)PoN}{GG{)I%*e|N%x&!f^fUgMwTzwL}pygtJ z>$>TE{<03h>dV}b&M*w0uOaAXcz)CvE^hV7Tx`zA)3i7o4y!*(7%qA!v*?2!XD%3A ze(1N(?HAlcugN6HumwZj{vCZQ1gChX`B3)Q`U*}5c;U&E z1W;}T4EyiPc7Uy*nm5Tn-HtKvG8mFej$ObYw}E9Wbqjoo{0(5hD;=P$TQ5CoD&L)i za-h17Ik7eoOY<8GrJt~`DRV~7O?24A69nQTwveOai){;bk`JpCD#K^a*TX0QAl*l5 z$$0Rnq`56)t@v&`hgQc{LVG!rN#s`vkG0QcZd&J~^P>(~mF6)%Q<7-iEE$oz(=Qsa z5V^MP6Mphz4KrCdIu1WC447FNc9yM^P$M@$@e$m7*-^DNS#SJcWFOA|cSWk2_&j+n z8P=iWLe8w~C;7$iU*&3)NIdeIyE zyhmAy9Ir*+&uy&mFWCFQHuba@U?!~J2g>n_uV;jaSVkvwO|CyuOdB%1Q2YAn5pyEc zR#y%qn>VKVxQ5Awt0_@vE#1e?ZloRwyC+s0{=R;8E<4(QVgHHAi++o9U;^wVM$<+8 zbfbJqC25AC97(u`rD%yk0qyh8B{;jJ>U;!=(G!gszVg75;K_Df+lrB|38vVOGIn81 z@cX!mfq?0OzXB-dhtCCE%gv|SL|os@-i(RqSFGt}arV;4k3)E3VUn)b8%9&88 z_UyFaFB9u=hQ0sPVGdsA?=s!n(O>g3F|xkw^gu@4Dn&+mO( z$59i4yE?koZ6q@jh(;{%l8dXuV@#0^{CFcpW0L=Co<=rix2xUh5!^OmffeBX%MaHE ztkx~dfAAYiRAZMb;)!-~s`3av;ZJ8S)z$jz#64L)vQ;yg(7fN?aO>Pvc;r9HGS z(oIX5Z>)-vX7*>Oow;bWBnpgzjal|l3)0UUXShzI2V(k$SCZ+5@q+lI&`ko#?KH2r;XSJ~WksD>!ohmZ1#Cnp-(%iNzPD>>)iy2Qc&6t3ty zR5_xOSqUVi1<{C>eIoj>dV}Z|V%kAsTD0*4grxp5sU8`J099K*#N%Sw-*$!L+}BY} zTHlJKKM*V{=?M;OeQ({Edzcb!?WVmpMT99o+kj(t=Q6i}^3fFU&w#eNFvY;PP^6EL z*N%!bS_8^!b@0u(9Uc?$YjF^$E+0;K4Utt-e!`3bJ+r@1$@cuGLP-P9LjfD-9ck@i z74%@{YlL#m7!kn7S)=xnhqnE(2HFgiSgvi4)P?Qv*xg2@b3Sj z?W@D0YP)v_QBf)B4yBY-q-&&0x?4(Oq@}?TkZuqVB&2gF>F$*7?(Rky#U|wetlWd$Gyb90X97Fy}DBZhLZp% zyJPs)|0JZiE!sNd}lRqe~ z%2F>m@!<8qSY815yJa;nIG~0(u$$?)I2_SYnji_E5ns~jv703D&1YgsQW>$Nsu(5) z)3SR@1kADu(#I}Gyrv1&N%aU=hpfL~qY(=DT+RiNk#!XRYg1R}9bicRdoZ~|51Ye& zNpab*LWE7cGI9K%*2Z8biG|H4lFLd;ll@)@SjpRBJKs>NQoDrvaM@tZQ{b(fT)Idm zB=WiXR7%mB*@w%R!_{c*$xhk{633qvjpPX~kBoD5sG}qBBeAcJm>Tpcsld;+KYppr zC@ZWiEh<-NW^=@1EaF(U^d-#eX6&?pxH4_$Qs;3#f1is3M{>bLl&TG^M8(ZM4-_R7 zGgZ;@$B?x)=LP1g?F#LOIllx-sC%1pm(H+Hpc4~bi$N;6o9644R0zCB+Ru>v`z zu)oz|`we-r*HEqX`W36 zhs0Q&1`D0e4w5eYtcWtyzLLCC%=hpH?UJ@YY`L~Xji_6SYxOXW$RPTxh)kE*r>lP4 zpr!0i_GqfdS#5htF1R`x8f5Kox{ts~NG`ZmMnPb>3V?w*2P1rHua5_y84pytW_2Xr zTfnO|Ecqs)ISvml3+;~LPYJKgdu*U)P`iWQpicne$>cZ4bss?Dd;s72zA}KtnFqoI z#K&MbA#fa9)%ZnOcEGy`@IJ~_($!uw;vt|HZ(VbZUzzg)EIG)v&NKwZf~O5z8IG_0S1kifTj-s073OAzd`+D*V?j;0zqFnY_Xol8#FkG7(%Qq0YqoeiQk}E zICwq*^Dh^u9THdKf*)v~-9yqzR%K=M#^?}*=wkE}H&NzQ+RjJ0b_A&Idx53Ze?8L^ z-1?H=plX0At%2ecdRC8lP6zDH?F9kwI5&6BZdlRHH=)zp zLD4^zXMp~MQr-GDRTZ{liM)kAn_8|U=)%e`UrJK{kZ3V)_N-q1ph{IYXzjG6NzPpT zO>SSe+SPJ}u6$^hCfHZSIM@&THD7FSl!1iH;;npW~EgSXmu7MBvB7B(trM7AE83c;Ptz`*4m(xRd>)0VK0r6it>C$s57a6i#F0HJcVpKnJ5ZNs<$vBl4LQty& z3+oQCbx_7_ZU{|7v#_$m)0e>XSu_>LxOKI9a`uqr-=LoYnc~1CK0yETBmjPffTwX} z#7h#DA?lW;B0s)(&r&3%}^8os&_(=?Va7@ioJi`y}+>;^w}Lz0_Z{pi_)ix5(G(2>`RFMicm-LY){s zNu1&7PPKicsEKfRKf8)VDPD66W3^5gQRy^wq^%8&Y9sl?q}5-yrHwIRQ0t0#Ucx&I_w(C`WWolg(x z#?b~ZPM|BAzraVB*#iixRx-N3n1RYc zYGjvqws$3H^BX+d!0Btj*(d9V|7{er1juG58rq5bwTLbF>MLKN>g(!{ z1~Y2FAyZ*;GtSV_)}jnJ<+7aXE~|>3h<^U{_)+^9sR>a&OTGbd6+EOqr(d@3h^OHE z4WBq5qo+W$?Hc}09D4hMO4mI?vocQ3mxOzCTEc|yVEPnd~ z?T%540(KoN7(2lW-A1;emZCd9KH)AGtw^n9y2T%-9S9`bXXb~VYP|XQG5ap|KE|?v zE%!-UQPgg9+%M#&fgdJvfDHNi0g6a>u>S8z@@s41<|k?|T^DzFdUqF*@OJtppi8UL zkayvfOW>De4a*+!%43PNLl#WiD{2uN1!7z3Fhf?pW+G~&apX>})GepPxlit@zQ?`3 z2pmi-Kx@~gL+5AZ0FUa{@A*{n2NG4jQju@s)HP20@^%-anTnr138Z4c;I#$wj0ij% z93>N)|M?qaT*zxu@j%K7v6-~UP5IW5E91Zk*a$QlpYOeax9<9~{k6Fz0&N(9id`Q;q4|3z{5BL+Jj>{x1T%GM6`NlHt{7jVR=Oq6W z-peyU7f=#4u*1$4Cz)Jr5xs%?eo$%}p6s&bmGnmOVwJ9Z9-PyUhy`AAP5O)A3xbk@ zzn2#P5s@3ekMsyH&OF~PurzUo2{f1Qywq+{lsI#ri>Y%p!=BhA$TjK*CR#;A z6|jqp6)vt=xfK)?Qa9{whh6m%=|q!ah=<~~=$4eo5ughLFn)z19{1w_&Brv@tuIK! zLO(eo^*%r$4VquV&N4S@XO7~Dm&DI~hZ{1rCNXuSrhh)7(5@UMO7y-H=9Te%+J!tB z2z3-X>;J*;LHTP(>3=Led(f5dO(y)DkTS@VzRGOWFC_5JJsThL)S-Qh?8ryQGDl;z9sNlrcSO7Cq*7erS{^x_2J- zukhCa@Oh6Rfy5}BIc{Z$s~tU?fy{Oz0m}EzJq0L)G0LjQoLER1Nxeqar*x8us(wezlLi&XQt)bMM`d|f>esWfowbhiAj?Jvq%pcJrAZy8 zY0;)AV}p>iHMD24oW=Jne0%PyVQt4E=i7OWUkpPG&BOVm?>O7KpR}W&g8iVG%46F4&f=P!zNguHm(*d{i%SjjKd3CkE^m!9HgXHc-C&duE@Z|*C3 zJ!hxCwA}WB>jZE8EY3dvyG8J@u6|jp<3mFIsAW+}psVl^ zO~C2KZhY0$3a*mE3g0&8hCff0gP4IzUNC)SupB+Ckt5`PZA^JIHyQ`Svj7scR}LtW z)j;1DX#N^yjP-Aj(T3`%Ct?N$7e!;8(&->drjA_kAyc|*7^%Y0UR$3{Cei*mTCXiZUv zaD>~oy(6!p@I=LiB z6Z&m6q32P?0hZe`e2&>J9!)h(i2wlkHA`$)~+u^7wty7Z8ne>ypp@ychtsX)n^T2!q2mGR|Uo?GISyh#?hUV zVUWW}btsZvd%=+`NtuD!<+^sECTB8#h26GFv``j#hFNo+!Vg$LzP6EH3H0 zt0~Q#%KZA1%w&XYl0lX-Uss+&fysH$3jvThVuzjNvSh@qRL zNAs$rwIZV9O_87v#Y2aH7}__H4NO$qZ-Q=DTv(3 z82(F!rls$dGj#f1XVaxJM;%80c0Ry{FmQxy<;j^S9)f=&!l)t#ESR_EbB(PF1bCHG zOlHRH$^{x;4QTl%gC7KLbL3Zm@16gUEd$_O-(4+?YFvzqUxtd-*i(z7`l$Qq?>1PH#y8~p}dk(dFMnm3bOARycI<2u`wcTFzP z3_v<(#%?9MK4IR_xN7`5m^U#>i3JR=F2UgbV}K*7^zcenNh1^*@?o%_4|u7+o4GHF zVEppyV@750D|!4OAgCsmO`A+GNhO>*H|Z1C{rs2gOPKb23Cu)xf7xX;3HFY+ zlO^-3$UubUK(BHp1&7#uUcA9!8e&xA!NjSaFzm9!SmaOb=eF@52@9lC za;KO+%-Om}4(XKfE$K#6Jf0dCh`C!P-%}uxa)JA32(i%^<+RiP@;I}lP{T=FdA}{! z#C9ihhbAu)laHpEaci^TaK>3_jD7u!!p^bOB;_ejxn1uuOxElqy1Z;8Wo*0WOY83B zICWVCvUw~Bw98+05ZB|SQaFqt7(aJucsY_0K)6(f71njx$V$$oi)y){o5^NiO6?bq zqF8y{krO8n7bg&GmMKmp5k2srxv=pq?})4E)hvWnS5|R!O=NYle@gEx-@zb@j#g5vyimKUAIeCOl~}m zDE2zo;ts2+o1kl6ZI0BcNSXiB&#XJm#X0ebFKXgLJZtX+@E5PFNMBsmckoBnPCe=y zsUO++4LZdGf+xrM^!<6YPH#pX0@(svYH>?9bex zVOF`1lE%;;+t$QxMs4Eagh81z{)Q3IpydwIwq1}xuDL&{OLj%IuUh|J+$32!vAXkO zJb+DA?aC5UUPfD@s&&D`rSCc`o8GK@Ts>luruC(?ngH$TBn~f!C9*d) zQhPqYkvDpz&Fs6}Or5A(U`4g9=#WLnXw#3WxxT@Q1PYHFOdmP>0xO4O<$T;yn_m3T zBaXmwW)CE}MBSi`yMW{x-plP`Ruv9Ien$CXU9eSt;P|rdVV#G2w$__{@yu=rIGpIq z6mJJL(W8fL0@|Nk)KV2xcFKp`TeoXY$B(X@E!`B%Ojb6^9m+!U8t{$8Ld>2%XiTRf z66{6dFB(k3%e!#&InhJYH9gwoI5j7p>bfT|g!De>1>q4+B2PS~_L!?E`INNQ>rOWE_{Ob%Qb`8EcoaGhZ6UNrb9%qPaJ@7mK&8dJFcO`kKq)+!uzKDJ7Ii z9I$eBQF>SnEvL~=dYxKb0>4La2oHFm;HgOsy&X8twcwcm*pFZP8BrD?j}hjaZn;7- z7u-8fSs)t)>o#L&YS8`gO^yri-XgO{+xRwSWocJg7DTkKed7;2)$o#1CiTo*IK4|r za#3sJ-k|!nkLGB+OXq$cG`HPx$ZgN_kEKv@RF#Px}^82h6F%*Xo%_9hUQFDva)Z zT_Z#ZDV+sws>zp)IiWLCn0KXhr3+vI^>x4ib?GAPQTN#A9uw2DLvhkIr6Q~vKOL32 z5&Vbpf>iv)uQusTq4)y1M`TtD-2;ia*K}>y2{)7vU^osC4(e3?TON<>I?gWxPe$k2yJOfGLYVrZuEOc}#=XndcIb%OUU}(w|BJr)FOlN^BV7-Y z?!`?0cDjf9bDK|cqX0y?vLaC;wZ!J-OzcvpaF9(qRY+8>sI;f6f9orH8D_Z>2JiA% z5%U0ASt_cktfi!;FtfJ~Pp)_-q2$lKx@ZOl6D(UQQq$e=sqkevYKab2 z{B&mAC_oMx|R?l9&G4iW)CDGd~3_x)EmbdhG zPV?KWO$#-2Kz*wvpyl44gO4@G>~4}eEx@0*e`YfVJ2UEwk$k1hFf4F>MiCK_i5r%Dve^qrU9xyR zu4U$HoM(QPBQRx85KETa*z&bwt&kfzu3=y|crA2IKbK7DWAIa^rlz8{pVW#f9Q)(F zE{U~}K5yy;ARJQAu$!Qo-W50i7b&wK5UGJaH?FHt&nJK^VwFTwg-HAYlA0LFi!dDN z_(n&!Z2Bw*Q8C*97yL<#xpurJ&CO`(`6QIyu}4pbP6!ouLZH@7oOd}h6@HEUvhSZ9 zE%s6=84&t;rx`D$)6RthO@hFp~L? z86X^d*+?_1Cg0_`4UhBO^;%D}QyQ~29K-nZEPZAj!mw6yU!Yfz;&JhBec6W}m|d87 zmhv9`kSq6DMTzz8zMD6$IGaFZH(7A*c_}>#OOI|E&_x-^OO4)tHu1p`^cviPYm*p1 zsBB220VD!2^{ZfQm(XVI?GJre8^k>=(WeC2j05?nGI^O+XP@4(^0(t*pZ^eYf32RM zjuqdzf`QTW8#FT)TwVJku5&CULXOld+tNXfHrB~+hWppci?}69r=-R(({dhoHwQk- zz(P^bG^2+vh?Kl$m)d3Z%QGP$AUCGRa!8sh#XL6#Aca`?k^m2z~FC zt#0(YhPc>ij&zyYJUiI)Pxxe}%D_G=&NV)P5UQ$bN>kR9?wCZcE(4$-@6`V{w+~fu z%BvzIWuwS(d8(9~{`dzxulWzAaHNHfX*}b>9S@WuEG8?p5ehBK7p$7^uXjw4)$zTS6mUgGmL-~`W%X{k3^{J{q+spk(H|e_8a6vwd{mU>l+Q(j&CyoG*SKU|Z3y_v2H^>YLwjYBn9^iSqI@d1y=0jfR8~ zg2IQ@hO`FAPpVXX-RSn3g@BHdlmroZ7ehEc4fW$?eEDWlLq1i$k66RK9*eopj<>W} zc92Lk1<^izZwig!dwbB?_K>|>Qc052GAXHG{ll=GMf#Q5#zkCc4#c5|_o8 zs`Jszig@p*`WYgIamUULB|WDfO2x$$BP7j(%rVGeeeCTq-164iB5Ik|sCPse!yv5Z zkz%9JVu#Qlk`{KDlaEX5)|Be`3H9wfdTh@GutxUS7^-<;_jzGv&1=e4HEQC^VG5Eo zl7t}r*C&~TLk_FubF1)HI`)rRfH85%D?RrZh%%T1PR06#b$z8e{^Gj`C@t5>j}jbD zX;Si+>O6_nP>%8i*RLO{ODesHFa>AoUs3Q^^RP2cGd>@uX!h*Ub(Gfi)PNt-TVCOW(P&+L-E}7=TN= z=WW_GV+1L-&8|Ys@+ar5*Cl=_5mC;&zna4vRHg7?n@V6EMDV6P7&0fo$si<6t)zE4 zg>^M$vT#-xmGT~5HeQ3Dz5m&6&t!9M^Y}LiD3}5g@M?IbD(E6a`5(TCteTZdi;0p+ zgcwKl&4(Ak8Qy~d6FLZRFRx8#penj9Y#-xh&B~t*-+$1n{eLu&P5ifMM%?J~Hu}+e z91Z!fLQms36fWWxw^^u$87Zp+ce^|aUA(3#gfQ>As>Q@+n%3*}AKRWCun75-LbEkH ztuPWGmLeW;TDjRAq7Mk$Y&z-qFwK|klW@$5W4!nzu}MU#hYm=8Ov?6WODMDuuRXlzCoiX4;%Pk(#NP3e!))ew(n%fQV?mfe}*lsK!D*>F)TzD!dm zVuPn`L4_QGLTT68u%Z3sDo(PVr4Jqw7r#?d4+H-UUmL4m;ObxaGU3gkSmGBUFl{N* zg66($ND&NHJ5c?RJ~_SlO4F?DwD6ThX=NpY4Lx#D5ZWCVj~*>|UhPPY0d|M-VjMxV z=re2osYiL;M^C0Yn~u4*&c^`r>w#?E2i_f)ZDQe>Uh0`Vsg;Zv8)~EX9dum1+Zf-8 zZB8wxM-ks}*3?$&kHd$wwSR^vlq-H#_$3 znIRI5xa(|8?_`ujhS^CfXro6Vu_mZtk0rjTP%VYkWyXE25LdCN7gJJ<4!J~R1-t3k zWHjb8S6V!hOBW|io2-1!+L`d(k_d&0^kUdymbu2~QZ7^x(Pe72@;Ixw>WJE7+%38N z0C|WiIon?Ep>*!7lW$KP*ZeT-Z2&D1V5pu- zj}WOjq_YSQt)~QB2sd!wviJTR6UzPmuh#Au(6(Dlc-lZ)u=-^bAOftf1K88j`u``* zA8(f^{12pH7`IB`qOzR8@})I#ORp*N`58wHqzec;kXXwLIOnG}r=r3ZAPx1>ZUR{=0N|X zY6gI5_5o>z>vlM=cK`s=eSme3o%0f^M)H|Z|mCgJq}ZW0^R(y1BY$wPs7v?@Oj$`Ce2&%b=>5 z+t+P=c&{%@13;Fk0$?t4d(B4*);Gz6;BjcK@1@DfTSPN3d*l)@oEUHtJ4UDlFh<$M4of??ck(bVrXWP*V)4)q z9bMJR$CS(xyY%4yILDlz!k3PE*{mU~Qhmp~Cn}fIYa+rcLu;UyZKyo1o0@J3V9SyBQ%wsOz6bVokNv3csM;oWGT31-hq&a zt5cv_XlxZuSeLDNTE6VkiU(}uRN@N&f*bqJi*3I_@`FF~bVYtR{e0gZ;UE!` zBj^6z=xDSmj^3ECQ&VawC|+l%s2xZR0g?G*c1Y4$YvykdVJhM=9RMk~W}d5K0-v`! z;3~U5((9^Q)+@_~_w8{3Y~iny5I!(F#Q8OB2p4+u>-kLq{N5}pdbXGrO23k3Z2$rG z0AHbNu3a4fsHV1OKqCAh046X8P)&hyh?hDTKXRH=OwTmGZ1#rH?)aZ2gg#20jsTsp zHDfgy#tgh z^vMDQ@&o{}41o0Hw*(zJH7kiIMD*PI3WClBG;a~gt-#UUeHeb#<3MgL0mY3eFko|^ zOc-iV9fV}8Py499?*NB}%gq5ae!t}k0GK}SKre`a?iY)|J1a<)zPo^{>%3W+ifa)SjCd*g?{0MUw^14X=TT8W0k%KO z5$HZSphtt0RCS!&)Y^!X$yh1>Fr?c5W_~yCq)^{Nam;C6e~xK#y?+;NIKQAsm;ZKZ zLjyH*`l&kX^AW3u=h zSQ-bw5dL%ug&V}%sH<1EYAwGh9QSA7gvM^*s(rh;bGN>4c5*@OE|onDRLuT?3$W_Z zAts(cg&PIz*X*<5jKFQ2J^%UAM@!xBvl$+6@pDmL28`WYW<;&n7{T!7YS)Z)nML$YtzMFdmx=ARQj5#W z*@F7q#$-gFi%J4#S<%Mv%@P2E{G)6s0Ol$V0H_%OV37DpaD(bc3}AifUux5Vrz7Cl zt2K&Se-FKJu<^zf#0%5mxx0cpInY;^z2N^Weueosst5SFS)1jmE~10u8WSD*5gP^? zG|>uYp7-ymDDO+&jlg_kmmndZNp2?U-u7ihGp*8)?hIf=`HzDxI{S|zQ(0H^8KKv( zeo9!d3YWP#gFmUJk4(ZUiHllJZZ=Tr18ANFsFZFjzS~!e!ZJJ?G9&dd}_`-PNo$+XYgC0ZsXL5j7Isdf`rT$H~AdedWkUWl? z$N%;g-JY$OC413}&*$BW*ino)oj<0<^Z7!){8PKg0orBlKWi6(+e5jbBot19=nfIc z|8DK@U@7MY%_4EqUuRu&CkPB@bhykoP9|Rf&6Bz0k@mEK_u0)k-ng-fIDt=KjY^$J zH@aU$Ndm&;59jJ1dgR6d#DnUkh!5u^h~Iim7Em52ZTU7{%q1)aOPSU+3Y;gR%IEWy zbww00g5l)%fXS_Qyvy8`M{Zfn@W#$6V+?IMb|dU;d3e1Uv?I$-B?@ujM@ z#PGxD*yEN6T(UP7#NVfR@82blCF;RySBNZaQs>FmZBHFg(9&*phMr&BmN5(^na2SUH|+`Me%n=^xIVl*G|IiT>8y^ z^8n*-!YtOnGG9rwjl|kWtnf>?ByFuh&9^4#A?-T!6w?bJ#D_Uo#x&y8BOSaLJ2~!G z>?lnelXJ9YS_&bxWK<3x3mJz)_?%J>+R}}gL)pu+E#5Ozd1Q!Ti%F7rVQ^6{Q%IFA zG6`&6oC!wEvbI}o^h;XOPu0O;@ zEddWh#T_^q34?akMIH+lx)v;ggGc(#jyYSPs)q%_56iONxfMM@>FGziR5MF6S2CBE zvShmqBw7e-J9xwTuz_2Zl=Wy>?KyMcheYiCtKi=t-yucMSx}5M0}>x2u70XTgM9~= zz*m?ain=-e%ekv$%Y~u{6%N(%$cM)M97oZ2=W+Ut7V~pHN5<%#3)E-+jMMJS0>U<* z#04^{pdswsmic6iuw$n+dq$~zYPT?c>*xFDWyTQhQgr;)uBMYwE@oYk_pjW0HZhdI zA-sz@TZtq!qwFxFdxtxFY@f?Q48-{QcgLynxyf>PhezR}`i5z8DLsb8rn-l#l2k7} zNbXMW@?d-}<%GGS&vCuxok%^sekWs`SqYoR+jB^C$Q)#-LL<5Q%3 zZM2;C?JHj6E>cI1V;x4#=*dL>31#+&s`m`{w(0KhM!hrWdXiP2tVOncE%paR2k0=+O=zQCPf=|6kxu@2x8ei4qP`TfS8W9id`)) zJ#*nYrh>iObhrqCWtau$pX*yiA;X5{J5rp|2=7RIb0X0qW5tw{>ZmOpIqt(~7!>73 z(Jpfr7(BG0_WS08mqZ723bB3`%*9JrUjV1rs*p+>>22$7QLM{in~{@=k*kcLjUyXm z45ZKJ1gV|-h>0pd38w|E@RM!6&orh>KFypo3P;X*_b6}R1QfylYvWA>x}MIMMJZNh zX9jbU1yAE?hR~qgQ}6Y3p8=ryd$7{A9%Iw-;;s!DvV4rDSn#gp9*fFItUCUo&mng^ zDG0^oiR`zJCjgM<9K4DhKR9#Q4?lRAInTo3V+KQSgErV{N0B}j?&(Hfc@rH@!vomtd7YLZ${Yaav$xX&c zF^d^wamv@}+OIG$kUDx9zW(U9+|?Y?8fJ?bwmYqcbVD@C_!LnqwQSwG2#pHc+|jw3 zNUL@-_2||RLSHg_GLu#ToDVb=BALtd)5?klwvv{p(pr*+FV-g5G4F{X2$Nc^p-M0H ztFPdZG7GeO`pbq$Q&Q?#11t8=SDo`8$V&&u*hRj?`48Nmwtv)qphy0ARNH;IICl7g z73ggDR<+ge{0Bm0#tzYV1X%24DM@XfP*nbRfFVovM>yfX37Bm3CGz|iT+A3iqmTd0 zb`G>B0u<#x{vpSl1B&ZuIFQl0#JpngDatij#DoWkBbx9jF4WKs)xu-lZ1~^QFPOY? zhCvk16W2rsVekEYCb@r}F;0msHWTVf&KqXBtnmrvn3F%*uS+bjFe4$;Vces1cqbNa z{0MX+B+k8BO=3aokPRoQQn4*CX+1W7$j6d~p==l*(X7(UDTr>a5vziIMji$Q=arRFm;< z=X0=|`A+VavoWzByWek@jR1(An~F*@&;1aIQFz?iN?I{;v67iZ9i}cnFDwJe(u1qy zvh+5`Y+gsaRMwW*{@G&p2}Q66Qs7^}mfKy;|B=Mr>HX{}Hw^WCIL;n3Nmz5p8h1|k zxwK+wacSbDtKJR+lhESsd7Qp1CsR|^J&xO_i_%1l`sJl| zbkPN8@kPi?4GnD!k~|h=r|>t6In)tz`?Q54e9|uqt1E{Xd!iO}1_Y^yU{OB8FZo`O znh>_Rj$(d3Im6#d-Y~3K5*_i;^3xU))&v0y=O6@;gm`29{h7uUUQ<@ zxt!zhm85MBmyWk}xo?v2?q?9k1Y@rv=mwvl>~*5>dm&$LIiTJR_N zy@QLT+5M`*ceTTjW;`+GYR9eJj{>suO?lcfX_j%+*Ds;~ulHb&W@3USL+vhlmLtd+ zh4M&Qbx3s(C=Zy}=tEyVoqQak@L}a{gT#GdkhPsx|AlLR>iv?^Z44zjt=;!i=YdM! z(Z0)wsKOnP)QBH8_dMCJo3+@q3fOq%FU)D zCC4n6%66TM3oek|d4$4|0NY97O;b4PK~$>0huXp$vkoUT40 zZpDUaVk-OEntP^57zfC*fb7{cqmoXB;O#CkBBMNKeg)gGQYW@*wINF~sj0l+({L;< z%%tjgw<&i3%M?naBj=ojE}%pHjPaeRgPzX&=!LO83uJGid7Svg^x{E2IpYj1Q~BwM zRujvvz$~irczva?It}gn4Q+M-9`KjhW@nlK8>htV+XRudEsC?Arrp%AS~cBtN#hx7))7}Mzh_W6xM~d>fjUf}tlKhVG7XGBdnkIY<$sJ6Gg?`d_H9_0R|ZeLtE%(nTKb(CwO|AX2IT?x=3vR9&64-=N%K z@HRshbOQ*^tm-xC)~E7XQ?z_cT2?-gEwWGEc<5+y*Y4*SmJn4~7>Vn>O&)TWkq4!6 zkF_8J8$2FEy3*Ym#--5@rWq~2c2zJ%Q@mJy43TTmd{fo)8GpaSZ(v54nEjfyg%t?%v>Rdg))QdMNT@h(gp&*=UJDOZNHO-iRp zAbQ@WiEu3MQhP*mZ+U2{e4$Re2$y{o7(h9aPgQbonq>F5wh;Bq75w$QUMg?@Z=Y_bid6|-CCk5}fpA19$_lu^Q ziQyO6PL_)o-Ivc*^f{lwPJO*4nszR~&m+Yrr9Z1+D>ynHrq1|Qep2Khqo*-qAGpPj zHZF?pCrCYJy0_ZVT|w_n1-irPyfs*8uZImaIFFPbvF|oCv$=|5EiJAvFk+>}li*q- zIwf-x08qWrUprZnGCTDPmE=W>WO`x)CBF*1y-QoCFR5drW>G~c z!>dskoshYa=>}uL{-s(hkIXewq3`C{f6z^=mAJK96MdnGv7r~U8O`t#KDZYWZe?PX zttE5H9c!TFO=HQ_(UewW^kQ4g1%n}J49;(pb}rXaeuT5Zdnk8J zkP}CuU{2*~gKw*A;B}-c_m$v^={_O`XLw5@=2b^zHAY`highJZA3O#7}@K#wnh1tcramS@59ce(r2!` zm!l6|rOoWj-66`u_8u9gxn@`)Bg~d`RbtlFN2PbNN%tkWb2LK(PMA2*dAbn%yt2c) z%Tbz;Y~P7+s^6e!prPq@&~wV8KB=-v@&f%V(QGrt3HFv~n<*VZTXibj%ly1lZX&9? zw*2%D;S4zvWyD^30H@sPU^f-&7g`jJmtPx}bAr$_6BCbPepJ=MfQ4xezLasS6oTS# zWH0;%;hIIxn4Q^8ZA5g%@VM>8}AO`+vr+%9u14w?oXTR>Mb+dWXzj9Z1{)vMApML(eRs_(|fGvi6t4id$ z8OM#LO;wNn*0tkb#eto-z*2c*kxp`eWr>Ms+y|Ni6mYVhKXoC3w)6xr5VVY)@qA1m z4bz=a&jq@x<^DKqiC@lHwzLwATYZ0W*&`)mA{nY4aKSzl{F5_fsGFfvRk7wiskJ!F zIXrE-GUS(vf&*_O9}QMZmZH!djs>5-H!j0bD`A@ECZhYaJqiS_1yP6~%$Tnjyzp@% zieO7f)yYbJx+|(vK(nlA{nF$B<)uh5dgaegwyds5gRuV5&fg%lBKooz$VJiAFeib- zsV9{Us|ZcFL8x=vnq>fgk1fY6^$2?h8>$w8=irw74@h&Pg`F?HEK)hyk1 zJe`{u)MNAq`G>W=Hg2GPEO;>{kjPgZ(uA%nqdV$o8_FC-mWp0fM1)GGI8VB~Z*8jX zEdADgv7DbLS9jyI%2Omp;UPXz#0MfOj7)l`WAX4)V@}4$Qv&Ew0r$ZK?hlkMd#wS&YN3S}y4d#%2dKwJ0fNyflle()eo0Ne z-~OsmEDU_ZvrxDR65?~b(LqU1*6jcgcjv!kSG?Ur>EBhS{}aM-%@79!fPdkJUVUZ+ zALkRd!IJ#IhtDZ4AE*QGZ}wnaG2?@l6M!n$+gs2cs9Pvv|JnUGTE&|aic{2-bw*n$ z_gI%PkaIJswGFp?@tEZN9rRHC2!gt zaQB+hFt1KbRRNd4prySrHCK6pr@`N3IV^!z&Rdg}-<+&!Ob&MmQ8ZbI^P_9Sn7<`=9A$Xi2eAdNSd|tbP^5QXhR&mUqhr zbrqM{^-+c-Jo`YbRzcrctqx=vpR$y=r*_HlTK2s%ohj#Tb6MigjoQa07~>&J`~l6? zo$0f;^1=<_ODCH5Zl7&fUWbD$Ksz|u9*pMmzn%qPPISON+R7d zEjQU$Cgm<_g%M4EH$6WZ!)dlIUX>%k#WxUQ-gv%?-RY$R81@ghgNoAxcNULmZRMte zpw5+`(=F}Z7@C%kEagK>Iu4^iVUWc}mX4KecR5B9FDq7J&Bv$}p%^yjd>6&+ZU(Y| ztw}?({W;j+H?s`%KWMxq#Zs7j3OO>(8QLClaWKd5l^zE9iiw z6TL-kU=RV1#Vd?w_JHHdaKNLTKD_#Rg=&@%xhXzm9Ir=Xlcx@rEbxO=fQl!+dc{-S zSkUyVVJQa@=y}84s6I_8?-Mz>bLb%n%z~)^&iN~`Kqi7~HK?7QA$c>86>~vb(C1S3 zz;`0Q6!r2M&YSW|S63}BavH{&cKtrt3i7DwQ6*oRQnsMVg6{w<`QA08j1a*HI53<* z9=6a8z>_iM8Wh1bu0hAZd;0A!AZ4qi+NyO<(0BaAP3~eBK*Iu|?6GgkD{!tV28e<7ESKr>=|yf@}Yar29KDdO*T!K!CiSA+HfvtuzV&@MMQ3j6xPt8s&FQ8KxRonL zN*Q!P&j~sN?qUx_^Z4O+}X{ultP&I0c~^a|aD0)Ym|;bIm5 zZMjK%b$!48=CX%ym2?@HljUaD#24)qS}*6OSStH)Bz_Mg(Ab?>>LI4ksO;(X^cfaR ztB*|tz*gUB(BEanG}NiT%}MnqHnASRX0h7*9m5o#r^;4_U6!l+!&S9y>1Ip3t`GCW zv6r=G8$^=DaN!N}pKQWaM#ma$)`QcV7r!|z5hPgzC}g!51-up{k=$hET|OGEb^S|C z+P-F7rM$U#MX1-H2u> z-0og#>SU_kRKXW{&&*Gpq-bsv^9c5BpqbwdZZzfF+-6GZ4DK121khq<{;hL^#f#O= zp*HJ{9S2aAwAQ&$w?`0sVsdn+Bkyj_oNLueJoC} zvUHu=rWC3q;s_HLPeD(W#?15Y%ifn<3pn|c{s;O2-jJb-kmyzYlOC(CQ0_tY&`0Z4 zGG*T^7MZvg%4Pfpbz9U!-5!IbssREP6JPN9TY!;;6?~At_Ltk@8zWj>$r>#nsUfmZ zzreV^c7;Z)hPW?|V1qVkEc9IvvK7G5Lg5g=w74b1*^~#F-j?sw-yU!TGv--mlfmD} z_1{muNo-g)T2#d0pq#|>!V`vAd;2auWxrr3`Ln3#Rg8GeJ1f#3`AN!&9;kM9x6 z>E~*4szoatQv!0_(I>vzLq^fkL<3mNON>9|KY>spAZiHgJ`8AoBS8OR1YOeUyM+bF zyhVm%C3)m>E% zuhGu^5t{)dSc-N!PhbNXgf#Yvn}2KMf3f%0VO6c&+W10JB}BTBmhMiGRtf1)q`SKX z>5>+Zl191~jYxNQcO%^_e-rn+-!1r_z4tl4bI$jD*Y*Cvwbq);e7x9o#kw`$nv$scg#4g`eUAJEET=)Q68WHFhX=J%F-iXzi308v`3IaLu3(?^ToNAv|xrzcnRo|MQzx=ACq58V!qxzlH_HZKT=G4{< zk%jwIpM2~pz+r6IP4BtLE`yV-NBJDPANe*|jn4>KFE|^=nj^Z|c1;9{%AtBud-Z|D z1bIgDZYvuAtMdydAaJk7P4{WBkJHA4=ru#R49k1S@vF+Qh-&7ktv%H*99IKcqX!%0 z{8L_Ih+w?i$008aln>U_dLiCo1;r;wtW@wf?joT{SNdUw`J$Q79QpG+o~!C}Njmpu zDv8r{<$+`6FmmEwdMg14-K}-EKvOLhj#K%VL-0iTHp?=p_jxAld*FVXa<( zK*^iShN~3%R=?BurU}?qZ#KU-qpaiP;Vm{#r7AjE(1}4%OL4=1yh|FB4jBO=B^;vD zQR|x#p|!^V6Q=0M3fAbd^Vq)mZB@i@{Atr~gb+WD=c3)GIU0IJ1kQnhGz3Io`SGKs zQm*V%t2IiyoyXcM!{OBH^irqenE3?yqo-A_zP`EiGvk#KX{QPlY0QfZ{hA1c>e!le*Q1D~fsqWj|POv~dudJ{&Y;Ww6G=AO7 z4@+b7<(wOt1PHy7py(eVVrm-a5A<^?+nFH-AKsG%H$Enw+vh_d-A%r0kmcQlWHr*s zt2d9%^Az9go&!us#oU2hAJp%Toe0K*%D4Hd{0NbC1uT7EP+DwJ`@i_9r*AOT4e8 zAR{e^%pJG>1*D01-Or39n23dK;*cRbJ`iJN zMrtQB!BKYdeiSn&rBS(koac~mY$}?8zBhh15>~F9&3bI~meH`Xe=w5}3 zNo*1q z64=MzO}g^d`3@SO$+@lxM!EFmr37kFR0IwKHat*p+}n^Rfw-eNB)aihhPOJ&-N*+0 zo)!96ufOz#yI#*Xu-(7)dFKUuub*_8eGFQwsVQAkCTf%S)H#UR*z45tJBZ%>nA%|W z!dhc#x7FI5>|y@el3Yk#au6s=vAN_Owz5dZ>P|>zv?Z&gNc`0w6 z@@;5*+gw+MCM<5GOuv`X-~d-C**%wIMSms|#;qKCey)i-;g;>|s>l53d#2G*hVyM@ zpE*m_ltk3Vo(4K;Dz=3J>1J43lovzmHBffPs&9Q|dRib>Sxf{PxB{Q(K6x7JvOqsw;_I6m^b-0B)Xcan>6{?z|a^bmtlT&S|Z%8a` z{`+?H@K-JSc(Jk2k$B1`G6*pH))OWCc*f_7!MlTEkcpvOn|8hEtsIT1G)Iz^=U>c7 zIosAZRYwU4_uyg|2&83qmuEgc8kXd1;%HlKkCJEj5bt&@i1RW;Of~6i9g>&$ZbC>{ zu9#tVuh9WV0vXiX19JT~M9zyw0ZGuxa=cRAX_?y$Uge;t^d~iOb9#)7=ZQMrcGQK_ z)m%N6Z)b8RPY;wrMT_g}%Ze4iBeUYe6O5BDk+mPTNSrA-K3nC*U8k+#K6YcP8A)|e z95Lm`x;CDo$oX2K7+4*6rd;l@6irVYsqc$;e znhZ=G#a>HhhCqPMpk`_KKaUI7f$ruUZu= z)r1Lh&5BhzHYD#f`@Odi;(XB zPmyd2PwktWnjyiIeg%hTv^JH|8%&0H{`46Nj|iVnQE;HW{TK&YSv}3Pwj@l=Rx#$s zDFc^AKlOT68+nznu1)q(&0)V&a~n@uHf}7bg`aqoZJI@Tx75kRVs@=P=j>!{OB;5t z)SHw~-r34}B3uB@AwmpXJ5Wkfi){!-lOV&wvS3kH@y9NKg$~w#oohw2YN&f;<3SSEbRzwC676 z>J#Ajj_11E!tei)#6-JHJMs#$LqesN1>1yOiMJDY!Wrp{jcFR!w=YE;cWH zpjjU$xL~{JJBSGBuAaxaMVEEr7ll9|R5?3!8ma>D+esJk9mf)UeArT*h=ZB{p4%1c z;ymwzQ+jv#I&0&^QvAHv_sEoQ&e06X!$7Y@6AIMQ(;T_a;$c*L3T{3)eIbq8Xr3PY zNZUAlP=%=1Sht;4ce<3nPYqA}v*OJC;W8BT^{SqlI5GxJvZ-^E7i87+-UwkI5E;l8 z6P<_d&n2NX&&To-A%D(9f{IGUGR2E818Z3m&Q`p*t1-Sl$qJ=gBMmH`=Kb|oM7Zj9 zOe&MF@#5Q-^d3`ui@Dbr4r*A$=Cf{fR$mF5k4QVs{|<(<{gAfz9;%nK+4>dgsI9_ork4vxa$e5t)Vkh=aeSG+ z_#-*2N9dkiDjxW6HCS=Q0>6FBOHv)CeNKm|Dt-!%GcLebP_8{SZqq>8}o zN#SyOt~^#VFpUR4#?}2-DkMO2EW*5z2Q+ZJI^86v`XV_z;I*ej?j!v<7khQvJR|XWC;5 z^M2_rQ{vI&dL4P6%@v1KVeQD_q;aD$GTlnm)A!48{BqX4Mk_lmC<_L>Y}pPkC5H~j z4y95UJyjO1c^^Nb)%tQK-icbB1)s7|P4gOMz1>)kev5X8KG}z}tM^Wct{@If&~Y*f zcRxL^f*FLfvTsr{WMWXY>O)}LjA+8wM^x;^TiDQvj(kPE(&?u?8z;)UeQ`6`rPJO?1rx)4ZeRHfB4%UkjoHT?6a-oP0jDEU6vjb{(09P=dZd14Z+J%pHB!}lXT z{5w)Fng_G9L5c9w1kBb-lg(9`K4of?C}?E~9by?$*+RCuXd<;*-$5!kqIzeQF!=j4 z94&FIRSdp0_fyvfTVcwpx=R_QNX#@k5o-!zbkIH?OpO$u^2m&`;cWuuy|i98e*#53 zQkjQom>A`(Yj3;tTcEeE6KKU0Lap8amy|QJ$&XV3u8L+$k}QdgkJowcfnd7oOfNcL zYQwp%w!2x!IN~CkzkV*c)~B-0L4y@*iym@d=C6!M17ehD{}$E?6FqjS)EuU^elCCR z6H4G6FG_ekGj)}-z`B;Mj3FloDx|0)xGxkhF#<5ak@Z&@*&AEpW^oCU$~T;J^ufWn zao5)FCi&)@Y;U)Kb7sgRCfjo5a4_SZ(BZ1csyX@N%EkrH>?G-Gx}Celfb zb!l*q-D0QB)Fkpk&1%Ye^KII2X`_4F21sdeqf8eRY2i+$LmJ-()Xm`TiB?(yZ9Jv68OIDq znWmG9Nd4)TLU+ukzas7n_E=7MU^S|K3*Z%6AP|olNba}UezDNp8uKVoGs*Vtbg4E? z((CxQ&2~p#^R95Fu(p=0{q4uwatmkd(EGXX9%S;h&^UQ;{W+TbKVfqh2z&>{JADUL zzWfd{rRk#qP&6wu^*i!F@vwWFgo| z0P$2?W<=MogN}rr6drH#Q@*p#dlrS78=CsF8(ClS9)R^eG>$P?toFO6gtXyiW!Y+=yrOC6Muc&G6}(Wo_6lIsD>}E}foGF!lsUAeNVVE!Fc^JWb_s zk=!m-brCu-BZE&v3a6U7lxV$+(;f)n)kO^$S6$7-W}g~OV9)CeGa#bH$VnDN_C__# zEsLp-(9S^DjncUY^VhqZU!~`&BxLbr!gUTPM zj=_)l)5CL0m2*kug zUzF(xyCYRzsJ~CamP(0mCgv8$sg zS;d3Or;6ey>d(-Z3sE>*h13*Z!5T~Z#SlK?kQ{+`*;RQ9$zKt98ZO=Z@<3ahmojg> zHD4-yqi<4UW}{Ko*G4y0XDmP~(zap3N+~fC>P2VENwg|lS?=pugY%IDZ4*kLZ<4Ow zvti5Jm|B^ft9<7#Gxy*FT?7bYvo|->r7~yKMa(T>(_VzD#3Cs0jQhR6BM5N%6ks(v z+Dg^xeN)q5Zrf0`9PGk=`+JzkslW!rIB zMciI0m*9Mbpg7DTS1We%T~-47X5171GVy+!ELKw%x#*rJ{cU<5s6CkuG*K%Zza@I% z9|IaU0nQW|Ah3z!f=kVja*2mJ!x#p=O#BXdoVNk`%cL$4LIUzrHxkJIqGIVE6fZz{ z^j3HmplY}o{s8gnUPm0w0hXuwb2X&v2*nLF%stlbj{`26QZI8#?B6n1Pc_AHrX~$O<*kuO55} zp)0Hzq>ESS$DW-kD-KKT+($(WUjj(dot}yJbZ z2d@mi#?%xY*iB|-DlUQFo1pd-*SiU;94x22!9JI(!~_}_9wv}Re&=C1g!*16r(S!{ zVLg8>DI}O4KqCV|!M~y*{+p+S4e0Ng<>`}mIraEsOT`D8u!lZw5775ZIz8W!Nopy$ zMV<<9gUYg%qHw?1Z5b3rkhtgBfuc_MsffQ7;OapJxOyl6)KGZVO@icPq}32E_HQDl z!Ik!ibLXK>wt$4R|BRybnt}BH=sCfer^TAuF-_b$;tmYPX~p6K(+oUW0t&1k!!+r| zJf{VI?SO%ve!n%cm%Q)XXIUOy8mHu_D#nK+bF%@**bp-iuOhz!22-ONCr{Hp~LYo&esz14LvjR(N;9si39}Oq}Tw9j7 z2r~rPxFZ==#cisR#`AFH-^mZ7jwrgXiDo6ZKIC=C3N?P6`~M4N*C#J+3KP-F(ugNT z=mOS-jo`CzR{SdsS+gfbG0TK;T&O$6;>nHr?M1%(EnWUMe!&0QDPh3wgZT{6>eTm{ zKML%&O|%63Mf$2O{6>Zc6085EZGpO2m!+!81QP*Ffd2c?ixnM#U>i0m$${-sXN=M^ zWF<@caM7uK9{iFLdo4$b{63(gO#K;UB0xY^s4xl%FuKlO0IXcS4}sC920V5VHk;q* zij@Qwh=3@RG~{cs`#Q$(^}c=s_bLc~o97Lf5jPVb|71x1oto}HdcB?1xQ_JQEN&1) zH3~*NiD^iE_u;{v6Esxta@$z4^HHxVT@XjON`Bh>CwA1TyF~Z8-1biRXi|(FUdhr3 zr3u~ouIsMV8T z3o}-!P-fO@)92ADO4e2-@rz0K3?rnMH{@m=(=p-6KRHnPTQyjSeUQ=SwdI6no5W6~?$CacZ}las(bWJt z?f`RQ;+$l!ltTQyusIE^C}mhp5Xhm2+l>u(J787D<2h2UUaz%lRO~RaWnzMbW2?~y zYz@s_?>9kDoswn2`zdtuL7?|7$w;Y2Z(Gwr)^I9x#B~7^RN-?IvCtEeDhTj6t$Gcc ziZ)BiWjwD*)I^km1JfSz#A4y2wJ;-d@n?WPd16C7XvWvvx1++j%nF&omcmX6zVEt_ z1Lsx>74?~}dZWt3{sqi4a#)qZT#E|0tkNPsopM3lGUMs!x$a`-J35|ImzBMQHHzEa zLH0^A&na-Uknb@sv2VG1KkIwa=ch{5C471>$W0!q>x-r`n?t&dcq&uzeLY&#wnhCq z!ArxZ#XwG@XEbEp!HiUD3g5&p?8LI)a&}XL3hbf%poJ;OB{j(AGP{M9R{2vjsUhc5 zs3d6B40GasV3tWfZ=kkbH;YKL@+2{hrVVHBi?12|9m3gc6#=k%zHXZ;Qlei7)1`%b zUmca{5VYc50)gGpnB&0D6EZCbI{1NwH&FFkhL zw`j86*G+~ij~!gT@wA6_Lm>Ut2e|@p0I~#m709E;K%%<=7Udr)eIUPki$3np!3wE2 zfwWM+28RBN4LtwxG_dFj&fm&{`Ip8VUz_a5w;jBFTu($lhrrpx(w2AW^_)AaJp*vx5F0xc-Gb9zg%!$N~SV zEj6yeEI@i30NWmzzdoUM-%fXebp)_E!!WL*l5m$ErUBB&4oO{B%MMog&OHso~^;mnTy_)zHXGvhj z@c~a#wjSkPGW7NjqDD+#ph_dQ{tPp;9H8X%P$hyO>j7@5>e{s*^Vg?9%j|v!`Lhw> z81j?Fi}YPyvEyim2y8oAH19&A9h=YNowh3WVz1u5Qq?`6&hsL)4zL1Il7XS%6gV>}(eDjN=e!Nu<}DkcIFGq*$*(Zr!DMw zyBLNn{1^v@=n1vt>LunK2iufaq@Iv-gwczv7`acO$qLZBKOit!tCr{w#nlWHDD7AR zo{y^=zJ4M}L-Lp+WM}lsK}A@<`#r&_`YH}*wrL<+DVDhnM8}t0qN7OQE4(?ROdEG} z{mKxcmmgL29i$4>6s3#k&`2B`^CXqF*Qva8m~Loaf|^Jo+ozAWgJ{F4#LBOR9avBu zZ;XPae2ZK!gUQ2G5`>nKQHX-Mi}T4NJGp7|>(^0N6gweW(`OM1{y0)4d^jzkj7{yT2{{_fi)h~2JR@`G8k`y z!*ckktMzw7(s&1#@?))b_LE7M<&cjc#5`F&t{c*rta-bW&39gg$C*w96>5h*t_qt0 z$WRl)(X%-}m*o5zaB;655~iUfbiLEu zEMpvU|B45M6M9DgTi0uH7Q&|i3lw7kaL}7aZx=uI0h`toA#A^CDpLbR$wN$i8bYvB za>>Ik3-S#Kvc#ph2*`tJ28df(?91b9gij)s^E5mB1&1`Kp5_M20n0BCv}=Me_1(#4H%@AG|z# zgTmecv2~v3ytOM;+tc-61}p*34$vhm=8eV*fQ|yr;sC-sNqIns03KX_=@fb@Y*T-B zJ}R*5>lo7xl`-@;*PI+EHGtF!k)$2nht4)a1S<{G`P+#<$e@hvn=sVI! z(`Y10Ft zH(Eg_wb)=r;5{D>{dp4}O>khChl_GHPj&dTDL3>iV*S{v6_Eo4=Ly}S9AbQ+Hu2nJ zjbsl_kkY4{*n4ro7`!=?olJpPDs{o`<&TSvQxe+t_SgK z@GeNXkl9{;XU@8Z=HqD)?UOZY4WmJWQKnZBhS`o%WQ-C;KIDj&Wj=x@J2XK_P3bNI zi)gNIQZ_v)N>yxmSar**Xa(U+(2{616VmqHHiMk(A{H5tjlmhc&qnCz&7OQiCB5vb zOD0U6l4o%^`D&rwY|h}vdKfKAl{1K(67b%YUsH^8{gs<77d?dXl%aQzKIU_kgw?(E z=mrunlG&Kt#aQ^*+(T}C^i4b-^HbSGdUq1Of%a_iRfDrw9oMvr;8XyZ`E7ADUu^-S z$D^g=*reKf-7=~U zjaV@%4BgUeY1NXlN@6tpaF9HqcYPIK4|OR&Y1E~yYieb=2LJ4{ zfWzl)CM6YICW~h1gh|0xs-i{72Fwl%qs>^Dg&NPZrCAH73Q=s0I(U;k_7~0Mp+4fD zk8iGB>ITpU+Gh4)bo?heY-R^64K-Ofl_iAat8%y1fQ`@1-|A4-XSfTihC;HP^|Ra!HxSam@d$VJ&Cm42 zCjkOrVp~qK(X`w93$VmVsdTBoV35KP9=!z!YRzJR9zv?C5EOIw(<63igmtE;`+PGM z7(!a^U~?yw3sZ^hECy0D9vXvK6%v`w6KOdrH6EMzZskaR>m=b#^#KQ`zUMHn^uXv! zI8$os)mnl$_-B#G;Y+Uky4@uo(nllv%NGHr z{1o9_Bu|x86xz@6OXDS_e1Yc_e?3x)Nkb`HK69 zb7?5T!Tg1bU9 zaI_HxzwjZ`+*$p4jH&p|5^1Jvg_5RTwZ?M2Kb;stk_>={v?NGc>8nYaPkJxOu)NAG z;1jwH9}BUk)#r%^=?(V}r7^Ww`Jr#$Z%tkkP;JKQ68w)PQ4{eO}Oq1H6(vMz@ud+(%oAZ1NC)aDadvtz2$_~LDgpUlQ0QRcrv5BjV zZRnx1VLGt4&F2rD_Y-U-ixgiLyxJAJDo=6d*cl_@J_+%Zl5$k%X-fBK9daU46)i5; zdCEkO#$(ySqCC6nv+jOyl=)IoA`+TOLooAG zJFl{L&&1FZIs6};S}!~eev&niUkHC?Wn=<@13GIhfh%HE;_k|)g-PVi6jz0LA~4T+ zayDe&;uLu^L0Xvx|2~|v+T(n?Y6+7Lgka^McW%=7 zxEoX%q}Tgz=k@-p0qnmpO7)j|m6a~$>4tTWM~@^-Z6jb*`BE`(MLNK3CnnZajOdy{ zEDvfO&tC97tgP6cuihJ}HMM*w1yjWrWFT`wjh|1SP=#p+vxzA@#lFYiZ2JYHUj`m-hG#hc zh)$Ic?@4(bcpzVO!r}R@ROP)zYce&2R$g77Eyb3+yv^+F`*c`OB))#3jNXA*o8Md7 z^?0Y%-6Ac>xl#d88l8$MjpW<1#($kKojxpP8Z2w07b%&{CVCV>#KY{YgKw8@D}Z3M z?+~PJRK@#>rUX+)*o>aONy?o14w+tXG4g@Ww5QlJ8Q9~~m`A(KA(@zd12fW}-E}1N z2|5x=bbSHRbHuFG5R|fK5UOE{B;{QQIIZ!G6Kkw_a`?nrH(u)rLc7AFx1?aMTn(Bp z6WbYqnJV4J`uNc^)VT3^C2t>ih1(%B!dYN}_B1F;=C_}mv_x!JwD=5P3Rr4lSHSl; zVnk{Wf(`&41;Ra%ZLs$xY05gT(*}2KPcX*d*}<8t@B;;noiQz|RuIT}B=nM&tz@9g zXSlSI?#PrvIVHLU=V_v~)Z4qkyUf^$bEsZn&H9EjID7EEC_YEVG*ssK+pNlmpD)lX zNIBIR4W=&BeAo zY`@5K7}%E5(o(0euy1p(ZsAEBwzFI$a5W>T7)&0Z^$yyrfm^0@&Q4%YhCvBKUs@&p z7(xS}obL^}kU;Qj`?uc&PxR36*>&Snw>??yflslu8i(w%Lak$TIaKY?-2VmUExzi<0zh%Sj%}d;VKsBEcT*NzvHE=2KM+#LWyTM$^fnCZUe2 z*@gRw;|(p}E^jvJEkI7vH+8ghgw17ApYlfQ2=f@&?wM|~ZF#}6sc7)SkiEH(LQkKS z)}I{#hO7u+$hH{&!;rmk(BPNl_rojy<>-I?TE2Vw{&Z{taGTMXcHB!bN~*3+jG4y- zb5q-RusBSbI$;wTg5M5LesYOH_asZ!hjQujl5m zrV4P};2eJWC=)MBkOoDf zbM_rDJ*Aqlp}s{k$&)Q~o%e7K7{2;9>frO*M-;(PnzkWw??{Vx?v2fb&BlQiYIqi1 zwO?s)U$N2{vId^qBQLE-dv1b2h7(}`oyn>)@M87BcgSf_d+PzwDBwG;klY(Nbk_$wBo?T3f0O1dIak;95ra- z&-S2^dS70q&&oy+jLNFF5BWG6SEVbOJi_R;TbPDieB&~6PdQX-y_)?InIf`{%9Rwy z-y`+6Xw67xPMJ|b8*MhLHqTUM8P?(TYu4kwQK(Mb_}2FFk;aVb8$3Es<`!(|fb;E)!>r>~@%lQ_Cu@L^juCEC8MgqiS)%!i^H+jEOXAm?d+eg>Az z*w%EQ7{l-~!_S^T79&kW(o=IFRqxwA#(A*s6ggr@rkayTBEbVQ& z{(I2fHFejun1wTyCsmrse3Yadq8WQEqluAgkX>`8LBJk;^9%hzQExP0V)9q5LJ$hvPrGf< zW#9qu<;{b)?lLMQ^51wR3kdBVc}C{kwAI(1R`u|WP3a0BbVy?jUP7v7Dg`ZKIQK~F zEme9QUjafjH}oq|$*+4Is>ugJHJ&uH5SHIVH8+o{oH*5R&-(GQM=LrNob#uU9q_fp zSl=OWjh<(4px$ZE0fw!0pLTRejK?zhWa1%LL!|IT~hnQsPuDS<>RlKU9-D3FJx zBZRrGxtuS!fCXZdYw0<0uHCKgl5rK5y|-K9?~-)v^h{?qshnxs2&yxOMm|6whJJ5F z1)UX5wd(&Vjy0SeTgSFdK*q30O|8*iCnJ3@$unJ4 zo-6kuQ=G~P^9G%J@C+r&Yq1bT$0J)+%&it+c>l$u=!4r0#sI^}j7`4hYE2g-NH1k+ z=T+Pz8*n|t0O_449(lT0f6Du+R;t>T(4RocO}?|DK+FW)(GDv1bT%U_qr4?fs4}|r z(*0K2kv$1AeX1RgDE_9yY<4QE3(HT;D__uHG!?cD`5BR;_Rgd~fS(x62;j~kd4 zH}_M+@Y4*biY{*YuoKkmjLPIP0m+i}79n%U0#U?;v8Hd{`BBKr-4o~R zaUfnKDb310en2xR-i518^3~mzr6pamgE!beORS3okf%t(Kv4@CuM$Ce<0;=k1NRP6Tx2f=R=FU*N5aovR z?PO%1fK|w!!jS+_*k7NqcxW7owQd)Zf7^sYB|7PXoUeW_Ja;(=NNBn(9Asi%$~*a!Tl~q0LuE(AjFBg#;Sm z3yi5M@8zYK^T_PwvZerZ5`7+0p|AOld=i}PoY_}@Sj z-M1n=_0NXqWLanT5jd`?G~-xYU&*{Wq)E>0ek~HH@IbPp+tAshId5mK<+Ob+=z;L# zMQR~<=n5FQ3#`Bx@6l2{=oDANC#{=-?*)W_kOxNZRVdKz?T@b9vNvIj|LPP)pk?V?`3XKCR_v)T-LW4j# zTwgsv?K=|0xHf>%`X6Cqu8;pxLHu72#Qr0%4As8QjTK;HO4uq@K6h9|}5#ow#EDRhxlz-GsTG$;Y&f9wMQr|zFG(;q&^pDy)(pK%k3`7g#JCowO@ zAycLQTWNKIyQne)-4Zw~wSO5AV2GgTqA*VB|4i(sJ~D{KP<{W*Hr&L1+@E| z1MsF5_|%^4y(meYHMHl31QotgYd0qqOYOj>$(9r1Ak&BM8&&yD2V(f<8k`4@6lv!e zI(g@%9giGLMpcy>4v#4M`+seF_dtuF_oZ4SV2|I_4!4o*Km3;e^eM`2tYv$0r&vvN zmbq&Kf~=hpBw}eDL7mI&g4$MsoZq}cg*BF~Aw1WX`NHc9SPl2JjAZCrv*;E$^hZ?V z-=~~W|BsE!t8c=4gc;?#Mz}MPBSq6{Q&p4ea-!SIlHnfMQ;axNa25`1Lw~Ww-q?QI zDFSF6!%6frR-jqWgNmikkFp z-ri4l$)&l;!-Ecm+lv-`mzwESg{I|`-utB7oj3{`4L-0E0Mcf)arrFK#F-^{(no+B z*FgbeLisCLl$Oy5cs}+-GHOD3xoPGvdXe1^-izu8u6~Nfxy(M4frjbFLDYIm&zR^8VvX`ANwq+YgUjw%j9sHV=KhT0v z#q`EYSo$Fb3iV^NSWOn-_agp0Y(HN;3HVk?)3T>zkWxN2zD&v2ww~e^1*gI{Z?8w` zZ1>DfKc28_(QZDp$0rd(#4egKuRjnG7s$^C|1Sy!8nOQ_jNVgZr^mp6y!}ZSnf-#O zxqDoi;i(v#KJe(YyTvTIMNzMEu{Gr*WnkP~_+Q7aw3cvAS7;xkWO~bLqzBR1Xf?v6 z0f2wYOji2To2 z=Ld%O53#4{dxs_H2IU-SHr@fgMS4T*NW9bm+q7@gQZccwNs#dV5WD2hS3jqM@-ztY zVIsjIkZ0^&7XL%$*8^qDOB=LFtE==G4jbL^XPIB$&Tw8{$x!>=Gjc?-}+-MQmM@A=&NW`*D5o9xt_t7Kd(h$>3g55rLYy(nthOd%;0~n#etuN z0St&{9OTK??4Y&40@^BN58>OmOGb4V09 zW6Z%2)V?Vf{~0!<6m))V!|wIocTj5_@&d4YhpK&0K#14Z$9+v>Kw$Tt0E8m+^eHdPchHlaFra@TuvGWhN<=`~ z?q46xC!J&2gkIhW1+bT?5L6UIpbu?zn)@8O`{i?!enEWKR__|#I)*__T;ymH=lfuT z=8w)Q@1o}(G7Z6*xJu_40~oU#7x}ab5z}CRG>j&lGf$KoK;zxIDABH;gFeK2n)Eve zlnSu%8tVmsS^$?zxKZGcex2wPmgqYOP3)3{5CJWT$%y8+eLN~wZ zj$r(ZwTVAKpZ)i$2Rv9>s|D%s&7+r-KmVR2_h-P@M5S*6hPR%konP#H2 z6$20B!P$oId~Yg84k(xzoHxHfxW;qZvqG)??A2i42c^s-R+8^#_H7FcvXP6CVmSCr z?U?8fdRm_YMN;JdENB6Gul(72M&E-WFJR{+yxuv_gR)stcjjo2>MBW-29J{O{-{Fj zx_kt5=1>IcN;Nm#uQ1EjqgLDiB7ra(AdFpTTMr3T1&x0yUa1Px9~v3!CU0V-p*4Z~ zB5q>(Eh5DIURRlWnObhN>QX_LCK~!aD7`xSnFaS@GIC--TXo#RH-+cyIC0aMMBI*~ z7gHZ$VGhrS3RPaI%(Q4#hdYbHh=WloBaf*q&t+h+1opYt-H(xLfH@UCT%t}JarL?7 zc-=MoxjwVnE9$H_*g`8pB`~4!q?EH5^^R;upFEZw)T0+UheC1|`WZx{W3O^`gA8R=M$W6GEzwbot)ZV+cd`&e6+#{L z3ZLWJRMnPJY&t|>cI7nD+q2jx9PtTogX%?O%ZEmc!_4Vwjt7~eMLwW>M6MtXYfWc) zF}yg`WxdLN0#A1pYp(?pdW8uL2>=;qpfkC| z$PJADpm~s+8Nk5y^_w-`8znx8Lv$#Cznt@>EFzXL)(s?5=m7*|qKLVf*CY%4zbJtJ z3#%Pa-iRuD3*xgv)z1d=bXLCUbN`sYlwiRft1WfurNyywloAp))dvr>!ubPn z{Sv>z;W*^J%SI-!V9u-@!P;{ht1zfA6`CDUNH&ewt=*6wquCXD%O7Cu;;NEtZVJ`Ls9 zYO}f0HvB!r70PF3T!`>))-OdI9i3gVBFx!pKk0M~W(=(2gSa7V*&-lId+Tl>*{t!7 zJ3pShUy9~S2B()lRQN0m@)$Jy2A|RC;7)44=_ketk|SMZp((9UH$loVh0dfi0-O2^ z_*d?0okruyaL>pL$Q~4`NWKpe737>7fjNk;765uz2v@{?GaGm+m~Actz})3!;8`dV ztj*3{PVQ{HApnOxE)Vm{ZX@0mA}u2ukpq}y7?2V=V&jPGT!Qw+F?`AJ7i(<}W#HV3 zV8T_&b{{zh8e$p@*_zlA`L*SamsZF0y&=cRuj`{<5i!ke`uHA23+l_cl86XMb6l`^ z%x{ZttQSP#vJe9RiK3_p-|}hC1-|StM2}&$_s?S9RyPF&O|LO zCb@p3{2lb|DhZ&fOuhLrf5KH-x~xLYIdIb;)4du$ zJg4aVj24Ig@7k`adv8|wBW}d@Na9oLs~i*x;dc>Tc*+-qzA2Yk&^`yYv*+k+?iWB} z4ygCoBx>fSC|Gbuo?d_HAf+fwy~4%2$x_rx?LK;bblmOaoqm;Y&Z08iV3A|df&=a0 zx+j~Fm(gi|EPixftg?qUJyvRU{bL_Il2KfDLK=JDDUYIZO0c%j>i90`3(!pSh~d_= zmW`yp(OQeKFCW?Z4zksak^v7=B-pwd>LSGLY#@;G!bb}xLnu5r-8nP>*Gm{kXVo9O zU%G|f{8&Av7qCDcjq^}^OjiT@fmjj7M>b8M@+IgK=N!0PqAPc-MG89c3!)4>FU9HyNu!z!=^Fg|iq`Ink$RKua%fXRXJe$OzTyR4^6gOw^4N4n z<7_=O5jyvT%AEkU^a+d_lR`cLDlwVdWQ6@KFlV^OfR+$}?rRmBdcUQSt`*LI_OE5n z9q`FduDtREfF5Ao+79)Qr+LnRYfN-bmj$i&Q%&hFTKlGk9iB*MxD4jU1k`m<(z%%R zKgwi(k1EKvr6-bB}vDr%TAT<-B^>?MXz<_hRshMeyOz+WF7k%j?E$Bc!`& z1qmJE;6Re1ByNT#B7d{QZYkMX=9>7%IRC**rnGSlL#G9GwQ*te?5kiqyW(UR9Wk%A zZ9xJyjM%G6lq+bUO6lP%_q`keFkpox+g$^oqTMzc{Yupsn z|HybOG1S1CCa>{79`Z@gdT}r@BUu>li<+gQ#Q`u^K@a=)s_WHm)WoX{Zyi0qFEcnY ztE5;}9u8_1t6sE<>OC)T+5p7r$8FpE4(%0CDHMa~(GG2J@4@xLL#`kLj<=HeR>=sKJqCA{505Pz z67p%IbV3N2^=s9%hehuphAzw(%@lsqnfzq1Q$LP5va$dV_MBy44*s&gbOfE<`s%3O zcwrWCT(o}zAdp&XCD-bs??XF$FeG~#`jzo4Q8CIXQGa%Feln%{J9tGhHiow2#$Rb2fW+>a)Ucp=;MjAupQ`e%v> zy#Y0JPHQJlZYeyO(}percFk_Fz5%wqfi5i04+3^N|G)OGGpxyMTL(dk3W$QzgN9zD zh*Tk0IQ;5`&F;~V~Ww7a3&u*T!IF*80!pW_X+Ih3)>rDBG`68Us|Mra-7PDk$rsE z=B%hhLO?X$FZoiM8UA?>DGc@6u6-3HiZIJcYQxT3mKfP;={-LXAkSl1G_CyMb5Dc8 zuTIR-Cx*w(+YT1-@C06pH}gODNykf!`bogSLkwqpe5bW9G|w^G`2JEUJ-bG?tLJJq zFJB`4o>^ysEI4j7NSY|oKomUFu{Q1_e3wKP%nw{?8IA(eJKR1YOQUkB*= zn(7Zh*(|OqvjcW=lA=Y@BJ25pp zpZChmc)oJN7;tYuTzk^3B`KTkx>NacD{yk!E*!;Fed2qynb(5^c|FT0;?klVFmo`i+^ojG|nOJ zh}zDEVx$n+htRcN6pNN|25Q%C7#^YWW4dh#osI575%LOv-RM_EK4GpbJhwZse}DTk zODCh_ow})&vDd@sBJPMHA&l;>f|PmBz%O^K?8p`EYgJ<%WoM}l{orN5yD7snwi)g^959%$`f^k5EuYJ6Tr>dTiIaQ zi{{HDofN#kD5WqRhh&{ z|8SJWHUx@0U~piSsT>Y4#~v3FGX`sxOb=i&j=BxV5`8g1W_1kCm(NYsnrR;uq?AVW zTjSx8D|%GkHuEPZY?kN&oYqo|!CH;H%aKK27lK_&o9KAXG%$wE68rTw9evxY8M~e0iwXM7EerWZMtLlsj`+~7%d2B|M*fg<)t*#)Te2vip zIjNG4yLBk`MIXi^oT&}^tvg6u-#E4G*!=T?RRrt4Ixins$G6NlJ%qihwk!$zF$uX*yp09lj>-Da3OU#)(VM+R|O+WK^zEm)rf?1y;sjEq{fb;rTcbUT9 zt~<1o2Snr`$aR7F+$Psx;cG%U&IOyCzei>7!oP5U45DZ8K{Df;g`a-#*EDyS=+lMj zG}Q$$ERBxkd5rZob3;JV1#4gw=^V+M|IPPDYY5NL<9a&@FQ~>ANLSAw;~2SHYFrXJ zZ8soRdX#_XR+~)`mU{j^vMeZ9W^EXaLbN~6e3ReJbl<;*m0}Wrn*dXf{u465s0^|@ z?)F204e5j>hgr1D+pHTABKuY;jwi1dn)mORu$I>;-d3db5LG|aLVG#(;*H4@;x17O zO@Oz^ndtpnm+YsK>2jy?SGlj}uTuk7m+>Bpp42_qqZ52+OIKb!TA%w!<-xT! z$p)ncqN$YSU8<{^qI;DClX6alL2gY8T)Bz0pyIIoRXFTh*t`x9hM<9vNCdQwvc15ng z(&z=eTGxZP0a4I-%hkbjws=ZP_=f$-!JD07Ezg3a&V@(Vxn? zGXEzuCQ~fm$z|WO?oqbzW5|aZph}*caOb$@)6_jCaqLU^&f?NZ!cDnWDg2f_qao>5 zL%!MdfSP(zN)<-K0`EF`VFRL)#JB;W2K2=z9JX_9ysUhxu#(3m=d`Zab#e^tO`|8j zP$NxodkFz5Sqq#7n@Q0CF+y{P5Eu2&*4%y=44Nzxm2xqWE4TkbyLZ0Fl`&;Wb{zsT;6d#^b%m3-E|#KY|5lZ6joNg+tGGbj<=NCI?P88#s1R#Z)a7;Vv=*)F#eFbv0(ngxmzkeyIT2uS^0^^4Vi| zF{@%H{OE<-=M~KbdOy5dnm$$y>4uEEBR+-s$N&27Vm05p!`y+H?XRRXH1<=k-XI?- z+Sk0B@f9F9yQu&r(`Nv3v#ZM6q~-)p)%&W)wpv=9e|F|^zl7G=g1Fg zBM*9k;x#D-WRTD`15d|E7967($r|`Xp2hlAoYp9G)OtN}F60~GsJK^W3%|-#koE#i zy0^3*jo7{dP)n}4wA?`Ox~DuCrK2fU%fZgmpJu&7okmLT@u%npsdLEDAK8D)3%vq~ zAHVXKteWo~@Z=%@KWfBh#&v3tH#DB{vb26K4k#U-O{GHsT&$3u?_K@-|0oYP9ItbH z-qUNP!zrPj)p;^ntHzkwx>lQa7Q#7N%DAFS1X}F$`OW#qidxC8nG+}8c7lHqdI5?F zS)J(6+wBKEd~OX1}&l(^TFh?X%Xo?6P%Bh-K%TNvMxOC(x!Sg z8?f6L7+Hp3FG@)QE-j5Lis+Sq{JfW>gD9>nn$0WV!@SdxMvQjTH&*X=z1%p6#=w`7 z^T7?!n>PUTV=E+xItW0>NCJ#33V1SuGx%y6IveIN@a-FL_(wP0>b9zGUo2^l0-i)J z+<*)%0Nxi6I3_p+U0x%A!c&I2BtPoM*eqgsQTH4t+Dt| z>xXukgxRXPHG8Z~K689bUz3PV#f%k;sJq6KZuGH*dbg`l!P zr5xclrVgD_@tg4Kiq=c*iyW4I%uW)@555+^OWk#{CD=tpmyVcC8cn~D=}>Ci7EMUE zvV1eU+Vbp6;lpEWoZJ$=3~*tvj)I?ox~Rb<4>Gw6HLp(7aCq2uMbQdIF5UbH@#t92 z_xu#$O;}g@B1v!}Cpa+>FoAPY48Xo(P;$IFq*n;)MFA0ZT7{S#ZBYX2XTdj2mM z)}`#V${fEO2X`}UFs3uMi^IL?k{hNDMZjZhxCN@%@V7BFDQk|vux(-t>Kx<26~$P~MsMT6hCjk+e{9>zO&FzV>vf)AW7Mp?1l-%po-Hp%aM@-1>7- zwFrMY`0O3WCrXHfEn(1u15CFEr(H>QzjuClJRCcr@SI~INtSfPz=KvexBfySY=nVg zxE&u_o)ckXSRT6O*gVDQ;x;0ze#DCOL3s``nCNRr2orCtnf1a&u1#%8W;{4>^CpnY z`1@yyV_9I&j01bfR8j}|RxF`s1CrH>`ho;|`c;zwA|G%`7X+qKmY23hZf9phJeqY& zuPb0Op1wSh_|oIJQlTYGdJ=yN$L3VGtIUakvG9tCMBoZlh!I^i1GU^fYF9_zmE@HN zh})jEnuz|*0TSPN56b7ya8)a3#KF#q=zz~m$8A7(jg<`R$ww9Z)3j&3(w1IHgid~L zG1exFisHQ#hQb#bQyFM#Hy{kx6)R-xO4qeB)5lvs*-M^pOu(L^>8Ei=`|PRqnrQH> z+OC7!EUe;_W1pKlwD=&*uT1l3v4pqZIASCSpCV;<{8Ya`x1s-k+i@%ZY+#HV)dK{7 zj;NH7Ci>j@>DadL{i-okOx?LVY?_ws8+c&QH3sJEL&KgIlJ zF+v6o0IyW0m0TN-BFENQd%CgTZ1*%O!r1JaG#Rw%KKl)EBClTSD=g|>&vDs4!)EsB zamvF0c#XPUo3g|lvjNfi>X@Bpy7QaVU1?E9<$MWL%0|9PFV^gzXAXB&a&^52j@(qy z2IOor)x*D;GRMMpQ9!My#ze6~LKB)lVPHsyQc{8Px5mswMNX+O?(k_Sqh>XUca=GT z(__7fv@}QU9$BwD!q58PK+uqzGA3m(a&^DIJOB&JHo?LyX{q!^9Jc$Nsbd!|U)t#OKkumV1=PZIRL` zPEUp(s2`As2zxkiNA4@Nu-0^!_#s4`ZLnrFUVb5aStjP}B@^tu&Xj0dPzbbZrU68O zY6WEr$3#=pE&jRO{69$0cl`;X`ggtGx5FL$t9yvn(3=glie51s_ez*oL!x_aU}H$N?HgPg<5;SwaV_q6iaff1Qr*%CumT)6}Ib?cD;AEkEAg zK6NU-AIALWILLqVd)87sF`SzjEQ#lvaYxq1?mNGLFyoJLE2Z5Uko~2GT=JroQ{HDO znbN<12n2ryL4SZ2|B)A^A8S(tJG6RBg0c;G;DMfe8eRcUk0-5jK+lGQQjTDx@=>l! z@c|9B{L5&mngA`9n7xzk-C`pd=`2%&$1U$R3la1f^+2$Ag}ObLUWC8H2&X_ynX)DH zamx9}c$9_r(#V}B*k1z@if>d}-teoCMKV*Qsq3eh7GlfRMatMp-iThSzx;IYdWKG( zS?yQ0VIdBu8z%gMFPeX0O{HfSPH#_lx-!90SdNwzRtxEI$%9OsJ$JrVNB&s*;0>nY z*Y31qXvwxkW{SoYuK*9t+4nISh)?v^qlJZ`{TgULsr0B_(Z}WF8o%y#KwSeNjtc^g zsO7ay8&xE*!vI?r`z*XMIR2}1SSZHQ48vhf8mb+5@S#md7}Owqw5`)HL`Pk2ok z78gA}HLCeAZ#qij{i(GFG`oT%P{k)uYX>n^>!dA1duOW*HXxR0+G`nDezULqdvQqn zRlgxFQt9GCj;2x3p_tw$T4B^eM}$=k6p{IEDy;Ax_9_xx_W+5!z_HglIBzSfVL_lM5f^&; zgQNgO>UWS;F!xI<@A%oF384>PBT;`7LSV@5E5=AhyU7ma5mihrN~S z?Q4sM+k6OuRr=k{P50Ak*Z%09*~ih1E5mufl4` z2J1L_@Iby&WHhIc$f)x~)Q2omI{XtmxoMtv-8*)8mD~LN*M!1Wu5Z@om?@%OoSK{R zZm8isy=w-RM`>T+e2gFW;pj4urrgRXTRRKHopr`d%ICR!3P9pi=XqH>;?_#jd0r zR}|(N!5PJ4ckLK0T&RX-wL?c8;yxteie-6KqFwYbchlhdim6yv)C|w zAGIqNC8I`IV7!U86lhGVz6zV~H9Joy^EXkYMm4a^P7w|bY@AN$8&Z9=rqpBmeK-js z-hume$5i~(l|QO0eb)~EcR3wtG4?Comler2c_5zNz{p+dPO#K)Ty4tff6gCee*Nua zeNWK~#aFI=Na)K=wg#!yU}lx1yU3Kc(Dtaa&x;pS6fZOpCN{}3vJ#Mo&=P$??y5vL z4g2|TBls#M*~CB}KoI~4EZB;L%!b5z5A!hMnI#{xbl%nU$FY(Cp$kRu+ffM99&bK0 zN)%;55%-Sg8P%85oMbbQ5((H|q@sITAk%jQ;OTqpI~&r%WoV1mvzEPSj%+xz;+KN=+&L zSXGU#GgO{fKyqMyqvN7RSS>bxkLXttWE`J}vj)oq4wZVZ|XPv2+XnT`y{x>^Onx1R8t z-EOa=OasR>;Aaf%zCmb16r=I*M<7J#=Ro`%h@UIs|GH1;B6YhC}H S)Bm5p`EiKce^MW9bp03C!qm6` literal 0 HcmV?d00001 diff --git a/docs/vm_ha.md b/docs/vm_ha.md new file mode 100644 index 0000000..68f1083 --- /dev/null +++ b/docs/vm_ha.md @@ -0,0 +1,172 @@ +# VM HA + +Virtink uses [sanlock](https://pagure.io/sanlock) for VM HA (high availability), and includes lock to avoid VM "split-brain" situation, which can lead to two active copies of a VM after recovery from a failure. Besides "split-brain" situation, the lock is also used to prevent two VM processes from having concurrent write access to the same disk image, as this will result in data corruption if the guest is not using a cluster aware filesystem. + +## Goals + +- [x] **VM HA with the ability to avoid VM "split-brain" situation**. The "split-brain" situation means there would be multiple active copies of a VM after recovery from a failure + +- [x] **VM HA based on the health state of the storage instead of node**. If a node can not acess storage A but can still access storage B, the HA VMs running on this node that have locks in storage A will be rebuilt on the other nodes, but the HA VMs without locks in storage A will still run on this node + +- [x] **Disk lock to prevent multiple VMs from having concurrent access to the same disk image** + +- [ ] **Live migration for the VMs with disk locks, or even enable HA** + +## Architecture + +![vm_ha_architecture](./images/vm_ha_architecture.jpg) + +- `Lockspace` is a no-namespaced CRD object refers to the specific storage, and each storage has its own Lockspace. E.g `nfs-1-lockspace`, `nfs-2-lockspace`, `iscsi-1-lockspace` and etc. + +- `Lock` is a CRD object refers to the disk locks in the Lockspace, the VM with Locks can get start only after the corresponding disk locks are obtained, and Lock can be used to prevent multiple VMs from having concurrent access to the same disk image. + +- `lockspace-volume` is a PVC object refers to the shared block device or file used for Locks in storage. + +- `sanlock` is a per-node daemon, responsible for acquiring/releasing/checking Locks in each Lockspace volume, and there is only one sanlock daemon instance on each node. + +- `lockspace-controller` is a cluster-wide controller, responsible for creating/cleaning volume, initializer, attacher and detector for each Lockspace. This controller is deployed with `virt-controller`. + +- `lockspace-initializer` is a run once Pod, responsible for initializing node lease space (also named delta lease in sanlock) in the corresponding Lockspace volume. + +- `lockspace-attacher` is a per-node daemon, responsible for attaching/detaching node to/from the corresponding Lockspace. And attacher will bind mount the Lockspace volume to host path for sanlock daemon to access. + +- `lockspace-detector` is a cluster-wide controller, responsible for heartbeat detection of each node in the corresponding Lockspace, and rebuild the HA VMs when node is "dead" in the Lockspace. + +- `lock-controller` is a cluster-wide controller, responsible for initializing/cleaning disk lock (also named paxos lease in sanlock) in the corresponding Lockspace volume. This controller is deployed with `lockspace-detector`. + +## Prerequisites + +### Configure Sanlock Host ID + +The sanlock host ID is an unique identifier for each node in the cluster, it's used for acquiring node lease by attacher and for heartbeat detection by detector. You can configure it by appending annotation `virtink.smartx.com/sanlock-host-id: "n"` to the Node object, and `n` should be unique for each node in the cluster. + +### Deploy CSI Plugin + +We currently use CSI driver to create PVC for Lockspace volume, you can try [NFS CSI driver](https://github.com/kubernetes-csi/csi-driver-nfs), [IOMesh](https://www.iomesh.com/), [Ceph CSI](https://github.com/ceph/ceph-csi) and etc. + +> **Note**: Currently, only the NFS CSI driver is tested officially. + +After the CSI installation, you should create a StorageClass for the CSI driver. + +### Enable Watchdog Device + +The watchdog device is used to reset the "dead" host when the VM process is not cleaned up by sanlock daemon within the pre-configured time, it's the guarantee to avoid VM "split-brain". + +You can try kernel module `softdog` for test only when hardware watch dog device is missing. + +### Deploy Sanlock Daemon + +You can deploy the sanlock daemon directly on the host or try project [Containerized Sanlock Daemon](https://github.com/carezkh/containerized-sanlock-daemon) to deploy this daemon in the container. And the watchdog fire timeout should be 60s. + +## Usage Guides + +### Create Lockspace + +You can create a Lockspace by the following yaml: + +```bash +apiVersion: virt.virtink.smartx.com/v1alpha1 +kind: Lockspace +metadata: + name: nfs-192.168.27.87 +spec: + storageClassName: nfs-192.168.27.87 + volumeMode: Filesystem + maxLocks: 1000 + ioTimeoutSeconds: 10 +``` + +The details of the fields are as follows: + + - `storageClassName` refers to a storage, and a Lockspace volume will be created here to hold node leases and disk locks. + - `volumeMode` is the mode of the Lockspace volume, use value `Filesystem` (default) for NAS and `Block` for SAN. + - `maxLocks` is the maximum number of Lock that can be held in this Lockspace, and a Lockspace volume with size `(n+3)Mi` will be created to hold `n` Locks. Currently this field is imutable after created. + - `ioTimeoutSeconds` can be used to configure the time of failure recovery only for this Lockspace, please refer to [Failure Recovery](#failure-recovery) for more details. + +After the Lockspace initializer succeeded, the Lockspace will be updated as `ready` in status. The attacher will try to attach the node to the Lockspace, and the detector will do heartbeat detection in the Lockspace volume for each node. + +### Create Lock + +You can create a Lock by the following yaml: + +```bash +apiVersion: virt.virtink.smartx.com/v1alpha1 +kind: Lock +metadata: + name: ubuntu + namespace: default +spec: + lockspaceName: nfs-192.168.27.87 +``` + +The field `lockspaceName` refers to the Lockspace created above. + +The Lock controller will initialize disk lock in the corresponding Lockspace volume, and the Lock will be updated as `ready` in status when succeed. + +### Create HA VM + +You can refer to the [ubuntu-ha.yaml](../samples/ubuntu-ha.yaml) to create a HA VM. + +The details of the fields releated to HA are as follows: + + - `enableHA` is a boolean value indicating whether HA is enabled. + - `locks` is a list of Lock needed to be obtained by the VM before starts. It's recommended to add a Lock from the same storage as the VM os disk DataVolume, the VM will most likely fail to run properly when the node can not access this storage, it makes sense to rebuild this VM on other nodes. You can also add a Lock for data disk if the state of the corresponding storage is important for the function of the VM. For a "dead" VM with Locks but without HA enabled, it will only be set as `Failed` phase by HA components, the run policy will decide to rebuild it or not. + - `runPolicy` can be only `RerunOnFailure` and `Always` for VM HA. + +## Failure Recovery + +When a failure happens, the HA VMs can be rebuilt in the pre-configured time for different situations, and the time here refers to the time elapsed after the failure happens. + +> **Note**: We assume that the HA components are running properly when failures happen, otherwise these components need to be restored automatically before recovery the HA VMs. E.g the `virt-controller` running on the node with power failure will be restored in 300s before create the new instances of the HA VMs. + +### Storage Failures + +The storage failures here refer to only part of nodes in the cluster can not access the storage, the HA VMs can not be recovered when all nodes can not access the storage. + +When storage failures happen, the HA VMs with Locks in the corresponding storage start to be cleaned at `T1`, and this cleaning process must be completed **before** `T2`, otherwise the host will be reset by watchdog device. At `T3`, the label `virtink.smartx.com/dead-lockspace-` will be appended to the corresponding Node object, then the VMs with Locks in the "dead" Lockspace will not be scheduled to this node. At `T4`, the "dead" HA VMs will be recovered on the other healthy nodes. + +The recovery time can be calculated as follows, and `T1` < `T3` < `T2` < `T4` : + + - t1 < `T1` < t2, t2 = 8 * io_timeout_seconds, t1 = 6 * io_timeout_seconds; + - t3 < `T2` < t4, t4 = 8 * io_timeout_seconds + 60s, t3 = 6 * io_timeout_seconds + 60s; + - t5 < `T3` < t6, t6 = 12 * io_timeout_seconds, t5 = 6 * io_timeout_seconds; + - t7 < `T4` < t8, t8 = 11 * io_timeout_seconds + 60s, t7 = 6 * io_timeout_seconds + 60s; + +E.g io_timeout_seconds = 10s, 60s < `T1` < 80s, 120s < `T2` < 140s, 60s < `T3` < 120s, 120s < `T4` < 170s. + +> **Note**: When the node can not access storage A but can still access storage B, the HA VMs running on this node that have Locks in storage A will be rebuilt on the other nodes, but the HA VMs without Locks in storage A will still run on this node. + +> **Note**: When the node can not access storage but the management network can still work, the HA VMs will be updated as `Failed` phase by `virt-daemon` at `T1`, and the VMs may be rebuilt on the "dead" node before `T3`. + +You can refer to the following steps to recover a node from the "dead" Lockspace: + + - Resolve the storage failures. E.g replace the faulty storage NIC or update the misconfigured firewall rules. + - Wait for sanlock daemon to remove the corresponding Lockspace in REM (recover mode) state, you can use command `sanlock client status` on host (or in container) to check the state of Lockspace. + - Delete the Lockspace attacher pod on "dead" node, and it will be recreated by `lockspace-attacher` DaemonSet. + +### Node Power Failures + +When node power failures happen, the HA VMs running on this node will be recovered on the other healthy nodes at `T4`. At `T3`, the label `virtink.smartx.com/dead-lockspace-` will be appended to the "dead" Node object, then the VMs with Locks in any Lockspace will not be scheduled to this node. + +The recovery time can be calculated as follows, and `T3` < `T4` : + + - t5 < `T3` < t6, t6 = 12 * io_timeout_seconds, t5 = 6 * io_timeout_seconds; + - t7 < `T4` < t8, t8 = 11 * io_timeout_seconds + 60s, t7 = 6 * io_timeout_seconds + 60s; + +E.g io_timeout_seconds = 10s, 60s < `T3` < 120s, 120s < `T2` < 170s. + +If the node is manually shut down, you may get a different result, it's because the sanlock daemon may release node lease in this situation, and the `free` node lease will be detected by `free state detector` in `lockspace-detector`. At `T5`, the label `virtink.smartx.com/dead-lockspace-` will be appended to the "dead" Node object, then the HA VMs running on this node will be recovered on the other healthy nodes. + +The recovery time can be calculated as follows: + + - t9 < `T5` < t10, t10 = 12 * io_timeout_seconds + 60s, t9 = 8 * io_timeout_seconds + 60s; + +E.g io_timeout_seconds = 10s, 140s < `T5` < 180s. + +## Known Issues + +- The VMs with Locks are not migratable. + +- When a node can not access storage but the management network can still work, the HA VMs will be updated as `Failed` phase by `virt-daemon`, and the VMs may be schduled to the "dead" node again before the "dead" Lockspace label is appended to the Node object. + +- If you delete a Lockspace that is not ready, or acquiring the node lease is in process (you can see the Lockspace is still in `ADD` state by using command `sanlock client status`), the mount point `/var/lib/sanlock/` will not be cleaned on host, and you should unmount it manually. Otherwise, if you recreate this Lockspace, the new `lockspace-attahcer` Pod will fail to run because of the uncleaned mount point. diff --git a/go.mod b/go.mod index f4a51a5..d9eea8d 100644 --- a/go.mod +++ b/go.mod @@ -14,18 +14,19 @@ require ( github.com/namsral/flag v1.7.4-pre github.com/nasa9084/go-openapi v0.0.0-20210722142352-4a81d737faf6 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.18.1 + github.com/onsi/gomega v1.24.2 github.com/opencontainers/runc v1.1.3 github.com/r3labs/diff/v2 v2.15.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.0 github.com/subgraph/libmacouflage v0.0.1 github.com/vishvananda/netlink v1.1.0 - golang.org/x/sys v0.0.0-20220908164124-27713097b956 + go.uber.org/zap v1.19.1 + golang.org/x/sys v0.3.0 google.golang.org/grpc v1.47.0 gopkg.in/fsnotify.v1 v1.4.7 inet.af/tcpproxy v0.0.0-20220326234310-be3ee21c9fa0 - k8s.io/api v0.24.1 - k8s.io/apimachinery v0.24.1 + k8s.io/api v0.26.0 + k8s.io/apimachinery v0.26.0 k8s.io/apiserver v0.24.1 k8s.io/client-go v0.24.1 k8s.io/kubelet v0.24.1 @@ -47,29 +48,29 @@ require ( github.com/docker/docker v20.10.16+incompatible // indirect github.com/emicklei/go-restful v2.15.0+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/fatih/color v1.12.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/zapr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/swag v0.21.1 // indirect - github.com/gobuffalo/flect v0.2.5 // indirect + github.com/gobuffalo/flect v0.3.0 // indirect github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.6 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ishidawataru/sctp v0.0.0-20210707070123-9a39160e9062 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -84,21 +85,19 @@ require ( github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/spf13/cobra v1.4.0 // indirect + github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.19.1 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/mod v0.7.0 // indirect + golang.org/x/net v0.4.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/tools v0.4.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -106,15 +105,15 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/apiextensions-apiserver v0.24.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.26.0 // indirect k8s.io/component-base v0.24.1 // indirect - k8s.io/klog/v2 v2.60.1 // indirect + k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect - sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index e38949d..3fbf403 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= @@ -154,8 +155,8 @@ github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMi github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -200,8 +201,8 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobuffalo/flect v0.2.5 h1:H6vvsv2an0lalEaCDRThvtBfmg44W/QHXBCYUXf/6S4= -github.com/gobuffalo/flect v0.2.5/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8= +github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk= +github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -264,8 +265,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -330,8 +332,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/intel/userspace-cni-network-plugin v0.0.0-20201014132026-387fb2be6a32 h1:jKU1kw+2zj9OPShnpQqnNKzKDDr+KgcVujegSoc6hro= github.com/intel/userspace-cni-network-plugin v0.0.0-20201014132026-387fb2be6a32/go.mod h1:quRf1vfJd4x3juI2y2mN67C4Om5/ghi4tAqHw+yVuGY= github.com/ishidawataru/sctp v0.0.0-20210707070123-9a39160e9062 h1:G1+wBT0dwjIrBdLy0MIG0i+E4CQxEnedHXdauJEIH6g= @@ -379,11 +382,12 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -434,15 +438,16 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.1.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE= +github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk= github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= @@ -521,8 +526,9 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -531,13 +537,16 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subgraph/libmacouflage v0.0.1 h1:DTzVhFxtANTflvNPhMLGP9c8NxSZzdK2JHIwDEdzX/g= github.com/subgraph/libmacouflage v0.0.1/go.mod h1:wTajnv9uXMczTL0k32UMV2oXzTsNDs0A5tfZT60J5oc= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -651,8 +660,9 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -706,8 +716,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -733,6 +743,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20170427041856-9ccfe848b9db/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -799,6 +810,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -809,11 +821,13 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -822,8 +836,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -891,12 +906,12 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 h1:hI3jKY4Hpf63ns040onEbB3dAkR/H/P83hw1TG8dD3Y= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= @@ -1048,8 +1063,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= @@ -1082,8 +1098,9 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= @@ -1091,8 +1108,9 @@ k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdi k8s.io/kubelet v0.24.1 h1:CLgXZ9kKDQoNQFSwKk6vUE5gXNaX1/s8VM8Oq/P5S+o= k8s.io/kubelet v0.24.1/go.mod h1:LShXfjNO1or7ktsorODSOu8+Kd5dHzWF3DtVLXeP3JE= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/containerized-data-importer-api v1.50.0 h1:O01F8L5K8qRLnkYICIfmAu0dU0P48jdO42uFPElht38= kubevirt.io/containerized-data-importer-api v1.50.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= @@ -1105,11 +1123,13 @@ sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLO sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= sigs.k8s.io/controller-tools v0.9.0 h1:b/vSEPpA8hiMiyzDfLbZdCn3hoAcy3/868OHhYtHY9w= sigs.k8s.io/controller-tools v0.9.0/go.mod h1:NUkn8FTV3Sad3wWpSK7dt/145qfuQ8CKJV6j4jHC5rM= -sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/Dockerfile b/hack/Dockerfile index 931cb4f..02483db 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -1,4 +1,7 @@ -FROM golang:1.19 +FROM golang:1.19-alpine + +RUN apk add --no-cache bash gcc git musl-dev libaio-dev +RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ WORKDIR /workspace diff --git a/hack/generate.sh b/hack/generate.sh index a5a8116..4890502 100755 --- a/hack/generate.sh +++ b/hack/generate.sh @@ -12,5 +12,7 @@ bash $GOPATH/src/k8s.io/code-generator/generate-groups.sh "deepcopy,client,infor controller-gen paths=./pkg/apis/... crd output:crd:artifacts:config=deploy/crd controller-gen paths=./cmd/virt-controller/... paths=./pkg/controller/... rbac:roleName=virt-controller output:rbac:artifacts:config=deploy/virt-controller webhook output:webhook:artifacts:config=deploy/virt-controller controller-gen paths=./cmd/virt-daemon/... paths=./pkg/daemon/... rbac:roleName=virt-daemon output:rbac:artifacts:config=deploy/virt-daemon +controller-gen paths=./cmd/lockspace-attacher/... rbac:roleName=lockspace-attacher output:rbac:artifacts:config=deploy/lockspace-attacher +controller-gen paths=./cmd/lockspace-detector/... rbac:roleName=lockspace-detector output:rbac:artifacts:config=deploy/lockspace-detector go generate ./... diff --git a/pkg/apis/virt/v1alpha1/register.go b/pkg/apis/virt/v1alpha1/register.go index 9c34c66..c17effc 100644 --- a/pkg/apis/virt/v1alpha1/register.go +++ b/pkg/apis/virt/v1alpha1/register.go @@ -35,6 +35,10 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VirtualMachineList{}, &VirtualMachineMigration{}, &VirtualMachineMigrationList{}, + &Lockspace{}, + &LockspaceList{}, + &Lock{}, + &LockList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/apis/virt/v1alpha1/types.go b/pkg/apis/virt/v1alpha1/types.go index 30765a4..b52f81a 100644 --- a/pkg/apis/virt/v1alpha1/types.go +++ b/pkg/apis/virt/v1alpha1/types.go @@ -1,6 +1,8 @@ package v1alpha1 import ( + "fmt" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,10 +35,12 @@ type VirtualMachineSpec struct { ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"` RunPolicy RunPolicy `json:"runPolicy,omitempty"` + EnableHA bool `json:"enableHA,omitempty"` Instance Instance `json:"instance"` Volumes []Volume `json:"volumes,omitempty"` Networks []Network `json:"networks,omitempty"` + Locks []string `json:"locks,omitempty"` } // +kubebuilder:validation:Enum=Always;RerunOnFailure;Once;Manual;Halted @@ -333,3 +337,95 @@ type VirtualMachineMigrationList struct { Items []VirtualMachineMigration `json:"items"` } + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:scope="Cluster" +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="StorageClass",type=string,JSONPath=`.spec.storageClassName` +// +kubebuilder:printcolumn:name="Ready",type=boolean,JSONPath=`.status.ready` + +type Lockspace struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LockspaceSpec `json:"spec,omitempty"` + Status LockspaceStatus `json:"status,omitempty"` +} + +func (ls *Lockspace) GeneratePVCName() string { + return fmt.Sprintf("lockspace-pvc-%s", ls.Name) +} + +func (ls *Lockspace) GenerateInitializerName() string { + return fmt.Sprintf("lockspace-initializer-%s", ls.Name) +} + +func (ls *Lockspace) GenerateDetectorName() string { + return fmt.Sprintf("lockspace-detector-%s", ls.Name) +} + +func (ls *Lockspace) GenerateAttacherName() string { + return fmt.Sprintf("lockspace-attacher-%s", ls.Name) +} + +type LockspaceSpec struct { + StorageClassName string `json:"storageClassName"` + // +kubebuilder:default=Filesystem + VolumeMode *corev1.PersistentVolumeMode `json:"volumeMode,omitempty"` + // The maximum number of Lock that can be held in a Lockspace. + // +kubebuilder:default=1000 + // +kubebuilder:validation:Maximum=16384 + // +kubebuilder:validation:Minimum=1 + MaxLocks int `json:"maxLocks,omitempty"` + // +kubebuilder:default=10 + // +kubebuilder:validation:Maximum=15 + // +kubebuilder:validation:Minimum=5 + IOTimeoutSeconds uint32 `json:"ioTimeoutSeconds,omitempty"` +} + +type LockspaceStatus struct { + Ready bool `json:"ready,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type LockspaceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []Lockspace `json:"items"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Lockspace",type=string,JSONPath=`.spec.lockspaceName` +// +kubebuilder:printcolumn:name="Ready",type=boolean,JSONPath=`.status.ready` + +type Lock struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LockSpec `json:"spec,omitempty"` + Status LockStatus `json:"status,omitempty"` +} + +type LockSpec struct { + LockspaceName string `json:"lockspaceName"` +} + +type LockStatus struct { + Ready bool `json:"ready,omitempty"` + Offset uint64 `json:"offset,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type LockList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []Lock `json:"items"` +} diff --git a/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go index 5b80684..8f8be78 100644 --- a/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go @@ -336,6 +336,197 @@ func (in *Kernel) DeepCopy() *Kernel { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Lock) DeepCopyInto(out *Lock) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Lock. +func (in *Lock) DeepCopy() *Lock { + if in == nil { + return nil + } + out := new(Lock) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Lock) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LockList) DeepCopyInto(out *LockList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Lock, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LockList. +func (in *LockList) DeepCopy() *LockList { + if in == nil { + return nil + } + out := new(LockList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LockList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LockSpec) DeepCopyInto(out *LockSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LockSpec. +func (in *LockSpec) DeepCopy() *LockSpec { + if in == nil { + return nil + } + out := new(LockSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LockStatus) DeepCopyInto(out *LockStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LockStatus. +func (in *LockStatus) DeepCopy() *LockStatus { + if in == nil { + return nil + } + out := new(LockStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Lockspace) DeepCopyInto(out *Lockspace) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Lockspace. +func (in *Lockspace) DeepCopy() *Lockspace { + if in == nil { + return nil + } + out := new(Lockspace) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Lockspace) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LockspaceList) DeepCopyInto(out *LockspaceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Lockspace, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LockspaceList. +func (in *LockspaceList) DeepCopy() *LockspaceList { + if in == nil { + return nil + } + out := new(LockspaceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LockspaceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LockspaceSpec) DeepCopyInto(out *LockspaceSpec) { + *out = *in + if in.VolumeMode != nil { + in, out := &in.VolumeMode, &out.VolumeMode + *out = new(v1.PersistentVolumeMode) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LockspaceSpec. +func (in *LockspaceSpec) DeepCopy() *LockspaceSpec { + if in == nil { + return nil + } + out := new(LockspaceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LockspaceStatus) DeepCopyInto(out *LockspaceStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LockspaceStatus. +func (in *LockspaceStatus) DeepCopy() *LockspaceStatus { + if in == nil { + return nil + } + out := new(LockspaceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Memory) DeepCopyInto(out *Memory) { *out = *in @@ -651,6 +842,11 @@ func (in *VirtualMachineSpec) DeepCopyInto(out *VirtualMachineSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Locks != nil { + in, out := &in.Locks, &out.Locks + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/controller/lockspace_controller.go b/pkg/controller/lockspace_controller.go new file mode 100644 index 0000000..e5744ce --- /dev/null +++ b/pkg/controller/lockspace_controller.go @@ -0,0 +1,549 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "reflect" + "strconv" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" +) + +const ( + LockspaceProtectionFinalizer = "virtink.smartx.com/lockspace-protection" + LockspaceDetectorProtectionFinalizer = "virtink.smartx.com/lockspace-detector-protection" + LockspaceAttacherProtectionFinalizer = "virtink.smartx.com/lockspace-attacher-protection" + + VirtinkNamespace = "virtink-system" +) + +type LockspaceReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + DetectorImageName string + AttacherImageName string + InitializerImageName string +} + +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=lockspaces,verbs=get;list;watch;update +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=lockspaces/status,verbs=get;update +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=locks,verbs=get;list;watch;update;delete +//+kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;delete +//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;delete +//+kubebuilder:rbac:groups="apps",resources=daemonsets,verbs=get;list;watch;create;update;delete + +func (r *LockspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var lockspace virtv1alpha1.Lockspace + if err := r.Get(ctx, req.NamespacedName, &lockspace); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + status := lockspace.Status.DeepCopy() + if err := r.reconcile(ctx, &lockspace); err != nil { + reconcileErr := reconcileError{} + if errors.As(err, &reconcileErr) { + return reconcileErr.Result, nil + } + r.Recorder.Eventf(&lockspace, corev1.EventTypeWarning, "FailedReconcile", "Failed to reconcile Lockspace: %s", err) + return ctrl.Result{}, err + } + + if !reflect.DeepEqual(lockspace.Status, status) { + if err := r.Status().Update(ctx, &lockspace); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("update Lockspace status: %s", err) + } + } + + return reconcile.Result{}, nil +} + +func (r *LockspaceReconciler) reconcile(ctx context.Context, lockspace *virtv1alpha1.Lockspace) error { + if lockspace.DeletionTimestamp != nil { + if err := r.cleanupLockspace(ctx, lockspace); err != nil { + return fmt.Errorf("cleanup Lockspace: %s", err) + } + } + + if !controllerutil.ContainsFinalizer(lockspace, LockspaceProtectionFinalizer) { + controllerutil.AddFinalizer(lockspace, LockspaceProtectionFinalizer) + return r.Client.Update(ctx, lockspace) + } + + if !lockspace.Status.Ready { + if err := r.reconcileNotReadyLockspace(ctx, lockspace); err != nil { + return fmt.Errorf("reconcile not ready Lockspace: %s", err) + } + } else { + if err := r.reconcileReadyLockspace(ctx, lockspace); err != nil { + return fmt.Errorf("reconcile ready Lockspace: %s", err) + } + } + return nil +} + +func (r *LockspaceReconciler) cleanupLockspace(ctx context.Context, lockspace *virtv1alpha1.Lockspace) error { + var lockList virtv1alpha1.LockList + lockSelector := client.MatchingFields{".spec.lockspaceName": lockspace.Name} + if err := r.Client.List(ctx, &lockList, lockSelector); err != nil { + return fmt.Errorf("list Lock: %s", err) + } + if len(lockList.Items) != 0 { + for _, lock := range lockList.Items { + if err := r.Client.Delete(ctx, &lock); err != nil { + return fmt.Errorf("delete Lock %q: %s", namespacedName(&lock), err) + } + } + return reconcileError{ctrl.Result{RequeueAfter: time.Second}} + } + + detector, err := r.getLockspaceDetectorOrNil(ctx, lockspace) + if err != nil { + return fmt.Errorf("get Lockspace detector: %s", err) + } + if detector != nil { + if controllerutil.ContainsFinalizer(detector, LockspaceDetectorProtectionFinalizer) { + controllerutil.RemoveFinalizer(detector, LockspaceDetectorProtectionFinalizer) + if err := r.Client.Update(ctx, detector); err != nil { + return fmt.Errorf("update Lockspace detector: %s", err) + } + } + + if r.Client.Delete(ctx, detector); err != nil { + return fmt.Errorf("delete Lockspace detector: %s", err) + } + return reconcileError{ctrl.Result{Requeue: true}} + } + + attacher, err := r.getLockspaceAttacherOrNil(ctx, lockspace) + if err != nil { + return fmt.Errorf("get Lockspace attacher: %s", err) + } + if attacher != nil { + if controllerutil.ContainsFinalizer(attacher, LockspaceAttacherProtectionFinalizer) { + controllerutil.RemoveFinalizer(attacher, LockspaceAttacherProtectionFinalizer) + if err := r.Client.Update(ctx, attacher); err != nil { + return fmt.Errorf("update Lockspace attacher: %s", err) + } + } + } + + if controllerutil.ContainsFinalizer(lockspace, LockspaceProtectionFinalizer) { + controllerutil.RemoveFinalizer(lockspace, LockspaceProtectionFinalizer) + if err := r.Client.Update(ctx, lockspace); err != nil { + return fmt.Errorf("update Lockspace: %s", err) + } + } + + return nil +} + +func (r *LockspaceReconciler) getLockspaceDetectorOrNil(ctx context.Context, lockspace *virtv1alpha1.Lockspace) (*appsv1.DaemonSet, error) { + detectorKey := types.NamespacedName{ + Namespace: VirtinkNamespace, + Name: lockspace.GenerateDetectorName(), + } + var detector appsv1.DaemonSet + if err := r.Client.Get(ctx, detectorKey, &detector); err != nil { + return nil, client.IgnoreNotFound(err) + } + + if !metav1.IsControlledBy(&detector, lockspace) { + return nil, fmt.Errorf("detector %q is not controlled by Lockspace %q", namespacedName(&detector), namespacedName(lockspace)) + } + return &detector, nil +} + +func (r *LockspaceReconciler) getLockspaceAttacherOrNil(ctx context.Context, lockspace *virtv1alpha1.Lockspace) (*appsv1.DaemonSet, error) { + attacherKey := types.NamespacedName{ + Namespace: VirtinkNamespace, + Name: lockspace.GenerateAttacherName(), + } + var attacher appsv1.DaemonSet + if err := r.Client.Get(ctx, attacherKey, &attacher); err != nil { + return nil, client.IgnoreNotFound(err) + } + + if !metav1.IsControlledBy(&attacher, lockspace) { + return nil, fmt.Errorf("attacher %q is not controlled by Lockspace %q", namespacedName(&attacher), namespacedName(lockspace)) + } + return &attacher, nil +} + +func (r *LockspaceReconciler) reconcileNotReadyLockspace(ctx context.Context, lockspace *virtv1alpha1.Lockspace) error { + pvc, err := r.getLockspacePVCOrNil(ctx, lockspace) + if err != nil { + return fmt.Errorf("get Lockspace PVC: %s", err) + } + if pvc == nil { + if err := r.createLockspacePVC(ctx, lockspace); err != nil { + return fmt.Errorf("create Lockspace PVC: %s", err) + } + return nil + } + + if pvc.Status.Phase == corev1.ClaimBound { + initializer, err := r.getLockspaceInitializerOrNil(ctx, lockspace) + if err != nil { + return fmt.Errorf("get Lockspace initializer: %s", err) + } + if initializer == nil { + if err := r.createLockspaceInitializer(ctx, lockspace, pvc); err != nil { + return fmt.Errorf("create Lockspace initializer: %s", err) + } + return nil + } + + if initializer.Status.Phase == corev1.PodSucceeded { + if err := r.Client.Delete(ctx, initializer); err != nil { + return fmt.Errorf("delete Lockspace initializer: %s", err) + } + lockspace.Status.Ready = true + } + } + + return nil +} + +func (r *LockspaceReconciler) getLockspacePVCOrNil(ctx context.Context, lockspace *virtv1alpha1.Lockspace) (*corev1.PersistentVolumeClaim, error) { + pvcKey := types.NamespacedName{ + Namespace: VirtinkNamespace, + Name: lockspace.GeneratePVCName(), + } + var pvc corev1.PersistentVolumeClaim + if err := r.Client.Get(ctx, pvcKey, &pvc); err != nil { + return nil, client.IgnoreNotFound(err) + } + + if !metav1.IsControlledBy(&pvc, lockspace) { + return nil, fmt.Errorf("PVC %q is not controlled by Lockspace %q", namespacedName(&pvc), namespacedName(lockspace)) + } + + return &pvc, nil +} + +func (r *LockspaceReconciler) createLockspacePVC(ctx context.Context, lockspace *virtv1alpha1.Lockspace) error { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: VirtinkNamespace, + Name: lockspace.GeneratePVCName(), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: func() *string { str := lockspace.Spec.StorageClassName; return &str }(), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + }, + } + + if pvc.Spec.Resources.Requests == nil { + pvc.Spec.Resources.Requests = corev1.ResourceList{} + } + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse(fmt.Sprintf("%dMi", 3+lockspace.Spec.MaxLocks)) + + volumeMode := corev1.PersistentVolumeFilesystem + if lockspace.Spec.VolumeMode != nil { + volumeMode = *lockspace.Spec.VolumeMode + } + pvc.Spec.VolumeMode = &volumeMode + + if err := controllerutil.SetControllerReference(lockspace, pvc, r.Scheme); err != nil { + return fmt.Errorf("set PVC controller reference: %s", err) + } + return r.Client.Create(ctx, pvc) +} + +func (r *LockspaceReconciler) getLockspaceInitializerOrNil(ctx context.Context, lockspace *virtv1alpha1.Lockspace) (*corev1.Pod, error) { + podKey := types.NamespacedName{ + Namespace: VirtinkNamespace, + Name: lockspace.GenerateInitializerName(), + } + var pod corev1.Pod + if err := r.Client.Get(ctx, podKey, &pod); err != nil { + return nil, client.IgnoreNotFound(err) + } + + if !metav1.IsControlledBy(&pod, lockspace) { + return nil, fmt.Errorf("pod %q is not controlled by Lockspace %q", namespacedName(&pod), namespacedName(lockspace)) + } + return &pod, nil +} + +func (r *LockspaceReconciler) createLockspaceInitializer(ctx context.Context, lockspace *virtv1alpha1.Lockspace, pvc *corev1.PersistentVolumeClaim) error { + initializerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: lockspace.GenerateInitializerName(), + Namespace: VirtinkNamespace, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{{ + Name: "lockspace-initializer", + Image: r.InitializerImageName, + Env: []corev1.EnvVar{{ + Name: "LOCKSPACE_NAME", + Value: lockspace.Name, + }, { + Name: "IO_TIMEOUT_SECONDS", + Value: fmt.Sprintf("%d", lockspace.Spec.IOTimeoutSeconds), + }}, + }}, + }, + } + initializerPod.Spec.Volumes = append(initializerPod.Spec.Volumes, corev1.Volume{ + Name: "lockspace-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + }, + }, + }) + container := &initializerPod.Spec.Containers[0] + switch *pvc.Spec.VolumeMode { + case corev1.PersistentVolumeFilesystem: + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: "lockspace-volume", + MountPath: filepath.Join("/var/lib/sanlock/", lockspace.Name), + }) + case corev1.PersistentVolumeBlock: + container.VolumeDevices = append(container.VolumeDevices, corev1.VolumeDevice{ + Name: "lockspace-volume", + DevicePath: filepath.Join("/var/lib/sanlock", lockspace.Name, "leases"), + }) + } + + if err := controllerutil.SetControllerReference(lockspace, initializerPod, r.Scheme); err != nil { + return fmt.Errorf("set initializer controller reference: %s", err) + } + + return r.Client.Create(ctx, initializerPod) +} + +func (r *LockspaceReconciler) reconcileReadyLockspace(ctx context.Context, lockspace *virtv1alpha1.Lockspace) error { + pvc, err := r.getLockspacePVCOrNil(ctx, lockspace) + if err != nil { + return fmt.Errorf("get Lockspace PVC: %s", err) + } + + attacher, err := r.getLockspaceAttacherOrNil(ctx, lockspace) + if err != nil { + return fmt.Errorf("get Lockspace attacher: %s", err) + } + if attacher == nil { + if err := r.createLockspaceAttacher(ctx, lockspace, pvc); err != nil { + return fmt.Errorf("create Lockspace attacher: %s", err) + } + } + + detector, err := r.getLockspaceDetectorOrNil(ctx, lockspace) + if err != nil { + return fmt.Errorf("get Lockspace detector: %s", err) + } + if detector == nil { + if err := r.createLockspaceDetector(ctx, lockspace); err != nil { + return fmt.Errorf("create Lockspace detector: %s", err) + } + } + return nil +} + +func (r *LockspaceReconciler) createLockspaceAttacher(ctx context.Context, lockspace *virtv1alpha1.Lockspace, pvc *corev1.PersistentVolumeClaim) error { + matchLabels := make(map[string]string) + matchLabels["name"] = lockspace.GenerateAttacherName() + + attacher := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: VirtinkNamespace, + Name: lockspace.GenerateAttacherName(), + Finalizers: []string{LockspaceAttacherProtectionFinalizer}, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: matchLabels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: matchLabels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "lockspace-attacher", + Containers: []corev1.Container{{ + Name: "lockspace-attacher", + Image: r.AttacherImageName, + SecurityContext: &corev1.SecurityContext{ + Privileged: func() *bool { b := true; return &b }(), + }, + Env: []corev1.EnvVar{{ + Name: "LOCKSPACE_NAME", + Value: lockspace.Name, + }, { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "sanlock-run-dir", + MountPath: "/var/run/sanlock", + }, { + Name: "sanlock-lib-dir", + MountPath: "/var/lib/sanlock", + MountPropagation: func() *corev1.MountPropagationMode { p := corev1.MountPropagationBidirectional; return &p }(), + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "sanlock-run-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/run/sanlock", + }, + }, + }, { + Name: "sanlock-lib-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/sanlock", + }, + }, + }}, + }, + }, + }, + } + + pod := &attacher.Spec.Template + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "lockspace-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + }, + }, + }) + container := &pod.Spec.Containers[0] + switch *pvc.Spec.VolumeMode { + case corev1.PersistentVolumeFilesystem: + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: "lockspace-volume", + MountPath: filepath.Join("/var/lib/sanlock", lockspace.Name), + }) + case corev1.PersistentVolumeBlock: + container.VolumeDevices = append(container.VolumeDevices, corev1.VolumeDevice{ + Name: "lockspace-volume", + DevicePath: filepath.Join("/var/lib/sanlock", lockspace.Name, "leases"), + }) + } + + if err := controllerutil.SetControllerReference(lockspace, attacher, r.Scheme); err != nil { + return fmt.Errorf("set attacher controller reference: %s", err) + } + + return r.Client.Create(ctx, attacher) +} + +func (r *LockspaceReconciler) createLockspaceDetector(ctx context.Context, lockspace *virtv1alpha1.Lockspace) error { + matchLabels := make(map[string]string) + matchLabels["name"] = lockspace.GenerateDetectorName() + + detector := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: VirtinkNamespace, + Name: lockspace.GenerateDetectorName(), + Finalizers: []string{LockspaceDetectorProtectionFinalizer}, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: matchLabels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: matchLabels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "lockspace-detector", + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: fmt.Sprintf("virtink.smartx.com/dead-lockspace-%s", lockspace.Name), + Operator: corev1.NodeSelectorOpDoesNotExist, + }}, + }}, + }, + }, + }, + Containers: []corev1.Container{{ + Name: "lockspace-detector", + Image: r.DetectorImageName, + Env: []corev1.EnvVar{{ + Name: "LOCKSPACE_NAME", + Value: lockspace.Name, + }, { + Name: "IO_TIMEOUT", + Value: strconv.Itoa(int(lockspace.Spec.IOTimeoutSeconds)), + }}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "sanlock-run-dir", + MountPath: "/var/run/sanlock", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "sanlock-run-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/run/sanlock", + }, + }, + }}, + }, + }, + }, + } + + if err := controllerutil.SetControllerReference(lockspace, detector, r.Scheme); err != nil { + return fmt.Errorf("set detector controller reference: %s", err) + } + + return r.Client.Create(ctx, detector) +} + +func (r *LockspaceReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &virtv1alpha1.Lock{}, ".spec.lockspaceName", func(obj client.Object) []string { + lock := obj.(*virtv1alpha1.Lock) + return []string{lock.Spec.LockspaceName} + }); err != nil { + return fmt.Errorf("index Lock by lockspaceName: %s", err) + } + + return ctrl.NewControllerManagedBy(mgr). + For(&virtv1alpha1.Lockspace{}). + Owns(&corev1.PersistentVolumeClaim{}). + Owns(&corev1.Pod{}). + Complete(r) +} + +func namespacedName(obj metav1.ObjectMetaAccessor) types.NamespacedName { + meta := obj.GetObjectMeta() + return types.NamespacedName{ + Namespace: meta.GetNamespace(), + Name: meta.GetName(), + } +} diff --git a/pkg/controller/vm_controller.go b/pkg/controller/vm_controller.go index fdd4a37..2f182e9 100644 --- a/pkg/controller/vm_controller.go +++ b/pkg/controller/vm_controller.go @@ -53,6 +53,7 @@ type VMReconciler struct { // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch // +kubebuilder:rbac:groups=cdi.kubevirt.io,resources=datavolumes,verbs=get;list;watch // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +//+kubebuilder:rbac:groups=virt.virtink.smartx.com,resources=locks,verbs=get;list;watch;update func (r *VMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var vm virtv1alpha1.VirtualMachine @@ -123,6 +124,26 @@ func (r *VMReconciler) reconcile(ctx context.Context, vm *virtv1alpha1.VirtualMa vm.Status.VMPodName = names.SimpleNameGenerator.GenerateName(fmt.Sprintf("vm-%s-", vm.Name)) vm.Status.Phase = virtv1alpha1.VirtualMachineScheduling case virtv1alpha1.VirtualMachineScheduling, virtv1alpha1.VirtualMachineScheduled: + if vm.Status.Phase == virtv1alpha1.VirtualMachinePending { + for _, l := range vm.Spec.Locks { + var lock virtv1alpha1.Lock + lockKey := types.NamespacedName{ + Name: l, + Namespace: vm.Namespace, + } + if err := r.Get(ctx, lockKey, &lock); err != nil { + if apierrors.IsNotFound(err) { + return reconcileError{Result: ctrl.Result{RequeueAfter: 30 * time.Second}} + } else { + return fmt.Errorf("get VM Lock: %s", err) + } + } + if lock.DeletionTimestamp != nil || !lock.Status.Ready { + return reconcileError{Result: ctrl.Result{RequeueAfter: 30 * time.Second}} + } + } + } + var vmPod corev1.Pod vmPodKey := types.NamespacedName{ Name: vm.Status.VMPodName, @@ -774,6 +795,77 @@ func (r *VMReconciler) buildVMPod(ctx context.Context, vm *virtv1alpha1.VirtualM vmPod.Annotations["k8s.v1.cni.cncf.io/networks"] = string(networksJSON) } + lockspaces := make(map[string]bool) + resources := []string{} + for _, l := range vm.Spec.Locks { + var lock virtv1alpha1.Lock + lockKey := types.NamespacedName{ + Name: l, + Namespace: vm.Namespace, + } + if err := r.Get(ctx, lockKey, &lock); err != nil { + return nil, fmt.Errorf("get VM Lock: %s", err) + } + lsName := lock.Spec.LockspaceName + lockspaces[lsName] = true + resources = append(resources, fmt.Sprintf("%s:%s:/var/lib/sanlock/%s/leases:%d", lsName, lock.Name, lsName, lock.Status.Offset)) + } + if len(vm.Spec.Locks) > 0 { + vmPod.Spec.Containers[0].VolumeMounts = append(vmPod.Spec.Containers[0].VolumeMounts, []corev1.VolumeMount{{ + Name: "sanlock-run-dir", + MountPath: "/var/run/sanlock", + }, { + Name: "sanlock-lib-dir", + MountPath: "/var/lib/sanlock", + MountPropagation: func() *corev1.MountPropagationMode { p := corev1.MountPropagationHostToContainer; return &p }(), + }}...) + + vmPod.Spec.Volumes = append(vmPod.Spec.Volumes, []corev1.Volume{{ + Name: "sanlock-run-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/run/sanlock", + }, + }, + }, { + Name: "sanlock-lib-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/sanlock", + }, + }, + }}...) + + if vmPod.Spec.Affinity == nil { + vmPod.Spec.Affinity = &corev1.Affinity{} + } + affinity := vmPod.Spec.Affinity + if affinity.NodeAffinity == nil { + affinity.NodeAffinity = &corev1.NodeAffinity{} + } + nodeAffinity := affinity.NodeAffinity + if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} + } + nodeSelector := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + if nodeSelector.NodeSelectorTerms == nil { + nodeSelector.NodeSelectorTerms = []corev1.NodeSelectorTerm{} + } + for ls := range lockspaces { + nodeSelector.NodeSelectorTerms = append(nodeSelector.NodeSelectorTerms, corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: fmt.Sprintf("virtink.smartx.com/dead-lockspace-%s", ls), + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}, + ) + } + + vmPod.Spec.Containers[0].Env = append(vmPod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: "LOCKSPACE_RESOURCE", + Value: strings.Join(resources, " "), + }) + } + vmJSON, err := json.Marshal(vm) if err != nil { return nil, fmt.Errorf("marshal VM: %s", err) @@ -1188,6 +1280,16 @@ func (r *VMReconciler) calculateMigratableCondition(ctx context.Context, vm *vir }, nil } + // TODO: make VM with locks migratable + if len(vm.Spec.Locks) > 0 { + return &metav1.Condition{ + Type: string(virtv1alpha1.VirtualMachineMigratable), + Status: metav1.ConditionFalse, + Reason: "HANotMigratable", + Message: "migration is disabled when VM has locks", + }, nil + } + return &metav1.Condition{ Type: string(virtv1alpha1.VirtualMachineMigratable), Status: metav1.ConditionTrue, diff --git a/pkg/controller/vm_webhook.go b/pkg/controller/vm_webhook.go index 16fe869..c14a0ec 100644 --- a/pkg/controller/vm_webhook.go +++ b/pkg/controller/vm_webhook.go @@ -316,6 +316,16 @@ func ValidateVMSpec(ctx context.Context, spec *virtv1alpha1.VirtualMachineSpec, errs = append(errs, ValidateNetwork(ctx, &network, fieldPath)...) } + if spec.EnableHA { + if spec.RunPolicy != virtv1alpha1.RunPolicyAlways && spec.RunPolicy != virtv1alpha1.RunPolicyRerunOnFailure { + errs = append(errs, field.Forbidden(fieldPath.Child("enableHA"), "only \"RerunOnFailure\" and \"Always\" run policy are allowed for VM HA")) + } + } + + if len(spec.Locks) > 8 { + errs = append(errs, field.Forbidden(fieldPath.Child("locks"), "may not use more than 8 Locks for VM")) + } + return errs } diff --git a/pkg/controller/vm_webhook_test.go b/pkg/controller/vm_webhook_test.go index 963765a..753434a 100644 --- a/pkg/controller/vm_webhook_test.go +++ b/pkg/controller/vm_webhook_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -472,6 +473,23 @@ func TestValidateVM(t *testing.T) { return vm }(), invalidFields: []string{"spec.networks[0].multus"}, + }, { + vm: func() *virtv1alpha1.VirtualMachine { + vm := validVM.DeepCopy() + vm.Spec.EnableHA = true + vm.Spec.RunPolicy = virtv1alpha1.RunPolicyOnce + return vm + }(), + invalidFields: []string{"spec.enableHA"}, + }, { + vm: func() *virtv1alpha1.VirtualMachine { + vm := validVM.DeepCopy() + for i := 0; i < 9; i++ { + vm.Spec.Locks = append(vm.Spec.Locks, strconv.Itoa(i)) + } + return vm + }(), + invalidFields: []string{"spec.locks"}, }} for _, tc := range tests { diff --git a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lock.go b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lock.go new file mode 100644 index 0000000..f1cc86d --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lock.go @@ -0,0 +1,126 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeLocks implements LockInterface +type FakeLocks struct { + Fake *FakeVirtV1alpha1 + ns string +} + +var locksResource = schema.GroupVersionResource{Group: "virt.virtink.smartx.com", Version: "v1alpha1", Resource: "locks"} + +var locksKind = schema.GroupVersionKind{Group: "virt.virtink.smartx.com", Version: "v1alpha1", Kind: "Lock"} + +// Get takes name of the lock, and returns the corresponding lock object, and an error if there is any. +func (c *FakeLocks) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Lock, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(locksResource, c.ns, name), &v1alpha1.Lock{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lock), err +} + +// List takes label and field selectors, and returns the list of Locks that match those selectors. +func (c *FakeLocks) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.LockList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(locksResource, locksKind, c.ns, opts), &v1alpha1.LockList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.LockList{ListMeta: obj.(*v1alpha1.LockList).ListMeta} + for _, item := range obj.(*v1alpha1.LockList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested locks. +func (c *FakeLocks) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(locksResource, c.ns, opts)) + +} + +// Create takes the representation of a lock and creates it. Returns the server's representation of the lock, and an error, if there is any. +func (c *FakeLocks) Create(ctx context.Context, lock *v1alpha1.Lock, opts v1.CreateOptions) (result *v1alpha1.Lock, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(locksResource, c.ns, lock), &v1alpha1.Lock{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lock), err +} + +// Update takes the representation of a lock and updates it. Returns the server's representation of the lock, and an error, if there is any. +func (c *FakeLocks) Update(ctx context.Context, lock *v1alpha1.Lock, opts v1.UpdateOptions) (result *v1alpha1.Lock, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(locksResource, c.ns, lock), &v1alpha1.Lock{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lock), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeLocks) UpdateStatus(ctx context.Context, lock *v1alpha1.Lock, opts v1.UpdateOptions) (*v1alpha1.Lock, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(locksResource, "status", c.ns, lock), &v1alpha1.Lock{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lock), err +} + +// Delete takes name of the lock and deletes it. Returns an error if one occurs. +func (c *FakeLocks) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(locksResource, c.ns, name, opts), &v1alpha1.Lock{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeLocks) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(locksResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.LockList{}) + return err +} + +// Patch applies the patch and returns the patched lock. +func (c *FakeLocks) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Lock, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(locksResource, c.ns, name, pt, data, subresources...), &v1alpha1.Lock{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lock), err +} diff --git a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lockspace.go b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lockspace.go new file mode 100644 index 0000000..896aada --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_lockspace.go @@ -0,0 +1,117 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeLockspaces implements LockspaceInterface +type FakeLockspaces struct { + Fake *FakeVirtV1alpha1 +} + +var lockspacesResource = schema.GroupVersionResource{Group: "virt.virtink.smartx.com", Version: "v1alpha1", Resource: "lockspaces"} + +var lockspacesKind = schema.GroupVersionKind{Group: "virt.virtink.smartx.com", Version: "v1alpha1", Kind: "Lockspace"} + +// Get takes name of the lockspace, and returns the corresponding lockspace object, and an error if there is any. +func (c *FakeLockspaces) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Lockspace, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(lockspacesResource, name), &v1alpha1.Lockspace{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lockspace), err +} + +// List takes label and field selectors, and returns the list of Lockspaces that match those selectors. +func (c *FakeLockspaces) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.LockspaceList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(lockspacesResource, lockspacesKind, opts), &v1alpha1.LockspaceList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.LockspaceList{ListMeta: obj.(*v1alpha1.LockspaceList).ListMeta} + for _, item := range obj.(*v1alpha1.LockspaceList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested lockspaces. +func (c *FakeLockspaces) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(lockspacesResource, opts)) +} + +// Create takes the representation of a lockspace and creates it. Returns the server's representation of the lockspace, and an error, if there is any. +func (c *FakeLockspaces) Create(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.CreateOptions) (result *v1alpha1.Lockspace, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(lockspacesResource, lockspace), &v1alpha1.Lockspace{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lockspace), err +} + +// Update takes the representation of a lockspace and updates it. Returns the server's representation of the lockspace, and an error, if there is any. +func (c *FakeLockspaces) Update(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.UpdateOptions) (result *v1alpha1.Lockspace, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(lockspacesResource, lockspace), &v1alpha1.Lockspace{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lockspace), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeLockspaces) UpdateStatus(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.UpdateOptions) (*v1alpha1.Lockspace, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(lockspacesResource, "status", lockspace), &v1alpha1.Lockspace{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lockspace), err +} + +// Delete takes name of the lockspace and deletes it. Returns an error if one occurs. +func (c *FakeLockspaces) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(lockspacesResource, name, opts), &v1alpha1.Lockspace{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeLockspaces) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(lockspacesResource, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.LockspaceList{}) + return err +} + +// Patch applies the patch and returns the patched lockspace. +func (c *FakeLockspaces) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Lockspace, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(lockspacesResource, name, pt, data, subresources...), &v1alpha1.Lockspace{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Lockspace), err +} diff --git a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_virt_client.go b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_virt_client.go index b5de326..38b4720 100644 --- a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_virt_client.go +++ b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/fake/fake_virt_client.go @@ -12,6 +12,14 @@ type FakeVirtV1alpha1 struct { *testing.Fake } +func (c *FakeVirtV1alpha1) Locks(namespace string) v1alpha1.LockInterface { + return &FakeLocks{c, namespace} +} + +func (c *FakeVirtV1alpha1) Lockspaces() v1alpha1.LockspaceInterface { + return &FakeLockspaces{c} +} + func (c *FakeVirtV1alpha1) VirtualMachines(namespace string) v1alpha1.VirtualMachineInterface { return &FakeVirtualMachines{c, namespace} } diff --git a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/generated_expansion.go index 248e348..fe58206 100644 --- a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/generated_expansion.go @@ -2,6 +2,10 @@ package v1alpha1 +type LockExpansion interface{} + +type LockspaceExpansion interface{} + type VirtualMachineExpansion interface{} type VirtualMachineMigrationExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/lock.go b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/lock.go new file mode 100644 index 0000000..82fdb65 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/lock.go @@ -0,0 +1,179 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + scheme "github.com/smartxworks/virtink/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// LocksGetter has a method to return a LockInterface. +// A group's client should implement this interface. +type LocksGetter interface { + Locks(namespace string) LockInterface +} + +// LockInterface has methods to work with Lock resources. +type LockInterface interface { + Create(ctx context.Context, lock *v1alpha1.Lock, opts v1.CreateOptions) (*v1alpha1.Lock, error) + Update(ctx context.Context, lock *v1alpha1.Lock, opts v1.UpdateOptions) (*v1alpha1.Lock, error) + UpdateStatus(ctx context.Context, lock *v1alpha1.Lock, opts v1.UpdateOptions) (*v1alpha1.Lock, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.Lock, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.LockList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Lock, err error) + LockExpansion +} + +// locks implements LockInterface +type locks struct { + client rest.Interface + ns string +} + +// newLocks returns a Locks +func newLocks(c *VirtV1alpha1Client, namespace string) *locks { + return &locks{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the lock, and returns the corresponding lock object, and an error if there is any. +func (c *locks) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Lock, err error) { + result = &v1alpha1.Lock{} + err = c.client.Get(). + Namespace(c.ns). + Resource("locks"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of Locks that match those selectors. +func (c *locks) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.LockList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.LockList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("locks"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested locks. +func (c *locks) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("locks"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a lock and creates it. Returns the server's representation of the lock, and an error, if there is any. +func (c *locks) Create(ctx context.Context, lock *v1alpha1.Lock, opts v1.CreateOptions) (result *v1alpha1.Lock, err error) { + result = &v1alpha1.Lock{} + err = c.client.Post(). + Namespace(c.ns). + Resource("locks"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lock). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a lock and updates it. Returns the server's representation of the lock, and an error, if there is any. +func (c *locks) Update(ctx context.Context, lock *v1alpha1.Lock, opts v1.UpdateOptions) (result *v1alpha1.Lock, err error) { + result = &v1alpha1.Lock{} + err = c.client.Put(). + Namespace(c.ns). + Resource("locks"). + Name(lock.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lock). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *locks) UpdateStatus(ctx context.Context, lock *v1alpha1.Lock, opts v1.UpdateOptions) (result *v1alpha1.Lock, err error) { + result = &v1alpha1.Lock{} + err = c.client.Put(). + Namespace(c.ns). + Resource("locks"). + Name(lock.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lock). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the lock and deletes it. Returns an error if one occurs. +func (c *locks) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("locks"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *locks) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("locks"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched lock. +func (c *locks) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Lock, err error) { + result = &v1alpha1.Lock{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("locks"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/lockspace.go b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/lockspace.go new file mode 100644 index 0000000..9f0530d --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/lockspace.go @@ -0,0 +1,168 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + scheme "github.com/smartxworks/virtink/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// LockspacesGetter has a method to return a LockspaceInterface. +// A group's client should implement this interface. +type LockspacesGetter interface { + Lockspaces() LockspaceInterface +} + +// LockspaceInterface has methods to work with Lockspace resources. +type LockspaceInterface interface { + Create(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.CreateOptions) (*v1alpha1.Lockspace, error) + Update(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.UpdateOptions) (*v1alpha1.Lockspace, error) + UpdateStatus(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.UpdateOptions) (*v1alpha1.Lockspace, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.Lockspace, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.LockspaceList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Lockspace, err error) + LockspaceExpansion +} + +// lockspaces implements LockspaceInterface +type lockspaces struct { + client rest.Interface +} + +// newLockspaces returns a Lockspaces +func newLockspaces(c *VirtV1alpha1Client) *lockspaces { + return &lockspaces{ + client: c.RESTClient(), + } +} + +// Get takes name of the lockspace, and returns the corresponding lockspace object, and an error if there is any. +func (c *lockspaces) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.Lockspace, err error) { + result = &v1alpha1.Lockspace{} + err = c.client.Get(). + Resource("lockspaces"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of Lockspaces that match those selectors. +func (c *lockspaces) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.LockspaceList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.LockspaceList{} + err = c.client.Get(). + Resource("lockspaces"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested lockspaces. +func (c *lockspaces) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("lockspaces"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a lockspace and creates it. Returns the server's representation of the lockspace, and an error, if there is any. +func (c *lockspaces) Create(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.CreateOptions) (result *v1alpha1.Lockspace, err error) { + result = &v1alpha1.Lockspace{} + err = c.client.Post(). + Resource("lockspaces"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lockspace). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a lockspace and updates it. Returns the server's representation of the lockspace, and an error, if there is any. +func (c *lockspaces) Update(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.UpdateOptions) (result *v1alpha1.Lockspace, err error) { + result = &v1alpha1.Lockspace{} + err = c.client.Put(). + Resource("lockspaces"). + Name(lockspace.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lockspace). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *lockspaces) UpdateStatus(ctx context.Context, lockspace *v1alpha1.Lockspace, opts v1.UpdateOptions) (result *v1alpha1.Lockspace, err error) { + result = &v1alpha1.Lockspace{} + err = c.client.Put(). + Resource("lockspaces"). + Name(lockspace.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lockspace). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the lockspace and deletes it. Returns an error if one occurs. +func (c *lockspaces) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Resource("lockspaces"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *lockspaces) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("lockspaces"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched lockspace. +func (c *lockspaces) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.Lockspace, err error) { + result = &v1alpha1.Lockspace{} + err = c.client.Patch(pt). + Resource("lockspaces"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/virt_client.go b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/virt_client.go index 7cccd32..0660ec6 100644 --- a/pkg/generated/clientset/versioned/typed/virt/v1alpha1/virt_client.go +++ b/pkg/generated/clientset/versioned/typed/virt/v1alpha1/virt_client.go @@ -12,6 +12,8 @@ import ( type VirtV1alpha1Interface interface { RESTClient() rest.Interface + LocksGetter + LockspacesGetter VirtualMachinesGetter VirtualMachineMigrationsGetter } @@ -21,6 +23,14 @@ type VirtV1alpha1Client struct { restClient rest.Interface } +func (c *VirtV1alpha1Client) Locks(namespace string) LockInterface { + return newLocks(c, namespace) +} + +func (c *VirtV1alpha1Client) Lockspaces() LockspaceInterface { + return newLockspaces(c) +} + func (c *VirtV1alpha1Client) VirtualMachines(namespace string) VirtualMachineInterface { return newVirtualMachines(c, namespace) } diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index fdc2448..92680b8 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -37,6 +37,10 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=virt.virtink.smartx.com, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("locks"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Virt().V1alpha1().Locks().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("lockspaces"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Virt().V1alpha1().Lockspaces().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("virtualmachines"): return &genericInformer{resource: resource.GroupResource(), informer: f.Virt().V1alpha1().VirtualMachines().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("virtualmachinemigrations"): diff --git a/pkg/generated/informers/externalversions/virt/v1alpha1/interface.go b/pkg/generated/informers/externalversions/virt/v1alpha1/interface.go index 1bd9b5d..4885521 100644 --- a/pkg/generated/informers/externalversions/virt/v1alpha1/interface.go +++ b/pkg/generated/informers/externalversions/virt/v1alpha1/interface.go @@ -8,6 +8,10 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // Locks returns a LockInformer. + Locks() LockInformer + // Lockspaces returns a LockspaceInformer. + Lockspaces() LockspaceInformer // VirtualMachines returns a VirtualMachineInformer. VirtualMachines() VirtualMachineInformer // VirtualMachineMigrations returns a VirtualMachineMigrationInformer. @@ -25,6 +29,16 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// Locks returns a LockInformer. +func (v *version) Locks() LockInformer { + return &lockInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// Lockspaces returns a LockspaceInformer. +func (v *version) Lockspaces() LockspaceInformer { + return &lockspaceInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + // VirtualMachines returns a VirtualMachineInformer. func (v *version) VirtualMachines() VirtualMachineInformer { return &virtualMachineInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/generated/informers/externalversions/virt/v1alpha1/lock.go b/pkg/generated/informers/externalversions/virt/v1alpha1/lock.go new file mode 100644 index 0000000..61b97cc --- /dev/null +++ b/pkg/generated/informers/externalversions/virt/v1alpha1/lock.go @@ -0,0 +1,74 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + versioned "github.com/smartxworks/virtink/pkg/generated/clientset/versioned" + internalinterfaces "github.com/smartxworks/virtink/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/smartxworks/virtink/pkg/generated/listers/virt/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// LockInformer provides access to a shared informer and lister for +// Locks. +type LockInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.LockLister +} + +type lockInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewLockInformer constructs a new informer for Lock type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewLockInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredLockInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredLockInformer constructs a new informer for Lock type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredLockInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtV1alpha1().Locks(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtV1alpha1().Locks(namespace).Watch(context.TODO(), options) + }, + }, + &virtv1alpha1.Lock{}, + resyncPeriod, + indexers, + ) +} + +func (f *lockInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredLockInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *lockInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&virtv1alpha1.Lock{}, f.defaultInformer) +} + +func (f *lockInformer) Lister() v1alpha1.LockLister { + return v1alpha1.NewLockLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/virt/v1alpha1/lockspace.go b/pkg/generated/informers/externalversions/virt/v1alpha1/lockspace.go new file mode 100644 index 0000000..6a0cd1d --- /dev/null +++ b/pkg/generated/informers/externalversions/virt/v1alpha1/lockspace.go @@ -0,0 +1,73 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + versioned "github.com/smartxworks/virtink/pkg/generated/clientset/versioned" + internalinterfaces "github.com/smartxworks/virtink/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/smartxworks/virtink/pkg/generated/listers/virt/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// LockspaceInformer provides access to a shared informer and lister for +// Lockspaces. +type LockspaceInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.LockspaceLister +} + +type lockspaceInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewLockspaceInformer constructs a new informer for Lockspace type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewLockspaceInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredLockspaceInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredLockspaceInformer constructs a new informer for Lockspace type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredLockspaceInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtV1alpha1().Lockspaces().List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VirtV1alpha1().Lockspaces().Watch(context.TODO(), options) + }, + }, + &virtv1alpha1.Lockspace{}, + resyncPeriod, + indexers, + ) +} + +func (f *lockspaceInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredLockspaceInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *lockspaceInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&virtv1alpha1.Lockspace{}, f.defaultInformer) +} + +func (f *lockspaceInformer) Lister() v1alpha1.LockspaceLister { + return v1alpha1.NewLockspaceLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/listers/virt/v1alpha1/expansion_generated.go b/pkg/generated/listers/virt/v1alpha1/expansion_generated.go index 3139e62..bc70955 100644 --- a/pkg/generated/listers/virt/v1alpha1/expansion_generated.go +++ b/pkg/generated/listers/virt/v1alpha1/expansion_generated.go @@ -2,6 +2,18 @@ package v1alpha1 +// LockListerExpansion allows custom methods to be added to +// LockLister. +type LockListerExpansion interface{} + +// LockNamespaceListerExpansion allows custom methods to be added to +// LockNamespaceLister. +type LockNamespaceListerExpansion interface{} + +// LockspaceListerExpansion allows custom methods to be added to +// LockspaceLister. +type LockspaceListerExpansion interface{} + // VirtualMachineListerExpansion allows custom methods to be added to // VirtualMachineLister. type VirtualMachineListerExpansion interface{} diff --git a/pkg/generated/listers/virt/v1alpha1/lock.go b/pkg/generated/listers/virt/v1alpha1/lock.go new file mode 100644 index 0000000..b8f524e --- /dev/null +++ b/pkg/generated/listers/virt/v1alpha1/lock.go @@ -0,0 +1,83 @@ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// LockLister helps list Locks. +// All objects returned here must be treated as read-only. +type LockLister interface { + // List lists all Locks in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.Lock, err error) + // Locks returns an object that can list and get Locks. + Locks(namespace string) LockNamespaceLister + LockListerExpansion +} + +// lockLister implements the LockLister interface. +type lockLister struct { + indexer cache.Indexer +} + +// NewLockLister returns a new LockLister. +func NewLockLister(indexer cache.Indexer) LockLister { + return &lockLister{indexer: indexer} +} + +// List lists all Locks in the indexer. +func (s *lockLister) List(selector labels.Selector) (ret []*v1alpha1.Lock, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.Lock)) + }) + return ret, err +} + +// Locks returns an object that can list and get Locks. +func (s *lockLister) Locks(namespace string) LockNamespaceLister { + return lockNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// LockNamespaceLister helps list and get Locks. +// All objects returned here must be treated as read-only. +type LockNamespaceLister interface { + // List lists all Locks in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.Lock, err error) + // Get retrieves the Lock from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.Lock, error) + LockNamespaceListerExpansion +} + +// lockNamespaceLister implements the LockNamespaceLister +// interface. +type lockNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all Locks in the indexer for a given namespace. +func (s lockNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.Lock, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.Lock)) + }) + return ret, err +} + +// Get retrieves the Lock from the indexer for a given namespace and name. +func (s lockNamespaceLister) Get(name string) (*v1alpha1.Lock, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("lock"), name) + } + return obj.(*v1alpha1.Lock), nil +} diff --git a/pkg/generated/listers/virt/v1alpha1/lockspace.go b/pkg/generated/listers/virt/v1alpha1/lockspace.go new file mode 100644 index 0000000..c7d6b8e --- /dev/null +++ b/pkg/generated/listers/virt/v1alpha1/lockspace.go @@ -0,0 +1,52 @@ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// LockspaceLister helps list Lockspaces. +// All objects returned here must be treated as read-only. +type LockspaceLister interface { + // List lists all Lockspaces in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.Lockspace, err error) + // Get retrieves the Lockspace from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.Lockspace, error) + LockspaceListerExpansion +} + +// lockspaceLister implements the LockspaceLister interface. +type lockspaceLister struct { + indexer cache.Indexer +} + +// NewLockspaceLister returns a new LockspaceLister. +func NewLockspaceLister(indexer cache.Indexer) LockspaceLister { + return &lockspaceLister{indexer: indexer} +} + +// List lists all Lockspaces in the indexer. +func (s *lockspaceLister) List(selector labels.Selector) (ret []*v1alpha1.Lockspace, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.Lockspace)) + }) + return ret, err +} + +// Get retrieves the Lockspace from the index for a given name. +func (s *lockspaceLister) Get(name string) (*v1alpha1.Lockspace, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("lockspace"), name) + } + return obj.(*v1alpha1.Lockspace), nil +} diff --git a/pkg/sanlock/sanlock.go b/pkg/sanlock/sanlock.go new file mode 100644 index 0000000..33b0915 --- /dev/null +++ b/pkg/sanlock/sanlock.go @@ -0,0 +1,297 @@ +package sanlock + +import ( + "fmt" + "strings" + "unsafe" +) + +/* +#cgo LDFLAGS: -lsanlock -lsanlock_client +#include +#include +#include +#include +#include "sanlock.h" +#include "sanlock_direct.h" +#include "sanlock_admin.h" +#include "sanlock_resource.h" + +struct aicb { + int used; + char *buf; + struct iocb iocb; +}; + +#define NAME_ID_SIZE 48 + +struct task { + char name[NAME_ID_SIZE+1]; // for log messages + + unsigned int io_count; // stats + unsigned int to_count; // stats + + int use_aio; + int cb_size; + char *iobuf; + io_context_t aio_ctx; + struct aicb *read_iobuf_timeout_aicb; + struct aicb *callbacks; +}; + +int direct_rindex_format(struct task *task, struct sanlk_rindex *ri); +*/ +import "C" + +const ( + OffsetLockspace = 0 + OffsetRIndex = 1048576 + + WatchdogFireTimeoutDefaultSeconds = 60 + IOTimeoutDefaultSeconds = 10 + IDRenewalDefaultSeconds = 2 * IOTimeoutDefaultSeconds + IDRenewalFailDefaultSeconds = 4 * IDRenewalDefaultSeconds + HostDeadDefaultSeconds = IDRenewalFailDefaultSeconds + WatchdogFireTimeoutDefaultSeconds +) + +type ErrorNumber int + +const ( + // TODO: more error + EPERM = ErrorNumber(C.EPERM) + ENOENT = ErrorNumber(C.ENOENT) + EINVAL = ErrorNumber(C.EINVAL) + EEXIST = ErrorNumber(C.EEXIST) + EINPROGRESS = ErrorNumber(C.EINPROGRESS) + EAGAIN = ErrorNumber(C.EAGAIN) +) + +func (e ErrorNumber) Error() string { + var err string + switch e { + case EPERM: + err = "EPERM" + case ENOENT: + err = "ENOENT" + case EINVAL: + err = "EINVAL" + case EEXIST: + err = "EEXIST" + case EINPROGRESS: + err = "EINPROGRESS" + case EAGAIN: + err = "EAGAIN" + } + return fmt.Sprintf("errCode(%d): %s", e, err) +} + +type HostStatusFlag uint32 + +const ( + HostStatusUnknown = HostStatusFlag(C.SANLK_HOST_UNKNOWN) + HostStatusFree = HostStatusFlag(C.SANLK_HOST_FREE) + HostStatusLive = HostStatusFlag(C.SANLK_HOST_LIVE) + HostStatusFail = HostStatusFlag(C.SANLK_HOST_FAIL) + HostStatusDead = HostStatusFlag(C.SANLK_HOST_DEAD) +) + +func WriteLockspace(lockspace string, path string) error { + return WriteLockspaceWithIOTimeout(lockspace, path, 0) +} + +func WriteLockspaceWithIOTimeout(lockspace string, path string, ioTimeout uint32) error { + ls := buildSanlockLockspace(lockspace, path, 0) + + if rv := C.sanlock_direct_write_lockspace(&ls, C.int(2000), 0, C.uint(ioTimeout)); rv < 0 { + return ErrorNumber(-rv) + } + return nil +} + +func FormatRIndex(lockspace string, path string) error { + rIndex := buildSanlockRIndex(lockspace, path) + + if rv := C.direct_rindex_format(&C.struct_task{}, &rIndex); rv < 0 { + return ErrorNumber(-rv) + } + return nil +} + +func CreateResource(lockspace string, path string, resource string) (uint64, error) { + rIndex := buildSanlockRIndex(lockspace, path) + rEntry := buildSanlockREntry(resource) + + if rv := C.sanlock_create_resource(&rIndex, 0, &rEntry, 0, 0); rv < 0 { + return 0, ErrorNumber(-rv) + } + return uint64(rEntry.offset), nil +} + +func DeleteResource(lockspace string, path string, resource string) error { + rIndex := buildSanlockRIndex(lockspace, path) + rEntry := buildSanlockREntry(resource) + + if rv := C.sanlock_delete_resource(&rIndex, 0, &rEntry); rv < 0 { + return ErrorNumber(-rv) + } + return nil +} + +func SearchResource(lockspace string, path string, resource string) (uint64, error) { + rIndex := buildSanlockRIndex(lockspace, path) + rEntry := buildSanlockREntry(resource) + + if rv := C.sanlock_lookup_rindex(&rIndex, 0, &rEntry); rv < 0 { + return 0, ErrorNumber(-rv) + } + return uint64(rEntry.offset), nil +} + +// AcquireDeltaLease returns: +// nil: acquire delta lease successfully +// EEXIST: the lockspace already exists +// EINPROGRESS: the lockspace is already in the process of being added (the in-progress add may or may not succeed) +// EAGAIN: the lockspace is being removed +func AcquireDeltaLease(lockspace string, path string, id uint64) error { + if id < 1 || id > 2000 { + return fmt.Errorf("invalid host ID, allowed value 1~2000") + } + + ls := buildSanlockLockspace(lockspace, path, id) + + if rv := C.sanlock_add_lockspace(&ls, 0); rv < 0 { + return ErrorNumber(-rv) + } + return nil +} + +// ReleaseDeltaLease returns: +// EINPROGRESS: the lockspace is already in the process of being removed +// ENOENT: lockspace not found +// +// The sanlock daemon will kill any pids using the lockspace when the +// lockspace is removed. +func ReleaseDeltaLease(lockspace string, path string, id uint64) error { + if id < 1 || id > 2000 { + return fmt.Errorf("invalid host ID, allowed value 1~2000") + } + + ls := buildSanlockLockspace(lockspace, path, id) + + if rv := C.sanlock_rem_lockspace(&ls, 0); rv < 0 { + return ErrorNumber(-rv) + } + return nil +} + +func HasDeltaLease(lockspace string, path string, id uint64) bool { + if id < 1 || id > 2000 { + return false + } + + ls := buildSanlockLockspace(lockspace, path, id) + + return C.sanlock_inq_lockspace(&ls, 0) == 0 +} + +func GetHostStatus(lockspace string, id uint64) (HostStatusFlag, error) { + if id < 1 || id > 2000 { + return 0, fmt.Errorf("invalid host ID, allowed value 1~2000") + } + + var host *C.struct_sanlk_host + var num C.int + lockspaceName := C.CString(lockspace) + defer C.free(unsafe.Pointer(lockspaceName)) + if rv := C.sanlock_get_hosts(lockspaceName, C.ulong(id), &host, &num, 0); rv < 0 { + return 0, ErrorNumber(-rv) + } + + return HostStatusFlag(host.flags & C.SANLK_HOST_MASK), nil +} + +func AcquireResourceLease(resources []string, owner string) error { + if len(resources) > C.SANLK_MAX_RESOURCES { + return fmt.Errorf("requested resource over max %d", C.SANLK_MAX_RESOURCES) + } + + sock := C.sanlock_register() + if sock < 0 { + return fmt.Errorf("failed to registr process") + } + + if rv := C.sanlock_restrict(sock, C.SANLK_RESTRICT_SIGTERM); rv < 0 { + return fmt.Errorf("restrict SIGTERM signal: %s", ErrorNumber(-rv)) + } + + var res_args **C.struct_sanlk_resource + var res_count C.int + res := C.CString(strings.Join(resources, " ")) + defer C.free(unsafe.Pointer(res)) + if rv := C.sanlock_state_to_args(res, &res_count, &res_args); rv < 0 { + return fmt.Errorf("convert sanlock resources: %s", ErrorNumber(-rv)) + } + opt := C.struct_sanlk_options{ + owner_name: buildSanlockName(owner), + } + + if rv := C.sanlock_acquire(sock, -1, 0, res_count, res_args, &opt); rv < 0 { + return fmt.Errorf("acquire resource lease: %s", ErrorNumber(-rv)) + } + return nil +} + +func buildSanlockLockspace(lockspace string, path string, id uint64) C.struct_sanlk_lockspace { + diskPath := buildSanlockPath(path) + lockspaceName := buildSanlockName(lockspace) + + disk := C.struct_sanlk_disk{ + path: diskPath, + offset: OffsetLockspace, + } + return C.struct_sanlk_lockspace{ + name: lockspaceName, + host_id: C.ulong(id), + host_id_disk: disk, + flags: C.SANLK_LSF_ALIGN1M | C.SANLK_LSF_SECTOR512, + } +} + +func buildSanlockPath(path string) [C.SANLK_PATH_LEN]C.char { + cPath := [C.SANLK_PATH_LEN]C.char{} + for i := 0; i < len(path); i++ { + cPath[i] = C.char(path[i]) + } + return cPath +} + +func buildSanlockName(name string) [C.SANLK_NAME_LEN]C.char { + cName := [C.SANLK_NAME_LEN]C.char{} + for i := 0; i < len(name); i++ { + cName[i] = C.char(name[i]) + } + return cName +} + +func buildSanlockRIndex(lockspace string, path string) C.struct_sanlk_rindex { + diskPath := buildSanlockPath(path) + lockspaceName := buildSanlockName(lockspace) + + disk := C.struct_sanlk_disk{ + path: diskPath, + offset: OffsetRIndex, + } + return C.struct_sanlk_rindex{ + flags: C.SANLK_RIF_ALIGN1M | C.SANLK_RIF_SECTOR512, + lockspace_name: lockspaceName, + disk: disk, + } +} + +func buildSanlockREntry(rEntry string) C.struct_sanlk_rentry { + rEntryName := buildSanlockName(rEntry) + + return C.struct_sanlk_rentry{ + name: rEntryName, + } +} diff --git a/samples/ubuntu-ha.yaml b/samples/ubuntu-ha.yaml new file mode 100644 index 0000000..69bdd27 --- /dev/null +++ b/samples/ubuntu-ha.yaml @@ -0,0 +1,47 @@ +apiVersion: virt.virtink.smartx.com/v1alpha1 +kind: VirtualMachine +metadata: + name: ubuntu-ha +spec: + runPolicy: RerunOnFailure + enableHA: true + locks: + - ubuntu + instance: + memory: + size: 1Gi + disks: + - name: ubuntu + - name: cloud-init + interfaces: + - name: pod + masquerade: {} + volumes: + - name: ubuntu + dataVolume: + volumeName: ubuntu + - name: cloud-init + cloudInit: + userData: |- + #cloud-config + password: password + chpasswd: { expire: False } + ssh_pwauth: True + networks: + - name: pod + pod: {} +--- +apiVersion: cdi.kubevirt.io/v1beta1 +kind: DataVolume +metadata: + name: ubuntu +spec: + source: + http: + url: https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img + pvc: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 8Gi diff --git a/skaffold.yaml b/skaffold.yaml index 79172b6..cdb24eb 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -12,12 +12,27 @@ build: requires: - image: virt-prerunner alias: PRERUNNER_IMAGE + - image: lockspace-initializer + alias: INITIALIZER_IMAGE + - image: lockspace-attacher + alias: ATTACHER_IMAGE + - image: lockspace-detector + alias: DETECTOR_IMAGE - image: virt-daemon docker: dockerfile: build/virt-daemon/Dockerfile - image: virt-prerunner docker: dockerfile: build/virt-prerunner/Dockerfile + - image: lockspace-initializer + docker: + dockerfile: build/lockspace-initializer/Dockerfile + - image: lockspace-attacher + docker: + dockerfile: build/lockspace-attacher/Dockerfile + - image: lockspace-detector + docker: + dockerfile: build/lockspace-detector/Dockerfile deploy: kustomize: paths: