diff --git a/api/v1/helpers.go b/api/v1/helpers.go index 9fa66be8d..dbbcf686c 100644 --- a/api/v1/helpers.go +++ b/api/v1/helpers.go @@ -740,6 +740,11 @@ func (v *VerticaDB) GetCreateDBNodeStartTimeout() int { return vmeta.GetCreateDBNodeStartTimeout(v.Annotations) } +// GetShutdownDrainSeconds returns time in seconds to wait for a subcluster/database users' disconnection +func (v *VerticaDB) GetShutdownDrainSeconds() int { + return vmeta.GetShutdownDrainSeconds(v.Annotations) +} + // IsNMASideCarDeploymentEnabled returns true if the conditions to run NMA // in a sidecar are met func (v *VerticaDB) IsNMASideCarDeploymentEnabled() bool { diff --git a/changes/unreleased/Added-20241112-143404.yaml b/changes/unreleased/Added-20241112-143404.yaml new file mode 100644 index 000000000..f89a09e97 --- /dev/null +++ b/changes/unreleased/Added-20241112-143404.yaml @@ -0,0 +1,5 @@ +kind: Added +body: Support planned subcluster shutdown. +time: 2024-11-12T14:34:04.504040638+01:00 +custom: + Issue: "969" diff --git a/pkg/controllers/sandbox/sandbox_controller.go b/pkg/controllers/sandbox/sandbox_controller.go index 98e5e904b..70938ad78 100644 --- a/pkg/controllers/sandbox/sandbox_controller.go +++ b/pkg/controllers/sandbox/sandbox_controller.go @@ -167,9 +167,11 @@ func (r *SandboxConfigMapReconciler) constructActors(vdb *v1.VerticaDB, log logr vdbcontroller.MakeAnnotateAndLabelPodReconciler(r, log, vdb, pfacts), // Stop Vertica in the sandbox if the shutdown state is true vdbcontroller.MakeStopDBReconciler(r, vdb, prunner, pfacts, dispatcher), + // Stop subclusters that have shutdown set to true. + vdbcontroller.MakeSubclusterShutdownReconciler(r, log, vdb, dispatcher, pfacts), // Restart any down pods vdbcontroller.MakeRestartReconciler(r, log, vdb, prunner, pfacts, true, dispatcher), - // Update the vdb status including subclusters[].shutdown, after a stopdb + // Update the vdb status including subclusters[].shutdown, after a stop_db, stop_sc // or a restart vdbcontroller.MakeStatusReconcilerWithShutdown(r.Client, r.Scheme, log, vdb, pfacts), // Scale down the subclusters' statefulsets to zero after the subclusters are shut down diff --git a/pkg/controllers/vdb/dbaddnode_reconciler.go b/pkg/controllers/vdb/dbaddnode_reconciler.go index 9477e15e2..f63fa94a4 100644 --- a/pkg/controllers/vdb/dbaddnode_reconciler.go +++ b/pkg/controllers/vdb/dbaddnode_reconciler.go @@ -75,7 +75,15 @@ func (d *DBAddNodeReconciler) Reconcile(ctx context.Context, _ *ctrl.Request) (c return ctrl.Result{Requeue: true}, nil } + scStatusMap := d.Vdb.GenSubclusterStatusMap() for i := range d.Vdb.Spec.Subclusters { + sc := &d.Vdb.Spec.Subclusters[i] + scStatus, found := scStatusMap[sc.Name] + if found && scStatus.Shutdown { + // subclusters that have been shut down must + // be ignored. + continue + } // Recollect pod facts to ensure correct options are used in AddNode() if err := d.PFacts.Collect(ctx, d.Vdb); err != nil { return ctrl.Result{}, err @@ -101,6 +109,9 @@ func (d *DBAddNodeReconciler) findAddNodePods(scName string) ([]*podfacts.PodFac continue } if !v.GetDBExists() { + if v.GetShutdown() { + continue + } if !v.GetIsPodRunning() || !v.GetIsInstalled() { // We want to group all of the add nodes in a single admintools call. // Doing so limits the impact on any running queries. So if there is at diff --git a/pkg/controllers/vdb/depobjcheck_reconciler.go b/pkg/controllers/vdb/depobjcheck_reconciler.go index 304faa169..2faec695c 100644 --- a/pkg/controllers/vdb/depobjcheck_reconciler.go +++ b/pkg/controllers/vdb/depobjcheck_reconciler.go @@ -135,6 +135,11 @@ func (d *DepObjCheckReconciler) checkSts(ctx context.Context, sc *vapi.Subcluste } func (d *DepObjCheckReconciler) checkPods(ctx context.Context, sc *vapi.Subcluster) (ctrl.Result, error) { + scStatus, found := d.Vdb.GenSubclusterStatusMap()[sc.Name] + // Ignore subclusters that are shut down + if sc.Shutdown || (found && scStatus.Shutdown) { + return ctrl.Result{}, nil + } for i := int32(0); i < sc.Size; i++ { if res, err := d.checkObj(ctx, "Pod", names.GenPodName(d.Vdb, sc, i), &corev1.Pod{}); verrors.IsReconcileAborted(res, err) { return res, err diff --git a/pkg/controllers/vdb/resizepv_reconciler.go b/pkg/controllers/vdb/resizepv_reconciler.go index e86d76f33..a292774dd 100644 --- a/pkg/controllers/vdb/resizepv_reconciler.go +++ b/pkg/controllers/vdb/resizepv_reconciler.go @@ -65,7 +65,12 @@ func (r *ResizePVReconcile) Reconcile(ctx context.Context, _ *ctrl.Request) (ctr } returnRes := ctrl.Result{} + scStatusMap := r.Vdb.GenSubclusterStatusMap() for _, pf := range r.PFacts.Detail { + scStatus, found := scStatusMap[pf.GetSubclusterName()] + if pf.GetShutdown() || (found && scStatus.Shutdown) { + continue + } if res, err := r.reconcilePod(ctx, pf); verrors.IsReconcileAborted(res, err) { // Errors always abort right away. But if we get a requeue, we // will remember this and go onto the next pod diff --git a/pkg/controllers/vdb/shutdownspec_reconciler.go b/pkg/controllers/vdb/shutdownspec_reconciler.go index e03b31fdf..fafe97a13 100644 --- a/pkg/controllers/vdb/shutdownspec_reconciler.go +++ b/pkg/controllers/vdb/shutdownspec_reconciler.go @@ -65,20 +65,18 @@ func (r *ShutdownSpecReconciler) updateSubclustersShutdownStateCallback() (bool, sb := &r.Vdb.Spec.Sandboxes[i] for j := range sb.Subclusters { sc := scMap[sb.Subclusters[j].Name] - // Proceed only if subcluster's shutdown is not equal - // to sandbox's shutdown - if sb.Shutdown == sc.Shutdown { - continue - } if sb.Shutdown { if sc.Annotations == nil { sc.Annotations = make(map[string]string, 1) } - // Add a label that indicate the shutdown/restart is controlled - // by the sandbox as opposed to the subcluster. It helps - // differentiate this case from when the user is explicitly - // changing the subcluster's shutdown field. - sc.Annotations[vmeta.ShutdownDrivenBySandbox] = "true" + if _, ok := sc.Annotations[vmeta.ShutdownDrivenBySandbox]; !ok { + // Add a label that indicate the shutdown/restart is controlled + // by the sandbox as opposed to the subcluster. It helps + // differentiate this case from when the user is explicitly + // changing the subcluster's shutdown field. + sc.Annotations[vmeta.ShutdownDrivenBySandbox] = "true" + needUpdate = true + } } else { // If the shutdown/restart is not controlled by the sandbox, // we skip to the next subcluster. @@ -86,9 +84,12 @@ func (r *ShutdownSpecReconciler) updateSubclustersShutdownStateCallback() (bool, continue } delete(sc.Annotations, vmeta.ShutdownDrivenBySandbox) + needUpdate = true + } + if sb.Shutdown != sc.Shutdown { + sc.Shutdown = sb.Shutdown + needUpdate = true } - sc.Shutdown = sb.Shutdown - needUpdate = true } } return needUpdate, nil diff --git a/pkg/controllers/vdb/stopdb_reconciler.go b/pkg/controllers/vdb/stopdb_reconciler.go index 6aea9eb70..0a7d70e04 100644 --- a/pkg/controllers/vdb/stopdb_reconciler.go +++ b/pkg/controllers/vdb/stopdb_reconciler.go @@ -116,6 +116,7 @@ func (s *StopDBReconciler) runATCmd(ctx context.Context, initiatorName types.Nam stopdb.WithInitiator(initiatorName, initiatorIP), stopdb.WithSandbox(s.PFacts.GetSandboxName()), stopdb.WithZeroDrain(false), + stopdb.WithDrainSeconds(s.Vdb.GetShutdownDrainSeconds()), } start := time.Now() if err := s.Dispatcher.StopDB(ctx, opts...); err != nil { diff --git a/pkg/controllers/vdb/subclustershutdown_reconciler.go b/pkg/controllers/vdb/subclustershutdown_reconciler.go new file mode 100644 index 000000000..6a04078f1 --- /dev/null +++ b/pkg/controllers/vdb/subclustershutdown_reconciler.go @@ -0,0 +1,181 @@ +/* + (c) Copyright [2021-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vdb + +import ( + "context" + "fmt" + "strings" + + "github.com/go-logr/logr" + vapi "github.com/vertica/vertica-kubernetes/api/v1" + "github.com/vertica/vertica-kubernetes/pkg/controllers" + "github.com/vertica/vertica-kubernetes/pkg/events" + "github.com/vertica/vertica-kubernetes/pkg/podfacts" + "github.com/vertica/vertica-kubernetes/pkg/vadmin" + "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/stopsubcluster" + config "github.com/vertica/vertica-kubernetes/pkg/vdbconfig" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +// SubclusterShutdownReconciler will handle the process when subclusters +// needs to be shut down or restart +type SubclusterShutdownReconciler struct { + VRec config.ReconcilerInterface + Log logr.Logger + Vdb *vapi.VerticaDB // Vdb is the CRD we are acting on + Dispatcher vadmin.Dispatcher + PFacts *podfacts.PodFacts +} + +// MakeSubclusterShutdownReconciler will build a SubclusterShutdownReconciler object +func MakeSubclusterShutdownReconciler(recon config.ReconcilerInterface, log logr.Logger, + vdb *vapi.VerticaDB, dispatcher vadmin.Dispatcher, pfacts *podfacts.PodFacts) controllers.ReconcileActor { + return &SubclusterShutdownReconciler{ + VRec: recon, + Log: log.WithName("SubclusterShutdownReconciler"), + Vdb: vdb, + Dispatcher: dispatcher, + PFacts: pfacts, + } +} + +func (s *SubclusterShutdownReconciler) Reconcile(ctx context.Context, _ *ctrl.Request) (ctrl.Result, error) { + // no-op for ScheduleOnly init policy or enterprise db + if s.Vdb.Spec.InitPolicy == vapi.CommunalInitPolicyScheduleOnly || !s.Vdb.IsEON() { + return ctrl.Result{}, nil + } + + if err := s.PFacts.Collect(ctx, s.Vdb); err != nil { + return ctrl.Result{}, err + } + subclusters, err := s.getSubclustersToShutdown() + if err != nil { + return ctrl.Result{}, err + } + for scName, initIP := range subclusters { + err := s.PFacts.RemoveStartupFileInSubclusterPods(ctx, scName, "removed startup.json before stop_subcluster") + if err != nil { + return ctrl.Result{}, err + } + err = s.runStopSubclusterVclusterAPI(ctx, scName, initIP) + if err != nil { + return ctrl.Result{}, err + } + s.PFacts.Invalidate() + } + return ctrl.Result{}, nil +} + +// getSubclustersToShutdown returns the subclusters that need to get +// shut down +func (s *SubclusterShutdownReconciler) getSubclustersToShutdown() (map[string]string, error) { + subclusters := map[string]string{} + primarySubclusters := []string{} + upPrimaryNodes := 0 + willLoseQuorum := false + scSbMap := s.Vdb.GenSubclusterSandboxMap() + scStatusMap := s.Vdb.GenSubclusterStatusMap() + sbMap := s.Vdb.GenSandboxMap() + s.Log.Info(fmt.Sprintf("Collecting subclusters to shut down in %s", s.PFacts.GetClusterExtendedName()), + "sandbox", s.PFacts.GetSandboxName()) + for i := range s.Vdb.Spec.Subclusters { + sc := &s.Vdb.Spec.Subclusters[i] + sandbox := scSbMap[sc.Name] + if sandbox != s.PFacts.GetSandboxName() { + s.Log.Info(fmt.Sprintf("Skipping subcluster because it is not in %s", s.PFacts.GetClusterExtendedName()), + "subcluster", sc.Name, "sandbox", s.PFacts.GetSandboxName()) + continue + } + // no-op if the subcluster is not marked for + // shutdown + if !sc.Shutdown { + continue + } + if sandbox != vapi.MainCluster { + sb := sbMap[sandbox] + // no-op if the subcluster shutdown is driven + // by its sandbox + if sb != nil && sb.Shutdown { + continue + } + } + if s.PFacts.IsDBReadOnly() { + return subclusters, fmt.Errorf("cannot shutdown subcluster because %s is read-only", s.PFacts.GetClusterExtendedName()) + } + hostIP, ok := s.PFacts.FindFirstUpPodIP(false, sc.Name) + if !ok { + scStatus := scStatusMap[sc.Name] + if scStatus == nil { + return subclusters, fmt.Errorf("subcluster %q not found in status", sc.Name) + } + if !scStatus.Shutdown { + s.Log.Info("Subcluster nodes are already all down, and were not shutdown gracefully.", "subcluster", sc.Name) + } + continue + } + if sc.IsPrimary() { + primarySubclusters = append(primarySubclusters, sc.Name) + upPrimaryNodes += s.PFacts.GetSubclusterUpNodeCount(sc.Name) + // If stopping a subcluster would cause cluster quorum, we abort + // the operation + if !s.PFacts.DoesDBHaveQuorum(upPrimaryNodes) { + willLoseQuorum = true + break + } + } + subclusters[sc.Name] = hostIP + } + if willLoseQuorum { + // TODO: we may remove this once we find a proper way to handle quorum loss + s.VRec.Eventf(s.Vdb, corev1.EventTypeWarning, events.ClusterWillLoseQuorum, + "Shutting down subclusters %s will cause quorum loss.", strings.Join(primarySubclusters, ",")) + return subclusters, fmt.Errorf("cannot shut down primaries %s because it will cause quorum loss. "+ + "please revert back", strings.Join(primarySubclusters, ",")) + } + return subclusters, nil +} + +// runStopSubclusterVclusterAPI will do the actual execution of stop subcluster. +// This handles logging of necessary events. +func (s *SubclusterShutdownReconciler) runStopSubclusterVclusterAPI(ctx context.Context, scName, host string) error { + opts := s.genStopSubclusterOpts(host, scName) + s.VRec.Eventf(s.Vdb, corev1.EventTypeNormal, events.StopSubclusterStart, "Starting stop subcluster %q.", + scName) + + err := s.Dispatcher.StopSubcluster(ctx, opts...) + if err != nil { + // For all other errors, return error + s.VRec.Eventf(s.Vdb, corev1.EventTypeWarning, events.StopSubclusterFailed, + "Failed to stop subcluster %q", scName) + return err + } + + s.VRec.Eventf(s.Vdb, corev1.EventTypeNormal, events.StopSubclusterSucceeded, + "Successfully stopped subcluster %q.", scName) + return nil +} + +// genStopSubclusterOpts will return the options to use with the stop subcluster api +func (s *SubclusterShutdownReconciler) genStopSubclusterOpts(initiatorIP, scName string) []stopsubcluster.Option { + opts := []stopsubcluster.Option{ + stopsubcluster.WithInitiator(initiatorIP), + stopsubcluster.WithSCName(scName), + stopsubcluster.WithDrainSeconds(s.Vdb.GetShutdownDrainSeconds()), + } + return opts +} diff --git a/pkg/controllers/vdb/subclustershutdown_reconciler_test.go b/pkg/controllers/vdb/subclustershutdown_reconciler_test.go new file mode 100644 index 000000000..b0cc94821 --- /dev/null +++ b/pkg/controllers/vdb/subclustershutdown_reconciler_test.go @@ -0,0 +1,117 @@ +/* + (c) Copyright [2021-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vdb + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + vapi "github.com/vertica/vertica-kubernetes/api/v1" + "github.com/vertica/vertica-kubernetes/pkg/cmds" + "github.com/vertica/vertica-kubernetes/pkg/podfacts" +) + +var _ = Describe("subclustershutdown_reconciler", func() { + It("should reconcile based on sandbox shutdown field", func() { + const ( + maincluster = "main" + subcluster1 = "sc1" + subcluster2 = "sc2" + subcluster3 = "sc3" + subcluster4 = "sc4" + sandbox1 = "sandbox1" + ) + vdb := vapi.MakeVDBForVclusterOps() + vdb.Spec.Subclusters = []vapi.Subcluster{ + {Name: maincluster, Size: 4, Type: vapi.PrimarySubcluster}, + {Name: subcluster1, Size: 3, Type: vapi.SecondarySubcluster}, + {Name: subcluster2, Size: 3, Type: vapi.PrimarySubcluster}, + {Name: subcluster3, Size: 3, Type: vapi.SecondarySubcluster}, + {Name: subcluster4, Size: 3, Type: vapi.SecondarySubcluster}, + } + // All subclusters are in main cluster. + // Shutting down secondaries + vdb.Spec.Subclusters[1].Shutdown = true + vdb.Spec.Subclusters[3].Shutdown = true + vdb.Spec.Subclusters[4].Shutdown = true + fpr := &cmds.FakePodRunner{} + pfacts := podfacts.MakePodFacts(vdbRec, fpr, logger, TestPassword) + upNodes := []uint{4, 3, 3, 3, 3} + pfacts.ConstructsDetail(vdb.Spec.Subclusters, upNodes) + Expect(len(pfacts.Detail)).Should(Equal(16)) + dispatcher := vdbRec.makeDispatcher(logger, vdb, fpr, TestPassword) + + act := MakeSubclusterShutdownReconciler(vdbRec, logger, vdb, dispatcher, &pfacts) + r := act.(*SubclusterShutdownReconciler) + scMap, err := r.getSubclustersToShutdown() + Expect(err).Should(BeNil()) + Expect(len(scMap)).Should(Equal(3)) + subclusters := getSubclusters(scMap) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[1].Name)) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[3].Name)) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[4].Name)) + + // let's shut down a primary + // We should still be good because 4 out of 7 primaries + // will be up + vdb.Spec.Subclusters[2].Shutdown = true + upNodes = []uint{4, 3, 3, 3, 3} + pfacts.ConstructsDetail(vdb.Spec.Subclusters, upNodes) + act = MakeSubclusterShutdownReconciler(vdbRec, logger, vdb, dispatcher, &pfacts) + r = act.(*SubclusterShutdownReconciler) + scMap, err = r.getSubclustersToShutdown() + Expect(err).Should(BeNil()) + Expect(len(scMap)).Should(Equal(4)) + subclusters = getSubclusters(scMap) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[1].Name)) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[2].Name)) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[3].Name)) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[4].Name)) + + // Let's try again but this time 3 out of 7 primaries + // will be up so we should return an error + upNodes = []uint{3, 3, 3, 3, 3} + pfacts.ConstructsDetail(vdb.Spec.Subclusters, upNodes) + act = MakeSubclusterShutdownReconciler(vdbRec, logger, vdb, dispatcher, &pfacts) + r = act.(*SubclusterShutdownReconciler) + _, err = r.getSubclustersToShutdown() + Expect(err).ShouldNot(BeNil()) + + // We sandbox 2 subclusters so they should be ignored + vdb.Spec.Sandboxes = []vapi.Sandbox{ + {Name: sandbox1, Shutdown: true, Subclusters: []vapi.SubclusterName{{Name: subcluster1}, {Name: subcluster3}}}, + } + upNodes = []uint{4, 3, 3, 3, 3} + pfacts.ConstructsDetail(vdb.Spec.Subclusters, upNodes) + act = MakeSubclusterShutdownReconciler(vdbRec, logger, vdb, dispatcher, &pfacts) + r = act.(*SubclusterShutdownReconciler) + scMap, err = r.getSubclustersToShutdown() + Expect(err).Should(BeNil()) + Expect(len(scMap)).Should(Equal(2)) + subclusters = getSubclusters(scMap) + Expect(subclusters).ShouldNot(ContainElement(vdb.Spec.Subclusters[1].Name)) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[2].Name)) + Expect(subclusters).ShouldNot(ContainElement(vdb.Spec.Subclusters[3].Name)) + Expect(subclusters).Should(ContainElement(vdb.Spec.Subclusters[4].Name)) + }) +}) + +func getSubclusters(scMap map[string]string) []string { + subclusters := []string{} + for scName := range scMap { + subclusters = append(subclusters, scName) + } + return subclusters +} diff --git a/pkg/controllers/vdb/unsandboximageversion_reconciler.go b/pkg/controllers/vdb/unsandboximageversion_reconciler.go index 830c2c72e..051a07256 100644 --- a/pkg/controllers/vdb/unsandboximageversion_reconciler.go +++ b/pkg/controllers/vdb/unsandboximageversion_reconciler.go @@ -73,7 +73,7 @@ func (r *UnsandboxImageVersion) Reconcile(ctx context.Context, _ *ctrl.Request) // reconcileVerticaImage recreates the StatefulSet of the secondary subclusters that // have the wrong vertica image func (r *UnsandboxImageVersion) reconcileVerticaImage(ctx context.Context) (ctrl.Result, error) { - scsWithWrongImage, priScImage := r.PFacts.FindSecondarySubclustersWithDifferentImage() + scsWithWrongImage, priScImage := r.PFacts.FindSecondarySubclustersWithDifferentImage(r.Vdb) if len(scsWithWrongImage) == 0 { return ctrl.Result{}, nil } diff --git a/pkg/controllers/vdb/verticadb_controller.go b/pkg/controllers/vdb/verticadb_controller.go index 70253cd75..6bb2db6ce 100644 --- a/pkg/controllers/vdb/verticadb_controller.go +++ b/pkg/controllers/vdb/verticadb_controller.go @@ -208,13 +208,15 @@ func (r *VerticaDBReconciler) constructActors(log logr.Logger, vdb *vapi.Vertica MakeOnlineUpgradeReconciler(r, log, vdb, pfacts, dispatcher), // Stop vertica if the status condition indicates MakeStopDBReconciler(r, vdb, prunner, pfacts, dispatcher), + // Stop subclusters that have shutdown set to true. + MakeSubclusterShutdownReconciler(r, log, vdb, dispatcher, pfacts), // Check the version information ahead of restart. The version is needed // to properly pick the correct NMA deployment (monolithic vs sidecar). MakeImageVersionReconciler(r, log, vdb, prunner, pfacts, false /* enforceUpgradePath */), // Handles restart + re_ip of vertica MakeRestartReconciler(r, log, vdb, prunner, pfacts, true, dispatcher), MakeMetricReconciler(r, log, vdb, prunner, pfacts), - MakeStatusReconciler(r.Client, r.Scheme, log, vdb, pfacts), + MakeStatusReconcilerWithShutdown(r.Client, r.Scheme, log, vdb, pfacts), // Ensure we add labels to any pod rescheduled so that Service objects route traffic to it. MakeClientRoutingLabelReconciler(r, log, vdb, pfacts, PodRescheduleApplyMethod, ""), // Remove Service label for any pods that are pending delete. This will @@ -270,8 +272,8 @@ func (r *VerticaDBReconciler) constructActors(log logr.Logger, vdb *vapi.Vertica // Trigger sandbox upgrade when the image field for the sandbox // is changed MakeSandboxUpgradeReconciler(r, log, vdb), - // Update the sandbox subclusters' shutdown field to match the value of - // the sandbox. + // Update the sandbox/subclusters' shutdown field to match the value of + // the spec. MakeShutdownSpecReconciler(r, vdb, pfacts), // Trigger sandbox shutdown when the shutdown field of the sandbox // is changed diff --git a/pkg/events/event.go b/pkg/events/event.go index e7e90a6d1..56eba7700 100644 --- a/pkg/events/event.go +++ b/pkg/events/event.go @@ -68,6 +68,9 @@ const ( SaveRestorePointStart = "SaveRestorePointStart" SaveRestorePointSucceeded = "SaveRestorePointSucceeded" SaveRestorePointFailed = "SaveRestorePointFailed" + StopSubclusterStart = "StopSubclusterStart" + StopSubclusterSucceeded = "StopSubclusterSucceeded" + StopSubclusterFailed = "StopSubclusterFailed" SlowRestartDetected = "SlowRestartDetected" SubclusterAdded = "SubclusterAdded" SubclusterRemoved = "SubclusterRemoved" @@ -99,6 +102,7 @@ const ( StopDBStart = "StopDBStart" StopDBSucceeded = "StopDBSucceeded" StopDBFailed = "StopDBFailed" + ClusterWillLoseQuorum = "ClusterWillLoseQuorum" DepotResized = "DepotResized" MgmtFailed = "MgmtFailed" MgmtFailedDiskFull = "MgmtFailedDiskfull" diff --git a/pkg/meta/annotations.go b/pkg/meta/annotations.go index 18c27e050..b79ec0fbe 100644 --- a/pkg/meta/annotations.go +++ b/pkg/meta/annotations.go @@ -101,6 +101,10 @@ const ( // waits for its startup. If omitted, we use the default timeout of 5 minutes. CreateDBTimeoutAnnotation = "vertica.com/createdb-timeout" + // The time in seconds to wait for a subcluster or database users' disconnection, its default value is 60 + ShutdownDrainSecondsAnnotation = "vertica.com/shutdown-drain-seconds" + ShutdownDefaultDrainSeconds = 60 + // The timeout, in seconds, to use when the operator is performing online upgrade // for various tasks. If omitted, we use the default timeout of 5 minutes. OnlineUpgradeTimeoutAnnotation = "vertica.com/online-upgrade-timeout" @@ -332,9 +336,11 @@ const ( // This will be set in a sandbox configMap by the vdb controller to wake up the sandbox // controller for unsandboxing the subclusters SandboxControllerUnsandboxTriggerID = "vertica.com/sandbox-controller-unsandbox-trigger-id" - // This will be set in a sandbox configMap bu the vdb controller to wake up the sandbox + // This will be set in a sandbox configMap by the vdb controller to wake up the sandbox // controller for stopping/starting a sandbox SandboxControllerShutdownTriggerID = "vertica.com/sandbox-controller-shutdown-trigger-id" + // This will be set in a subclusters configMap by the vdb controller to stop/start a subcluter + VdbControllerShutdownClusterTriggerID = "vertica.com/vdb-controller-shutdown-subcluster-trigger-id" // Use this to override the name of the statefulset and its pods. This needs // to be set in the spec.subclusters[].annotations field to take effect. If @@ -350,6 +356,10 @@ const ( // This indicates that the subcluster shutdown is controlled by the sandbox // through the sandbox's shutdown field. ShutdownDrivenBySandbox = "vertica.com/shutdown-driven-by-sandbox" + + // This indicates that the subcluster shutdown is controlled by the subcluster's + // shutdown field. + ShutdownDrivenBySubcluster = "vertica.com/shutdown-driven-by-subcluster" ) // IsPauseAnnotationSet will check the annotations for a special value that will @@ -406,6 +416,11 @@ func GetCreateDBNodeStartTimeout(annotations map[string]string) int { return lookupIntAnnotation(annotations, CreateDBTimeoutAnnotation, 0 /* default value */) } +// GetShutdownCDrainSeconds returns the time in seconds to wait for a subcluster/database users' disconnection +func GetShutdownDrainSeconds(annotations map[string]string) int { + return lookupIntAnnotation(annotations, ShutdownDrainSecondsAnnotation, ShutdownDefaultDrainSeconds /* default value */) +} + // GetOnlineUpgradeTimeout returns the timeout to use for pause/redirect sessions func GetOnlineUpgradeTimeout(annotations map[string]string) int { return lookupIntAnnotation(annotations, OnlineUpgradeTimeoutAnnotation, OnlineUpgradeDefaultTimeout) @@ -665,6 +680,12 @@ func GetShutdownDrivenBySandbox(annotations map[string]string) bool { return lookupBoolAnnotation(annotations, ShutdownDrivenBySandbox, false) } +// GetShutdownDrivenBySubcluster returns the bool value if the operator +// will shutdown the subcluster and not try to restart it. +func GetShutdownDrivenBySubcluster(annotations map[string]string) bool { + return lookupBoolAnnotation(annotations, ShutdownDrivenBySubcluster, false) +} + // GetExtraLocalPaths returns the comma separated list of extra local paths func GetExtraLocalPaths(annotations map[string]string) string { return lookupStringAnnotation(annotations, ExtraLocalPathsAnnotation, "") diff --git a/pkg/mockvops/mockvops.go b/pkg/mockvops/mockvops.go index 316ec3e28..36049be59 100644 --- a/pkg/mockvops/mockvops.go +++ b/pkg/mockvops/mockvops.go @@ -69,6 +69,9 @@ func (*MockVClusterOps) VStartNodes(_ *vclusterops.VStartNodesOptions) error { func (*MockVClusterOps) VStopDatabase(_ *vclusterops.VStopDatabaseOptions) error { return nil } +func (*MockVClusterOps) VStopSubcluster(_ *vclusterops.VStopSubclusterOptions) error { + return nil +} func (*MockVClusterOps) VInstallPackages(_ *vclusterops.VInstallPackagesOptions) (*vclusterops.InstallPackageStatus, error) { return nil, nil } diff --git a/pkg/podfacts/podfacts.go b/pkg/podfacts/podfacts.go index 6fc3fa0e2..8036ed84d 100644 --- a/pkg/podfacts/podfacts.go +++ b/pkg/podfacts/podfacts.go @@ -246,6 +246,32 @@ func MakePodFactsForSandbox(vrec config.ReconcilerInterface, prunner cmds.PodRun return pf } +// ConstructsDetail sets the Detail field in PodFacts, for test purposes +func (p *PodFacts) ConstructsDetail(subclusters []vapi.Subcluster, upNodes []uint) { + p.Detail = make(PodFactDetail) + if len(subclusters) != len(upNodes) { + return + } + for i := range subclusters { + sc := &subclusters[i] + upNode := upNodes[i] + for j := int32(0); j < sc.Size; j++ { + isUp := upNode > 0 + pf := PodFact{ + name: types.NamespacedName{Name: fmt.Sprintf("%s-%d", sc.Name, j)}, + subclusterName: sc.Name, + isPrimary: sc.IsPrimary(), + shutdown: sc.Shutdown, + upNode: isUp, + podIP: "10.10.10.10", + podIndex: j, + } + upNode-- + p.Detail[pf.name] = &pf + } + } +} + // Copy will make a new copy of the podfacts, but setup for the given sandbox name. func (p *PodFacts) Copy(sandbox string) PodFacts { ret := *p @@ -776,6 +802,11 @@ func (p *PodFact) GetSandbox() string { return p.sandbox } +// GetShutdown returns the value of shutdown +func (p *PodFact) GetShutdown() bool { + return p.shutdown +} + // GetReadOnly returns the bool value of readonly in PodFact func (p *PodFact) GetReadOnly() bool { return p.readOnly @@ -1355,6 +1386,17 @@ func (p *PodFacts) GetUpNodeCount() int { }) } +// GetSubclusterUpNodeCount returns the number of up nodes in the given subcluster. +// A pod is considered down if it doesn't have a running vertica process. +func (p *PodFacts) GetSubclusterUpNodeCount(scName string) int { + return p.countPods(func(v *PodFact) int { + if v.subclusterName == scName && v.upNode { + return 1 + } + return 0 + }) +} + // GetUpNodeAndNotReadOnlyCount returns the number of nodes that are up and // writable. Starting in 11.0SP2, nodes can be up but only in read-only state. // This function filters out those *up* nodes that are in read-only state. @@ -1420,7 +1462,17 @@ func (p *PodFacts) AnyUninstalledTransientPodsNotRunning() (bool, types.Namespac return false, types.NamespacedName{} } -// GetHostList will return a host and podName list from the given pods +// IsDBReadOnly return true if the database is read-only +func (p *PodFacts) IsDBReadOnly() bool { + for _, v := range p.Detail { + if v.isPodRunning && v.readOnly { + return true + } + } + return false +} + +// GetHostList will returns a host and podName list from the given pods func GetHostAndPodNameList(podList []*PodFact) ([]string, []types.NamespacedName) { hostList := make([]string, 0, len(podList)) podNames := make([]types.NamespacedName, 0, len(podList)) @@ -1603,6 +1655,22 @@ func (p *PodFacts) QuorumCheckForRestartCluster(restartOnly bool) bool { return restartablePrimaryNodeCount >= (primaryNodeCount+1)/2 } +// DoesDBHaveQuorum returns true if the cluster will keep quorum +func (p *PodFacts) DoesDBHaveQuorum(offset int) bool { + totalPrimaryCount := 0 + upPrimaryCount := 0 + for _, pod := range p.Detail { + if !pod.GetIsPrimary() { + continue + } + totalPrimaryCount++ + if pod.GetUpNode() { + upPrimaryCount++ + } + } + return 2*(upPrimaryCount-offset) > totalPrimaryCount +} + // IsSandboxEmpty returns true if we cannot find any pods in the target sandbox func (p *PodFacts) IsSandboxEmpty(sandbox string) bool { pods := p.filterPods(func(v *PodFact) bool { @@ -1615,13 +1683,13 @@ func (p *PodFacts) IsSandboxEmpty(sandbox string) bool { // return the secondary subclusters that have different vertica image than primary subcluster with primary // subcluster image. This function is used in post-unsandbox process. If the pods in the sandbox upgraded // vertica, after unsandbox, we will find those pods out and restore their vertica images. -func (p *PodFacts) FindSecondarySubclustersWithDifferentImage() (scs []string, priScImage string) { +func (p *PodFacts) FindSecondarySubclustersWithDifferentImage(vdb *vapi.VerticaDB) (scs []string, priScImage string) { scs = []string{} // we expect the pfacts only contains the main cluster pods if p.GetSandboxName() != vapi.MainCluster { return scs, "" } - + scStatusMap := vdb.GenSubclusterStatusMap() for _, v := range p.Detail { if v.isPrimary { priScImage = v.image @@ -1634,6 +1702,11 @@ func (p *PodFacts) FindSecondarySubclustersWithDifferentImage() (scs []string, p if _, ok := seenScs[v.subclusterName]; ok { continue } + scStatus, found := scStatusMap[v.subclusterName] + // subclusters that are shut down must be ignored + if v.shutdown || (found && scStatus.Shutdown) { + continue + } if !v.isPrimary && v.image != priScImage { scs = append(scs, v.subclusterName) } diff --git a/pkg/vadmin/interface.go b/pkg/vadmin/interface.go index 77b9c199d..0c79c330d 100644 --- a/pkg/vadmin/interface.go +++ b/pkg/vadmin/interface.go @@ -50,6 +50,7 @@ import ( "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/showrestorepoints" "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/startdb" "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/stopdb" + "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/stopsubcluster" "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/unsandboxsc" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -128,6 +129,9 @@ type Dispatcher interface { // SaveRestorePoint will create a restore point to an existing archive SaveRestorePoint(ctx context.Context, opts ...saverestorepoint.Option) error + // StopSubcluster will stop a subcluster from Vertica db + StopSubcluster(ctx context.Context, opts ...stopsubcluster.Option) error + AlterSubclusterType(ctx context.Context, opts ...altersc.Option) error // SetConfigurationParameter will set a config parameter to a certain value at a certain level in a given cluster @@ -256,6 +260,7 @@ type VClusterProvider interface { VCreateDatabase(options *vops.VCreateDatabaseOptions) (vops.VCoordinationDatabase, error) VStopDatabase(options *vops.VStopDatabaseOptions) error VStartDatabase(options *vops.VStartDatabaseOptions) (*vops.VCoordinationDatabase, error) + VStopSubcluster(options *vops.VStopSubclusterOptions) error VReviveDatabase(options *vops.VReviveDatabaseOptions) (string, *vops.VCoordinationDatabase, error) VFetchNodeState(options *vops.VFetchNodeStateOptions) ([]vops.NodeInfo, error) VAddSubcluster(options *vops.VAddSubclusterOptions) error diff --git a/pkg/vadmin/opts/stopdb/opts.go b/pkg/vadmin/opts/stopdb/opts.go index 564956280..da0d0b6c3 100644 --- a/pkg/vadmin/opts/stopdb/opts.go +++ b/pkg/vadmin/opts/stopdb/opts.go @@ -25,6 +25,7 @@ type Parms struct { InitiatorIP string Sandbox string ZeroDrain bool + DrainSeconds int } type Option func(*Parms) @@ -54,3 +55,9 @@ func WithZeroDrain(hasZeroDrainSeconds bool) Option { s.ZeroDrain = hasZeroDrainSeconds } } + +func WithDrainSeconds(drainSeconds int) Option { + return func(s *Parms) { + s.DrainSeconds = drainSeconds + } +} diff --git a/pkg/vadmin/opts/stopsubcluster/opts.go b/pkg/vadmin/opts/stopsubcluster/opts.go new file mode 100644 index 000000000..4a817e830 --- /dev/null +++ b/pkg/vadmin/opts/stopsubcluster/opts.go @@ -0,0 +1,57 @@ +/* + (c) Copyright [2021-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package stopsubcluster + +// Parms holds all of the option for a stop DB invocation. +type Parms struct { + InitiatorIP string + SCName string + DrainSeconds int + Force bool +} + +type Option func(*Parms) + +// Make will fill in the Parms based on the options chosen +func (s *Parms) Make(opts ...Option) { + for _, opt := range opts { + opt(s) + } +} + +func WithInitiator(ip string) Option { + return func(s *Parms) { + s.InitiatorIP = ip + } +} + +func WithSCName(scName string) Option { + return func(s *Parms) { + s.SCName = scName + } +} + +func WithDrainSeconds(drainSeconds int) Option { + return func(s *Parms) { + s.DrainSeconds = drainSeconds + } +} + +func WithForce(force bool) Option { + return func(s *Parms) { + s.Force = force + } +} diff --git a/pkg/vadmin/stop_db_vc.go b/pkg/vadmin/stop_db_vc.go index e59b5cd15..abd10f80e 100644 --- a/pkg/vadmin/stop_db_vc.go +++ b/pkg/vadmin/stop_db_vc.go @@ -25,6 +25,8 @@ import ( ) // StopDB will stop all the vertica hosts of a running cluster +// +//nolint:dupl func (v *VClusterOps) StopDB(_ context.Context, opts ...stopdb.Option) error { v.setupForAPICall("StopDB") defer v.tearDownForAPICall() @@ -56,9 +58,11 @@ func (v *VClusterOps) genStopDBOptions(s *stopdb.Parms) vops.VStopDatabaseOption opts.DBName = v.VDB.Spec.DBName opts.IsEon = v.VDB.IsEON() // we use zero timeout to stop special sandbox for only replication upgrade + opts.DrainSeconds = new(int) if s.ZeroDrain { - opts.DrainSeconds = new(int) *opts.DrainSeconds = 0 + } else { + opts.DrainSeconds = &s.DrainSeconds } opts.SandboxName = s.Sandbox // We want to stop db on either the main cluster or a sandbox, diff --git a/pkg/vadmin/stop_sc_at.go b/pkg/vadmin/stop_sc_at.go new file mode 100644 index 000000000..f60ea6ad8 --- /dev/null +++ b/pkg/vadmin/stop_sc_at.go @@ -0,0 +1,27 @@ +/* + (c) Copyright [2021-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vadmin + +import ( + "context" + "errors" + + "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/stopsubcluster" +) + +func (a *Admintools) StopSubcluster(_ context.Context, opts ...stopsubcluster.Option) error { + return errors.New("StopSubcluster is not supported for admintools deployments") +} diff --git a/pkg/vadmin/stop_sc_vc.go b/pkg/vadmin/stop_sc_vc.go new file mode 100644 index 000000000..de426efda --- /dev/null +++ b/pkg/vadmin/stop_sc_vc.go @@ -0,0 +1,69 @@ +/* + (c) Copyright [2021-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vadmin + +import ( + "context" + + vops "github.com/vertica/vcluster/vclusterops" + "github.com/vertica/vertica-kubernetes/pkg/net" + "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/stopsubcluster" +) + +// StopSubcluster will stop the subcluster hosts of a running Vertica db +// +//nolint:dupl +func (v *VClusterOps) StopSubcluster(_ context.Context, opts ...stopsubcluster.Option) error { + v.setupForAPICall("StopSubcluster") + defer v.tearDownForAPICall() + v.Log.Info("Starting vcluster StopSubcluster") + + // get stop_sc options + s := stopsubcluster.Parms{} + s.Make(opts...) + + // call vcluster-ops library to stop subcluster + vopts := v.genStopSubclusterOptions(&s) + err := v.VStopSubcluster(&vopts) + if err != nil { + v.Log.Error(err, "failed to stop a subcluster") + return err + } + + v.Log.Info("Successfully stopped a subcluster", "scName", vopts.SCName) + return nil +} + +func (v *VClusterOps) genStopSubclusterOptions(s *stopsubcluster.Parms) vops.VStopSubclusterOptions { + opts := vops.VStopSubclusterOptionsFactory() + + opts.RawHosts = append(opts.RawHosts, s.InitiatorIP) + v.Log.Info("Setup stop subcluster options", "hosts", opts.RawHosts[0]) + opts.IPv6 = net.IsIPv6(s.InitiatorIP) + + opts.DBName = v.VDB.Spec.DBName + opts.IsEon = v.VDB.IsEON() + + opts.SCName = s.SCName + opts.DrainSeconds = s.DrainSeconds + opts.Force = s.Force + + // auth options + opts.UserName = v.VDB.GetVerticaUser() + opts.Password = &v.Password + + return opts +} diff --git a/pkg/vadmin/stop_sc_vc_test.go b/pkg/vadmin/stop_sc_vc_test.go new file mode 100644 index 000000000..9dfb2c050 --- /dev/null +++ b/pkg/vadmin/stop_sc_vc_test.go @@ -0,0 +1,62 @@ +/* + (c) Copyright [2021-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vadmin + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + vops "github.com/vertica/vcluster/vclusterops" + "github.com/vertica/vertica-kubernetes/pkg/vadmin/opts/stopsubcluster" +) + +const scName = "sc1" + +// mock version of VStopDatabase() that is invoked inside VClusterOps.StopDB() +func (m *MockVClusterOps) VStopSubcluster(options *vops.VStopSubclusterOptions) error { + // verify common options + err := m.VerifyCommonOptions(&options.DatabaseOptions) + if err != nil { + return err + } + + // verify hosts and eon mode + err = m.VerifyInitiatorIPAndEonMode(&options.DatabaseOptions) + if err != nil { + return err + } + + if options.SCName != scName { + return fmt.Errorf("failed to retrieve subcluster name") + } + return nil +} + +var _ = Describe("stop_subcluster_vc", func() { + ctx := context.Background() + + It("should call vcluster-ops library with stop_subcluster task", func() { + dispatcher := mockVClusterOpsDispatcher() + dispatcher.VDB.Spec.DBName = TestDBName + opts := []stopsubcluster.Option{ + stopsubcluster.WithInitiator(TestInitiatorIP), + stopsubcluster.WithSCName(scName), + } + Ω(dispatcher.StopSubcluster(ctx, opts...)).Should(Succeed()) + }) +}) diff --git a/tests/e2e-leg-10/shutdown-subcluster/05-create-secrets.yaml b/tests/e2e-leg-10/shutdown-subcluster/05-create-secrets.yaml new file mode 100644 index 000000000..ecced2c12 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/05-create-secrets.yaml @@ -0,0 +1,19 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE + - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE + - script: kustomize build ../../manifests/vertica-license/overlay | kubectl apply -f - --namespace $NAMESPACE diff --git a/tests/e2e-leg-10/shutdown-subcluster/10-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/10-assert.yaml new file mode 100644 index 000000000..9faca75c9 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/10-assert.yaml @@ -0,0 +1,21 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + namespace: verticadb-operator + labels: + control-plane: verticadb-operator +status: + phase: Running diff --git a/tests/e2e-leg-10/shutdown-subcluster/10-verify-operator.yaml b/tests/e2e-leg-10/shutdown-subcluster/10-verify-operator.yaml new file mode 100644 index 000000000..0cd372046 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/10-verify-operator.yaml @@ -0,0 +1,12 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/e2e-leg-10/shutdown-subcluster/15-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/15-assert.yaml new file mode 100644 index 000000000..ff9584756 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/15-assert.yaml @@ -0,0 +1,52 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-pri1 +status: + currentReplicas: 4 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-pri2 +status: + currentReplicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec1 +status: + currentReplicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec2 +status: + currentReplicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec3 +status: + currentReplicas: 1 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown diff --git a/tests/e2e-leg-10/shutdown-subcluster/15-setup-vdb.yaml b/tests/e2e-leg-10/shutdown-subcluster/15-setup-vdb.yaml new file mode 100644 index 000000000..c9ed69089 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/15-setup-vdb.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-leg-10/shutdown-subcluster/20-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/20-assert.yaml new file mode 100644 index 000000000..f09ca43f9 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/20-assert.yaml @@ -0,0 +1,69 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-pri1 +status: + currentReplicas: 4 + readyReplicas: 4 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-pri2 +status: + currentReplicas: 3 + readyReplicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec1 +status: + currentReplicas: 3 + readyReplicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec2 +status: + currentReplicas: 1 + readyReplicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec3 +status: + currentReplicas: 1 + readyReplicas: 1 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +status: + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + - addedToDBCount: 3 + upNodeCount: 3 + - addedToDBCount: 3 + upNodeCount: 3 + - addedToDBCount: 1 + upNodeCount: 1 + - addedToDBCount: 1 + upNodeCount: 1 diff --git a/tests/e2e-leg-10/shutdown-subcluster/20-wait-for-createdb.yaml b/tests/e2e-leg-10/shutdown-subcluster/20-wait-for-createdb.yaml new file mode 100644 index 000000000..bf3726035 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/20-wait-for-createdb.yaml @@ -0,0 +1 @@ +# Intentionally empty to give this step a name in kuttl \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/25-wait-for-steady-state.yaml b/tests/e2e-leg-10/shutdown-subcluster/25-wait-for-steady-state.yaml new file mode 100644 index 000000000..7cd771cbb --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/25-wait-for-steady-state.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n verticadb-operator -t 360 $NAMESPACE" diff --git a/tests/e2e-leg-10/shutdown-subcluster/30-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/30-assert.yaml new file mode 100644 index 000000000..f3a75066b --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/30-assert.yaml @@ -0,0 +1,61 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Event +reason: SandboxSubclusterSucceeded +source: + component: verticadb-operator +involvedObject: + apiVersion: vertica.com/v1 + kind: VerticaDB + name: v-subcluster-shutdown +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + subclusters: + - name: pri1 + type: primary + - name: pri2 + type: primary + - name: sec1 + type: sandboxprimary + - name: sec2 + type: secondary + - name: sec3 + type: secondary +status: + sandboxes: + - name: sandbox1 + subclusters: + - sec1 + - sec3 + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + name: pri1 + - addedToDBCount: 3 + upNodeCount: 3 + name: pri2 + - addedToDBCount: 3 + upNodeCount: 3 + name: sec1 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec2 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/30-sandbox-sec1-sec3.yaml b/tests/e2e-leg-10/shutdown-subcluster/30-sandbox-sec1-sec3.yaml new file mode 100644 index 000000000..c31fac00e --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/30-sandbox-sec1-sec3.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + sandboxes: + - name: sandbox1 + subclusters: + - name: sec1 + - name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/35-wait-for-steady-state.yaml b/tests/e2e-leg-10/shutdown-subcluster/35-wait-for-steady-state.yaml new file mode 100644 index 000000000..7cd771cbb --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/35-wait-for-steady-state.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n verticadb-operator -t 360 $NAMESPACE" diff --git a/tests/e2e-leg-10/shutdown-subcluster/40-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/40-assert.yaml new file mode 100644 index 000000000..f3a097ab6 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/40-assert.yaml @@ -0,0 +1,74 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Event +reason: StopSubclusterSucceeded +source: + component: verticadb-operator +involvedObject: + apiVersion: vertica.com/v1 + kind: VerticaDB + name: v-subcluster-shutdown +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec2 +status: + replicas: 0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec3 +status: + replicas: 0 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + subclusters: + - name: pri1 + type: primary + - name: pri2 + type: primary + - name: sec1 + type: sandboxprimary + - name: sec2 + type: secondary + shutdown: true + - name: sec3 + shutdown: true + type: secondary +status: + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + name: pri1 + - addedToDBCount: 3 + upNodeCount: 3 + name: pri2 + - addedToDBCount: 3 + upNodeCount: 3 + name: sec1 + - addedToDBCount: 0 + upNodeCount: 0 + shutdown: true + name: sec2 + - addedToDBCount: 0 + upNodeCount: 0 + shutdown: true + name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/40-shutdown-secondaries.yaml b/tests/e2e-leg-10/shutdown-subcluster/40-shutdown-secondaries.yaml new file mode 100644 index 000000000..118f40663 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/40-shutdown-secondaries.yaml @@ -0,0 +1,37 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + subclusters: + - name: pri1 + size: 4 + type: primary + - name: pri2 + size: 3 + type: primary + - name: sec1 + size: 3 + type: sandboxprimary + - name: sec2 + size: 1 + shutdown: true + type: secondary + - name: sec3 + size: 1 + shutdown: true + type: secondary + diff --git a/tests/e2e-leg-10/shutdown-subcluster/45-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/45-assert.yaml new file mode 100644 index 000000000..e4478d11a --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/45-assert.yaml @@ -0,0 +1,50 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec2 +status: + replicas: 0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec3 +status: + replicas: 0 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +status: + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + name: pri1 + - addedToDBCount: 3 + upNodeCount: 2 + name: pri2 + - addedToDBCount: 3 + upNodeCount: 2 + name: sec1 + - addedToDBCount: 0 + upNodeCount: 0 + shutdown: true + name: sec2 + - addedToDBCount: 0 + upNodeCount: 0 + shutdown: true + name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/45-kill-pods.yaml b/tests/e2e-leg-10/shutdown-subcluster/45-kill-pods.yaml new file mode 100644 index 000000000..bb0aab0bd --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/45-kill-pods.yaml @@ -0,0 +1,18 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl delete pods v-subcluster-shutdown-pri2-0 v-subcluster-shutdown-sec1-0 + namespaced: true \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/50-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/50-assert.yaml new file mode 100644 index 000000000..88732a0e9 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/50-assert.yaml @@ -0,0 +1,50 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec2 +status: + replicas: 0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-sec3 +status: + replicas: 0 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +status: + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + name: pri1 + - addedToDBCount: 3 + upNodeCount: 3 + name: pri2 + - addedToDBCount: 3 + upNodeCount: 3 + name: sec1 + - addedToDBCount: 0 + upNodeCount: 0 + shutdown: true + name: sec2 + - addedToDBCount: 0 + upNodeCount: 0 + shutdown: true + name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/50-wait-for-pods-up.yaml b/tests/e2e-leg-10/shutdown-subcluster/50-wait-for-pods-up.yaml new file mode 100644 index 000000000..bf3726035 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/50-wait-for-pods-up.yaml @@ -0,0 +1 @@ +# Intentionally empty to give this step a name in kuttl \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/55-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/55-assert.yaml new file mode 100644 index 000000000..c68f0d20f --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/55-assert.yaml @@ -0,0 +1,38 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +status: + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + name: pri1 + - addedToDBCount: 3 + upNodeCount: 3 + name: pri2 + - addedToDBCount: 3 + upNodeCount: 3 + name: sec1 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec2 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/55-restart-secondaries.yaml b/tests/e2e-leg-10/shutdown-subcluster/55-restart-secondaries.yaml new file mode 100644 index 000000000..c2f6792cc --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/55-restart-secondaries.yaml @@ -0,0 +1,37 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + subclusters: + - name: pri1 + size: 4 + type: primary + - name: pri2 + size: 3 + type: primary + - name: sec1 + size: 3 + type: sandboxprimary + - name: sec2 + size: 1 + shutdown: false + type: secondary + - name: sec3 + size: 1 + shutdown: false + type: secondary + diff --git a/tests/e2e-leg-10/shutdown-subcluster/60-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/60-assert.yaml new file mode 100644 index 000000000..978587396 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/60-assert.yaml @@ -0,0 +1,66 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Event +reason: ClusterWillLoseQuorum +source: + component: verticadb-operator +involvedObject: + apiVersion: vertica.com/v1 + kind: VerticaDB + name: v-subcluster-shutdown +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-subcluster-shutdown-pri2 +status: + replicas: 0 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + subclusters: + - name: pri1 + type: primary + - name: pri2 + type: primary + shutdown: true + - name: sec1 + type: sandboxprimary + shutdown: true + - name: sec2 + type: secondary + - name: sec3 + type: secondary +status: + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + name: pri1 + - addedToDBCount: 0 + upNodeCount: 0 + shutdown: true + name: pri2 + - addedToDBCount: 3 + upNodeCount: 3 + name: sec1 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec2 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/60-shutdown-primaries.yaml b/tests/e2e-leg-10/shutdown-subcluster/60-shutdown-primaries.yaml new file mode 100644 index 000000000..32092f6c6 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/60-shutdown-primaries.yaml @@ -0,0 +1,37 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + subclusters: + - name: pri1 + size: 4 + type: primary + - name: pri2 + size: 3 + shutdown: true + type: primary + - name: sec1 + size: 3 + shutdown: true + type: sandboxprimary + - name: sec2 + size: 1 + type: secondary + - name: sec3 + size: 1 + type: secondary + diff --git a/tests/e2e-leg-10/shutdown-subcluster/65-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/65-assert.yaml new file mode 100644 index 000000000..c68f0d20f --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/65-assert.yaml @@ -0,0 +1,38 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +status: + subclusters: + - addedToDBCount: 4 + upNodeCount: 4 + name: pri1 + - addedToDBCount: 3 + upNodeCount: 3 + name: pri2 + - addedToDBCount: 3 + upNodeCount: 3 + name: sec1 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec2 + - addedToDBCount: 1 + upNodeCount: 1 + name: sec3 \ No newline at end of file diff --git a/tests/e2e-leg-10/shutdown-subcluster/65-restart-primaries.yaml b/tests/e2e-leg-10/shutdown-subcluster/65-restart-primaries.yaml new file mode 100644 index 000000000..312e80748 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/65-restart-primaries.yaml @@ -0,0 +1,36 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown +spec: + subclusters: + - name: pri1 + size: 4 + type: primary + - name: pri2 + size: 3 + shutdown: false + type: primary + - name: sec1 + size: 3 + shutdown: false + type: sandboxprimary + - name: sec2 + size: 1 + type: secondary + - name: sec3 + size: 1 + type: secondary diff --git a/tests/e2e-leg-10/shutdown-subcluster/95-delete-cr.yaml b/tests/e2e-leg-10/shutdown-subcluster/95-delete-cr.yaml new file mode 100644 index 000000000..96d43c1de --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/95-delete-cr.yaml @@ -0,0 +1,18 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: vertica.com/v1 + kind: VerticaDB diff --git a/tests/e2e-leg-10/shutdown-subcluster/95-errors.yaml b/tests/e2e-leg-10/shutdown-subcluster/95-errors.yaml new file mode 100644 index 000000000..0de5ac42e --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/95-errors.yaml @@ -0,0 +1,24 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: verticadb-operator +--- +apiVersion: vertica.com/v1 +kind: VerticaDB diff --git a/tests/e2e-leg-10/shutdown-subcluster/96-assert.yaml b/tests/e2e-leg-10/shutdown-subcluster/96-assert.yaml new file mode 100644 index 000000000..861c7dc8f --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/96-assert.yaml @@ -0,0 +1,19 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: clean-communal +status: + phase: Succeeded diff --git a/tests/e2e-leg-10/shutdown-subcluster/96-cleanup-storage.yaml b/tests/e2e-leg-10/shutdown-subcluster/96-cleanup-storage.yaml new file mode 100644 index 000000000..dcc6306f3 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/96-cleanup-storage.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "kustomize build clean-communal/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-leg-10/shutdown-subcluster/96-errors.yaml b/tests/e2e-leg-10/shutdown-subcluster/96-errors.yaml new file mode 100644 index 000000000..671be36cf --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/96-errors.yaml @@ -0,0 +1,15 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolumeClaim diff --git a/tests/e2e-leg-10/shutdown-subcluster/99-delete-ns.yaml b/tests/e2e-leg-10/shutdown-subcluster/99-delete-ns.yaml new file mode 100644 index 000000000..1674b3e8f --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/99-delete-ns.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl delete ns $NAMESPACE diff --git a/tests/e2e-leg-10/shutdown-subcluster/setup-vdb/base/kustomization.yaml b/tests/e2e-leg-10/shutdown-subcluster/setup-vdb/base/kustomization.yaml new file mode 100644 index 000000000..681396735 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/setup-vdb/base/kustomization.yaml @@ -0,0 +1,15 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - setup-vdb.yaml diff --git a/tests/e2e-leg-10/shutdown-subcluster/setup-vdb/base/setup-vdb.yaml b/tests/e2e-leg-10/shutdown-subcluster/setup-vdb/base/setup-vdb.yaml new file mode 100644 index 000000000..a24c99288 --- /dev/null +++ b/tests/e2e-leg-10/shutdown-subcluster/setup-vdb/base/setup-vdb.yaml @@ -0,0 +1,48 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-subcluster-shutdown + annotations: + vertica.com/include-uid-in-path: true +spec: + image: kustomize-vertica-image + dbName: shutdown_sc + initPolicy: CreateSkipPackageInstall + communal: {} + local: + requestSize: 250Mi + # We pick a cluster size that is too big for the CE license. This relies on + # having a license, which will be added by kustomize. + subclusters: + - name: pri1 + size: 4 + type: primary + - name: pri2 + size: 3 + type: primary + - name: sec1 + size: 3 + type: secondary + - name: sec2 + size: 1 + type: secondary + - name: sec3 + size: 1 + type: secondary + certSecrets: [] + imagePullSecrets: [] + volumes: [] + volumeMounts: []