Skip to content

Commit

Permalink
PWX-30230 : Create all px configmaps in same namespace as portworx
Browse files Browse the repository at this point in the history
  • Loading branch information
nikita-bhatia committed Apr 28, 2023
1 parent 272b607 commit 73ac0bd
Show file tree
Hide file tree
Showing 5 changed files with 286 additions and 172 deletions.
229 changes: 72 additions & 157 deletions k8s/core/configmap/configmap.go
Original file line number Diff line number Diff line change
@@ -1,188 +1,103 @@
package configmap

import (
"fmt"
"strings"
"time"

"github.com/portworx/sched-ops/k8s/core"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/pborman/uuid"
log "github.com/sirupsen/logrus"
)

// New returns the ConfigMap interface. It also creates a new
// configmap in k8s for the given name if not present and puts the data in it.
func New(
name string,
data map[string]string,
lockTimeout time.Duration,
lockAttempts uint,
v2LockRefreshDuration time.Duration,
v2LockK8sLockTTL time.Duration,
) (ConfigMap, error) {
if data == nil {
data = make(map[string]string)
}

labels := map[string]string{
configMapUserLabelKey: TruncateLabel(name),
}
data[pxOwnerKey] = ""

cm := &corev1.ConfigMap{
ObjectMeta: meta_v1.ObjectMeta{
Name: name,
Namespace: k8sSystemNamespace,
Labels: labels,
},
Data: data,
}
func (c *configMap) Instance() ConfigMap {

if _, err := core.Instance().CreateConfigMap(cm); err != nil &&
!k8s_errors.IsAlreadyExists(err) {
return nil, fmt.Errorf("failed to create configmap %v: %v",
name, err)
}

if v2LockK8sLockTTL == 0 {
v2LockK8sLockTTL = v2DefaultK8sLockTTL
}
//iskvdbhealthy
//kvdb.instance

if v2LockRefreshDuration == 0 {
v2LockRefreshDuration = v2DefaultK8sLockRefreshDuration
if c.pxNs == c.config.nameSpace {
//fresh install
//upgrade completed
return c.config
} else {
existingConfig := c.config
c.copylock.Lock(uuid.New())
defer c.copylock.Unlock()
lockMap, err := c.copylock.Get()
if err != nil {
log.Error("Error during fetching data from copy lock %s", err)
return existingConfig
}
status := lockMap["UPGRADE_DONE"]
if status == "true" {
// upgrade is completed
//create configmap in portworx namespace
newConfig := &coreConfigMap{
name: existingConfig.name,
defaultLockHoldTimeout: existingConfig.defaultLockHoldTimeout,
kLocksV2: existingConfig.kLocksV2,
lockAttempts: existingConfig.lockAttempts,
lockRefreshDuration: existingConfig.lockRefreshDuration,
lockK8sLockTTL: existingConfig.lockK8sLockTTL,
nameSpace: "portworx",
}

configData, err := existingConfig.Get()
if err != nil {
log.Errorf("Error during fetching data from old config map %s", err)
return existingConfig
}
//copy data from old configmap to new configmap
if err = newConfig.Update(configData); err != nil {
log.Errorf("Error during copying data from old config map %s", err)
return existingConfig
}

//delete old configmap
err = c.config.Delete()
if err != nil {
log.Errorf("Error during deleting configmap %s in namespace %s ", c.config.name, c.config.nameSpace)
}
c.config = newConfig
} else {
return existingConfig
}
}

return &configMap{
name: name,
defaultLockHoldTimeout: lockTimeout,
kLocksV2: map[string]*k8sLock{},
lockAttempts: lockAttempts,
lockRefreshDuration: v2LockRefreshDuration,
lockK8sLockTTL: v2LockK8sLockTTL,
}, nil
return c.config
}

func (c *configMap) Get() (map[string]string, error) {
cm, err := core.Instance().GetConfigMap(
c.name,
k8sSystemNamespace,
)
if err != nil {
return nil, err
}

return cm.Data, nil
return c.Instance().Get()
}

func (c *configMap) Delete() error {
return core.Instance().DeleteConfigMap(
c.name,
k8sSystemNamespace,
)
return c.Instance().Delete()
}

func (c *configMap) Patch(data map[string]string) error {
var (
err error
cm *corev1.ConfigMap
)
for retries := 0; retries < maxConflictRetries; retries++ {
cm, err = core.Instance().GetConfigMap(
c.name,
k8sSystemNamespace,
)
if err != nil {
return err
}

if cm.Data == nil {
cm.Data = make(map[string]string, 0)
}

for k, v := range data {
cm.Data[k] = v
}
_, err = core.Instance().UpdateConfigMap(cm)
if k8s_errors.IsConflict(err) {
// try again
continue
}
return err
}
return err
return c.Instance().Patch(data)
}

func (c *configMap) Update(data map[string]string) error {
var (
err error
cm *corev1.ConfigMap
)
for retries := 0; retries < maxConflictRetries; retries++ {
cm, err = core.Instance().GetConfigMap(
c.name,
k8sSystemNamespace,
)
if err != nil {
return err
}
cm.Data = data
_, err = core.Instance().UpdateConfigMap(cm)
if k8s_errors.IsConflict(err) {
// try again
continue
}
return err
}
return err
return c.Instance().Update(data)
}

// SetFatalCb sets the fatal callback for the package which will get invoked in panic situations
func SetFatalCb(fb FatalCb) {
fatalCb = fb
func (c *configMap) Lock(id string) error {
return c.Instance().Lock(id)
}

func configMapLog(fn, name, owner, key string, err error) *logrus.Entry {
if len(owner) > 0 && len(key) > 0 {
return logrus.WithFields(logrus.Fields{
"Module": "ConfigMap",
"Name": name,
"Owner": owner,
"Key": key,
"Function": fn,
"Error": err,
})
}
if len(owner) > 0 {
return logrus.WithFields(logrus.Fields{
"Module": "ConfigMap",
"Name": name,
"Owner": owner,
"Function": fn,
"Error": err,
})
}
return logrus.WithFields(logrus.Fields{
"Module": "ConfigMap",
"Name": name,
"Function": fn,
"Error": err,
})
func (c *configMap) LockWithHoldTimeout(id string, holdTimeout time.Duration) error {
return c.Instance().LockWithHoldTimeout(id, holdTimeout)
}

// GetName is a helper function that returns a valid k8s
// configmap name given a prefix identifying the component using
// the configmap and a clusterID
func GetName(prefix, clusterID string) string {
return prefix + strings.ToLower(configMapNameRegex.ReplaceAllString(clusterID, ""))
func (c *configMap) LockWithKey(owner, key string) error {
return c.Instance().LockWithKey(owner, key)
}

// TruncateLabel is a helper function that returns a valid k8s
// label stripped down to 63 characters. It removes the trailing characters
func TruncateLabel(label string) string {
if len(label) > 63 {
return label[:63]
}
return label
func (c *configMap) Unlock() error {
return c.Instance().Unlock()
}

func (c *configMap) UnlockWithKey(key string) error {
return c.Instance().UnlockWithKey(key)
}
func (c *configMap) IsKeyLocked(key string) (bool, string, error) {
return c.Instance().IsKeyLocked(key)
}
10 changes: 5 additions & 5 deletions k8s/core/configmap/configmap_lock_v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ import (
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
)

func (c *configMap) Lock(id string) error {
func (c *coreConfigMap) Lock(id string) error {
return c.LockWithHoldTimeout(id, c.defaultLockHoldTimeout)
}

func (c *configMap) LockWithHoldTimeout(id string, holdTimeout time.Duration) error {
func (c *coreConfigMap) LockWithHoldTimeout(id string, holdTimeout time.Duration) error {
fn := "LockWithHoldTimeout"
count := uint(0)
// try acquiring a lock on the ConfigMap
Expand Down Expand Up @@ -40,7 +40,7 @@ func (c *configMap) LockWithHoldTimeout(id string, holdTimeout time.Duration) er
return nil
}

func (c *configMap) Unlock() error {
func (c *coreConfigMap) Unlock() error {
fn := "Unlock"
// Get the existing ConfigMap
c.kLockV1.Lock()
Expand Down Expand Up @@ -91,7 +91,7 @@ func (c *configMap) Unlock() error {
return err
}

func (c *configMap) tryLockV1(id string, refresh bool) (string, error) {
func (c *coreConfigMap) tryLockV1(id string, refresh bool) (string, error) {
// Get the existing ConfigMap
cm, err := core.Instance().GetConfigMap(
c.name,
Expand Down Expand Up @@ -140,7 +140,7 @@ increase_expiry:
return id, nil
}

func (c *configMap) refreshLockV1(id string) {
func (c *coreConfigMap) refreshLockV1(id string) {
fn := "refreshLock"
refresh := time.NewTicker(v1DefaultK8sLockRefreshDuration)
var (
Expand Down
20 changes: 10 additions & 10 deletions k8s/core/configmap/configmap_lock_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
k8s_errors "k8s.io/apimachinery/pkg/api/errors"
)

func (c *configMap) LockWithKey(owner, key string) error {
func (c *coreConfigMap) LockWithKey(owner, key string) error {
if key == "" {
return fmt.Errorf("key cannot be empty")
}
Expand Down Expand Up @@ -54,7 +54,7 @@ func (c *configMap) LockWithKey(owner, key string) error {
return nil
}

func (c *configMap) UnlockWithKey(key string) error {
func (c *coreConfigMap) UnlockWithKey(key string) error {
if key == "" {
return fmt.Errorf("key cannot be empty")
}
Expand Down Expand Up @@ -136,7 +136,7 @@ func (c *configMap) UnlockWithKey(key string) error {
return err
}

func (c *configMap) IsKeyLocked(key string) (bool, string, error) {
func (c *coreConfigMap) IsKeyLocked(key string) (bool, string, error) {
// Get the existing ConfigMap
cm, err := core.Instance().GetConfigMap(
c.name,
Expand Down Expand Up @@ -170,7 +170,7 @@ func (c *configMap) IsKeyLocked(key string) (bool, string, error) {
return false, "", nil
}

func (c *configMap) tryLock(owner string, key string) (string, error) {
func (c *coreConfigMap) tryLock(owner string, key string) (string, error) {
// Get the existing ConfigMap
cm, err := core.Instance().GetConfigMap(
c.name,
Expand Down Expand Up @@ -209,7 +209,7 @@ func (c *configMap) tryLock(owner string, key string) (string, error) {
// parseLocks reads the lock data from the given ConfigMap and then converts it to:
// * a map of keys to lock owners
// * a map of keys to lock expiration times
func (c *configMap) parseLocks(cm *v1.ConfigMap) (map[string]string, map[string]time.Time, error) {
func (c *coreConfigMap) parseLocks(cm *v1.ConfigMap) (map[string]string, map[string]time.Time, error) {
// Check all the locks: will be an empty string if key is not present indicating no lock
parsedLocks := []lockData{}
if lock, ok := cm.Data[pxLockKey]; ok && len(lock) > 0 {
Expand All @@ -233,7 +233,7 @@ func (c *configMap) parseLocks(cm *v1.ConfigMap) (map[string]string, map[string]

// checkAndTakeLock tries to take the given lock (owner, key) given the current state of the lock
// (lockOwners, lockExpirations).
func (c *configMap) checkAndTakeLock(
func (c *coreConfigMap) checkAndTakeLock(
owner, key string,
lockOwners map[string]string,
lockExpirations map[string]time.Time,
Expand Down Expand Up @@ -279,7 +279,7 @@ func (c *configMap) checkAndTakeLock(

// generateConfigMapData converts the given lock data (lockOwners, lockExpirations) to JSON and
// stores it in the given ConfigMap.
func (c *configMap) generateConfigMapData(cm *v1.ConfigMap, lockOwners map[string]string, lockExpirations map[string]time.Time) error {
func (c *coreConfigMap) generateConfigMapData(cm *v1.ConfigMap, lockOwners map[string]string, lockExpirations map[string]time.Time) error {
var locks []lockData
for key, lockOwner := range lockOwners {
locks = append(locks, lockData{
Expand All @@ -297,7 +297,7 @@ func (c *configMap) generateConfigMapData(cm *v1.ConfigMap, lockOwners map[strin
return nil
}

func (c *configMap) updateConfigMap(cm *v1.ConfigMap) (bool, error) {
func (c *coreConfigMap) updateConfigMap(cm *v1.ConfigMap) (bool, error) {
if _, err := core.Instance().UpdateConfigMap(cm); err != nil {
return k8s_errors.IsConflict(err), err
}
Expand All @@ -308,7 +308,7 @@ func (c *configMap) updateConfigMap(cm *v1.ConfigMap) (bool, error) {
// It keeps the lock refreshed in k8s until we call Unlock. This is so that if the
// node dies, the lock can have a short timeout and expire quickly but we can still
// take longer-term locks.
func (c *configMap) refreshLock(id, key string) {
func (c *coreConfigMap) refreshLock(id, key string) {
fn := "refreshLock"
refresh := time.NewTicker(c.lockRefreshDuration)
var (
Expand Down Expand Up @@ -356,7 +356,7 @@ func (c *configMap) refreshLock(id, key string) {

}

func (c *configMap) checkLockTimeout(holdTimeout time.Duration, startTime time.Time, id string) {
func (c *coreConfigMap) checkLockTimeout(holdTimeout time.Duration, startTime time.Time, id string) {
if holdTimeout > 0 && time.Since(startTime) > holdTimeout {
panicMsg := fmt.Sprintf("Lock hold timeout (%v) triggered for K8s configmap lock key %s", holdTimeout, id)
if fatalCb != nil {
Expand Down
Loading

0 comments on commit 73ac0bd

Please sign in to comment.