From 3c50bbf1a4a0b8deb9cb9f46dacb338fd85a43b0 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Mon, 2 Sep 2024 16:33:38 +0200 Subject: [PATCH 01/16] Fix missing RunTimeout field in task drivers The kw declaration is actually working now it has moved from core to drivers, so better have the corresponding field, or all tasks are ignored in services. --- drivers/restaskdocker/main.go | 1 + drivers/restaskhost/main.go | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/restaskdocker/main.go b/drivers/restaskdocker/main.go index f8ef82820..28cad4f71 100644 --- a/drivers/restaskdocker/main.go +++ b/drivers/restaskdocker/main.go @@ -69,6 +69,7 @@ type T struct { IPCNS string `json:"ipcns"` UTSNS string `json:"utsns"` RegistryCreds string `json:"registry_creds"` + RunTimeout *time.Duration `json:"run_timeout"` PullTimeout *time.Duration `json:"pull_timeout"` Timeout *time.Duration `json:"timeout"` diff --git a/drivers/restaskhost/main.go b/drivers/restaskhost/main.go index 791699265..a774d7316 100644 --- a/drivers/restaskhost/main.go +++ b/drivers/restaskhost/main.go @@ -32,12 +32,13 @@ import ( // T is the driver structure. type T struct { resapp.T - RunCmd string - OnErrorCmd string Check string - Schedule string Confirmation bool LogOutputs bool + OnErrorCmd string + RunCmd string + RunTimeout *time.Duration + Schedule string Snooze *time.Duration } From 091bdc16e5aee3070096d66c9dee3c8ce50d07e9 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 4 Sep 2024 11:19:16 +0200 Subject: [PATCH 02/16] Remove a unnecessary test in the object run codepath --- core/object/actor_run.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/object/actor_run.go b/core/object/actor_run.go index 578388be0..f59679355 100644 --- a/core/object/actor_run.go +++ b/core/object/actor_run.go @@ -28,9 +28,6 @@ func (t *actor) masterRun(ctx context.Context) error { return t.action(ctx, func(ctx context.Context, r resource.Driver) error { t.log.Attr("rid", r.RID()).Debugf("run resource") err := resource.Run(ctx, r) - if err == nil { - return nil - } if errors.Is(err, resource.ErrActionReqNotMet) && actioncontext.IsCron(ctx) { return nil } From c6b6a1ffd7ae8474bad6e5ceadeffd7711907e02 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 4 Sep 2024 11:21:19 +0200 Subject: [PATCH 03/16] Make task and sync resources optional=false by default And change the instance status evaluation algo to never merge the status of these resources into availstatus. The only role left to optional=true is to continue with the next resource action on error and don't propagate the error to the command exitcode. With this patch, if not explicitely changed by optional=true, a task run error causes a exitcode 1. --- CHANGELOG.md | 2 ++ core/manifest/dbkeywords.go | 14 ++------------ core/manifest/text/kw/optional | 13 ++++++------- core/object/actor_status.go | 8 +++++++- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34af0cac4..8f5ad813e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,8 @@ * **breaking change:** The raw protocol is dropped. `echo | socat - /var/lib/opensvc/lsnr/lsnr.sock` +* **breaking change:** Task and sync resources are now non-optional by default, but their status is never aggregated in the instance availability status. Errors in the run produce a non-zero exitcode if optional=false, zero if optional=true. + ### objects * **breaking change:** drop support of some DEFAULT keywords: diff --git a/core/manifest/dbkeywords.go b/core/manifest/dbkeywords.go index 7a5d7e964..9a449d335 100644 --- a/core/manifest/dbkeywords.go +++ b/core/manifest/dbkeywords.go @@ -122,16 +122,6 @@ var ( Text: keywords.NewText(fs, "text/kw/optional"), } - KWOptionalTrue = keywords.Keyword{ - Attr: "Optional", - Converter: converters.Bool, - Default: "true", - Inherit: keywords.InheritHead2Leaf, - Option: "optional", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/optional"), - } - KWPostProvision = keywords.Keyword{ Attr: "PostProvision", Option: "post_provision", @@ -358,12 +348,12 @@ var ( } syncerKeywords = []Attr{ - KWOptionalTrue, + KWOptional, KWSyncRequires, } runnerKeywords = []Attr{ - KWOptionalTrue, + KWOptional, KWBlockingPostRun, KWBlockingPreRun, KWPostRun, diff --git a/core/manifest/text/kw/optional b/core/manifest/text/kw/optional index 5a63c52ca..7afd44664 100644 --- a/core/manifest/text/kw/optional +++ b/core/manifest/text/kw/optional @@ -1,10 +1,9 @@ -Action errors on optional resources are logged but do not stop the action -sequence. +Action errors on optional resources are logged but do not interrupt the action sequence. -The optional resources status is not aggregated in the instance availability -status, but aggregated in the overall status. +The status of optional resources is not included in the instance availability status but is considered in the overall status. -Resource tagged `noaction` and sync resources are considered optional by -default. +The status of task and sync resources is always included in the overall status, regardless of whether they are marked as optional. -Dump filesystems are a typical use-case for `optional=true`. +Resources tagged as `noaction` are considered optional by default. + +Dump filesystems are a typical use case for optional=true. diff --git a/core/object/actor_status.go b/core/object/actor_status.go index ecbeeb081..b5e838ce3 100644 --- a/core/object/actor_status.go +++ b/core/object/actor_status.go @@ -9,6 +9,7 @@ import ( "github.com/opensvc/om3/core/actioncontext" "github.com/opensvc/om3/core/colorstatus" + "github.com/opensvc/om3/core/driver" "github.com/opensvc/om3/core/instance" "github.com/opensvc/om3/core/provisioned" "github.com/opensvc/om3/core/rawconfig" @@ -199,7 +200,12 @@ func (t *actor) resourceStatusEval(ctx context.Context, data *instance.Status, m data.Resources[r.RID()] = xd data.Overall.Add(xd.Status) if !xd.Optional { - data.Avail.Add(xd.Status) + switch r.ID().DriverGroup() { + case driver.GroupSync: + case driver.GroupTask: + default: + data.Avail.Add(xd.Status) + } } data.Provisioned.Add(xd.Provisioned.State) mu.Unlock() From 7cadb4d5f8c84b2660cc613bc06ef9fba8a56d29 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 4 Sep 2024 11:26:45 +0200 Subject: [PATCH 04/16] Avoid repeating logs from the objectaction humanrenderer --- core/objectaction/object.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/objectaction/object.go b/core/objectaction/object.go index c66c98232..ee9e5e6fb 100644 --- a/core/objectaction/object.go +++ b/core/objectaction/object.go @@ -277,8 +277,8 @@ func rsHumanRender(rs []actionrouter.Result) string { fmt.Printf("%s\n", r.Error) } rs[i].Error = nil - case (r.Error != nil) && fmt.Sprint(r.Error) != "": - log.Error().Msgf("%s: %s", r.Path, r.Error) + // case (r.Error != nil) && fmt.Sprint(r.Error) != "": + // log.Error().Msgf("%s: %s", r.Path, r.Error) case r.Panic != nil: switch err := r.Panic.(type) { case error: From 927d829937df9f3ef302a1d1b1cc62d66c4a99d1 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Wed, 4 Sep 2024 11:28:18 +0200 Subject: [PATCH 05/16] Log optional resources errors as warnings With a downgraded level indication. Example: $ om test/svc/hdoc run --rid task#1 --local --log=info ... 09:15:48.851 WRN instance: test/svc/hdoc: task#1: error from optional resource: run: open /var/lib/opensvc/namespaces/test/svc/hdoc/task#1/run: is a directory --- core/resourceset/resourceset.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/resourceset/resourceset.go b/core/resourceset/resourceset.go index 97637fa99..94f1d5c02 100644 --- a/core/resourceset/resourceset.go +++ b/core/resourceset/resourceset.go @@ -225,13 +225,16 @@ func (t T) doParallel(ctx context.Context, l ResourceLister, resources resource. nResources := len(resources) for i := 0; i < nResources; i++ { res := <-q - if res.Resource.IsOptional() { + switch { + case res.Error == nil: continue - } - if res.Error != nil { + case res.Resource.IsOptional(): + res.Resource.Log().Errorf("error from optional resource: %s", res.Error) + continue + default: res.Resource.Log().Errorf("%s", res.Error) + errors.Join(errs, fmt.Errorf("%s: %w", res.Resource.RID(), res.Error)) } - errors.Join(errs, fmt.Errorf("%s: %w", res.Resource.RID(), res.Error)) } return errs } @@ -253,6 +256,7 @@ func (t T) doSerial(ctx context.Context, l ResourceLister, resources resource.Dr case err == nil: continue case r.IsOptional(): + r.Log().Warnf("error from optional resource: %s", err) continue default: r.Log().Errorf("%s", err) From 99c9a77023f2175237f3cad0a49490c55073f01d Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 5 Sep 2024 08:34:55 +0200 Subject: [PATCH 06/16] Add a util/runfiles package to help manage a runfile directory The tasks support for concurrent runs will need such runfile dirs. The package doc is: VARIABLES var ( ErrProcNotFound = errors.New("process not found") ErrProcTooYoung = errors.New("process too young") ) FUNCTIONS func IsValid(file string) (bool, error) TYPES type Dir struct { Path string Log *plog.Logger } func (t Dir) Count() (int, error) func (t Dir) CountAndClean() (int, error) func (t Dir) Create(content []byte) error func (t Dir) HasRunning() (bool, error) func (t Dir) Remove() error Add the following tests: Count Create CountAndClean CreateStale CountAndCleanWithStale Remove --- util/runfiles/main.go | 165 +++++++++++++++++++++++++++++++++++++ util/runfiles/main_test.go | 53 ++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 util/runfiles/main.go create mode 100644 util/runfiles/main_test.go diff --git a/util/runfiles/main.go b/util/runfiles/main.go new file mode 100644 index 000000000..1a33a4032 --- /dev/null +++ b/util/runfiles/main.go @@ -0,0 +1,165 @@ +package runfiles + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/opensvc/om3/util/plog" +) + +type ( + Dir struct { + Path string + Log *plog.Logger + } + cleaning int +) + +var ( + ErrProcNotFound = errors.New("process not found") + ErrProcTooYoung = errors.New("process too young") +) + +const ( + doClean cleaning = iota + noClean +) + +func (t Dir) filename(pid int) string { + return filepath.Join(t.Path, fmt.Sprint(pid)) +} + +func (t Dir) Remove() error { + return t.remove(os.Getpid()) +} + +func (t Dir) Create(content []byte) error { + return t.create(os.Getpid(), content) +} + +func (t Dir) remove(pid int) error { + filename := t.filename(pid) + return os.Remove(filename) +} + +func (t Dir) create(pid int, content []byte) error { + filename := t.filename(pid) + file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) + switch { + case os.IsNotExist(err): + if err := os.MkdirAll(t.Path, os.ModePerm); err != nil { + return err + } + file, err = os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) + if err != nil { + return err + } + case err != nil: + return err + } + defer file.Close() + defer file.Sync() + _, err = file.Write(content) + return err +} + +func (t Dir) HasRunning() (bool, error) { + var v bool + err := filepath.WalkDir(t.Path, func(path string, e os.DirEntry, err error) error { + if err != nil { + return err + } + if e.IsDir() { + return filepath.SkipDir + } + v, err = IsValid(path) + if err != nil { + return err + } + if v { + return filepath.SkipAll + } + return nil + }) + if os.IsNotExist(err) { + return v, nil + } + return v, err +} + +func IsValid(file string) (bool, error) { + info, err := os.Lstat(file) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + + if !info.Mode().IsRegular() { + return false, nil + } + pid := filepath.Base(file) + procFile := "/proc/" + pid + procInfo, err := os.Lstat(procFile) + if os.IsNotExist(err) { + return false, fmt.Errorf("%w: %s", ErrProcNotFound, pid) + } else if err != nil { + return false, err + } + if info.ModTime().Before(procInfo.ModTime()) { + return false, fmt.Errorf("%w: %s created %s after run file", ErrProcTooYoung, pid, procInfo.ModTime().Sub(info.ModTime())) + } + return true, nil +} + +func (t Dir) count(clean cleaning) (int, error) { + var n int + err := filepath.WalkDir(t.Path, func(path string, e os.DirEntry, err error) error { + if err != nil { + return err + } + if path == t.Path { + return nil + } + if e.IsDir() { + return filepath.SkipDir + } + + v, err := IsValid(path) + if err != nil && !errors.Is(err, ErrProcNotFound) && !errors.Is(err, ErrProcTooYoung) { + return err + } + + if v { + n += 1 + } else if clean == doClean { + removeErr := os.Remove(path) + switch { + case os.IsNotExist(removeErr): + return nil + case removeErr != nil: + return removeErr + } + if t.Log != nil { + if errors.Is(err, ErrProcNotFound) || errors.Is(err, ErrProcTooYoung) { + t.Log.Infof("clean up stale run file (%s)", err) + } + } + } + return nil + }) + if os.IsNotExist(err) { + return 0, nil + } + return n, err +} + +func (t Dir) CountAndClean() (int, error) { + return t.count(doClean) +} + +func (t Dir) Count() (int, error) { + return t.count(noClean) +} diff --git a/util/runfiles/main_test.go b/util/runfiles/main_test.go new file mode 100644 index 000000000..5e780b45d --- /dev/null +++ b/util/runfiles/main_test.go @@ -0,0 +1,53 @@ +package runfiles + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRunFiles(t *testing.T) { + d := Dir{ + Path: t.TempDir(), + } + + t.Run("Count", func(t *testing.T) { + n, err := d.Count() + assert.NoError(t, err) + assert.Zero(t, n) + }) + + t.Run("Create", func(t *testing.T) { + content := "foo" + err := d.Create([]byte(content)) + assert.NoError(t, err) + filename := d.filename(os.Getpid()) + b, err := os.ReadFile(filename) + assert.NoError(t, err) + assert.Equal(t, string(b), content) + }) + + t.Run("CountAndClean", func(t *testing.T) { + n, err := d.CountAndClean() + assert.NoError(t, err) + assert.Equal(t, 1, n) + }) + + t.Run("CreateStale", func(t *testing.T) { + content := "foo" + err := d.create(2, []byte(content)) + assert.NoError(t, err) + }) + + t.Run("CountAndCleanWithStale", func(t *testing.T) { + n, err := d.CountAndClean() + assert.NoError(t, err) + assert.Equal(t, 1, n) + }) + + t.Run("Remove", func(t *testing.T) { + err := d.Remove() + assert.NoError(t, err) + }) +} From 2c52eb20ad251d87254def235ae16b910a9b66bd Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 5 Sep 2024 08:39:52 +0200 Subject: [PATCH 07/16] Return "TODO" on embed.FS String() calls from keyword doc Instead of panic, because panic does not hint about the driver that is missing the text file. --- core/keywords/keywords.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/keywords/keywords.go b/core/keywords/keywords.go index 989a5e93a..466699413 100644 --- a/core/keywords/keywords.go +++ b/core/keywords/keywords.go @@ -141,7 +141,7 @@ func (t Indices) Swap(i, j int) { func (t Text) String() string { if b, err := t.fs.ReadFile(t.path); err != nil { - panic("missing documentation text file: " + t.path) + return "TODO" } else { return string(b) } From ccad1ba9bd238b93bab39424da37f24a7b87137e Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 5 Sep 2024 08:43:48 +0200 Subject: [PATCH 08/16] Support task.max_parallel= Which allows running a task more than once concurrently. The scheduler still serializes strictly the runs, but the user can run while the scheduler holds a run too, if max_parallel>1. The default value is 1, which is the backward compatible behaviour. This patch also move common code from restask{host,docker} to a new restask package. --- drivers/resapp/base_keywords.go | 122 ++++--- drivers/resapp/unix_keywords.go | 324 ++++++++++-------- drivers/restask/keywords.go | 95 +++++ drivers/restask/main.go | 206 +++++++++++ .../{restaskdocker => restask}/text/kw/check | 0 .../text/kw/confirmation | 0 .../{restaskdocker => restask}/text/kw/log | 0 drivers/restask/text/kw/max_parallel | 13 + .../text/kw/on_error | 0 .../text/kw/retcodes | 0 .../text/kw/run_timeout | 0 .../{restaskhost => restask}/text/kw/schedule | 0 .../{restaskdocker => restask}/text/kw/snooze | 0 .../text/kw/timeout | 0 drivers/restaskdocker/keywords.go | 67 ---- drivers/restaskdocker/main.go | 165 +-------- drivers/restaskdocker/manifest.go | 5 + drivers/restaskhost/keywords.go | 62 ---- drivers/restaskhost/main.go | 247 ++++--------- drivers/restaskhost/manifest.go | 26 +- drivers/restaskhost/text/kw/check | 4 - drivers/restaskhost/text/kw/confirmation | 3 - drivers/restaskhost/text/kw/log | 1 - drivers/restaskhost/text/kw/on_error | 1 - drivers/restaskhost/text/kw/run_timeout | 3 - drivers/restaskhost/text/kw/snooze | 3 - drivers/restaskhost/text/kw/timeout | 3 - 27 files changed, 652 insertions(+), 698 deletions(-) create mode 100644 drivers/restask/keywords.go create mode 100644 drivers/restask/main.go rename drivers/{restaskdocker => restask}/text/kw/check (100%) rename drivers/{restaskdocker => restask}/text/kw/confirmation (100%) rename drivers/{restaskdocker => restask}/text/kw/log (100%) create mode 100644 drivers/restask/text/kw/max_parallel rename drivers/{restaskdocker => restask}/text/kw/on_error (100%) rename drivers/{restaskdocker => restask}/text/kw/retcodes (100%) rename drivers/{restaskdocker => restask}/text/kw/run_timeout (100%) rename drivers/{restaskhost => restask}/text/kw/schedule (100%) rename drivers/{restaskdocker => restask}/text/kw/snooze (100%) rename drivers/{restaskdocker => restask}/text/kw/timeout (100%) delete mode 100644 drivers/restaskhost/text/kw/check delete mode 100644 drivers/restaskhost/text/kw/confirmation delete mode 100644 drivers/restaskhost/text/kw/log delete mode 100644 drivers/restaskhost/text/kw/on_error delete mode 100644 drivers/restaskhost/text/kw/run_timeout delete mode 100644 drivers/restaskhost/text/kw/snooze delete mode 100644 drivers/restaskhost/text/kw/timeout diff --git a/drivers/resapp/base_keywords.go b/drivers/resapp/base_keywords.go index 5b8244851..a8dbd5baa 100644 --- a/drivers/resapp/base_keywords.go +++ b/drivers/resapp/base_keywords.go @@ -6,63 +6,71 @@ import ( ) var ( + BaseKeywordTimeout = keywords.Keyword{ + Attr: "Timeout", + Converter: converters.Duration, + Example: "180", + Option: "timeout", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/timeout"), + } + BaseKeywordStopTimeout = keywords.Keyword{ + Attr: "StopTimeout", + Converter: converters.Duration, + Example: "180", + Option: "stop_timeout", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/stop_timeout"), + } + BaseKeywordSecretsEnv = keywords.Keyword{ + Attr: "SecretsEnv", + Converter: converters.Shlex, + Example: "CRT=cert1/server.pem sec1/*", + Option: "secrets_environment", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/secrets_environment"), + } + BaseKeywordConfigsEnv = keywords.Keyword{ + Attr: "ConfigsEnv", + Converter: converters.Shlex, + Example: "PORT=http/port webapp/app1* {name}/* {name}-debug/settings", + Option: "configs_environment", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/configs_environment"), + } + BaseKeywordEnv = keywords.Keyword{ + Attr: "Env", + Example: "CRT=cert1/server.crt PEM=cert1/server.pem", + Option: "environment", + Converter: converters.Shlex, + Scopable: true, + Text: keywords.NewText(fs, "text/kw/environment"), + } + BaseKeywordRetCodes = keywords.Keyword{ + Attr: "RetCodes", + Default: "0:up 1:down", + Example: "0:up 1:down 3:warn 4: n/a 5:undef", + Option: "retcodes", + Required: false, + Scopable: true, + Text: keywords.NewText(fs, "text/kw/retcodes"), + } + BaseKeywordUmask = keywords.Keyword{ + Attr: "Umask", + Converter: converters.Umask, + Example: "022", + Option: "umask", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/umask"), + } + BaseKeywords = []keywords.Keyword{ - { - Attr: "Timeout", - Converter: converters.Duration, - Example: "180", - Option: "timeout", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/timeout"), - }, - { - Attr: "StopTimeout", - Converter: converters.Duration, - Example: "180", - Option: "stop_timeout", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/stop_timeout"), - }, - { - Attr: "SecretsEnv", - Converter: converters.Shlex, - Example: "CRT=cert1/server.pem sec1/*", - Option: "secrets_environment", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/secrets_environment"), - }, - { - Attr: "ConfigsEnv", - Converter: converters.Shlex, - Example: "PORT=http/port webapp/app1* {name}/* {name}-debug/settings", - Option: "configs_environment", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/configs_environment"), - }, - { - Attr: "Env", - Example: "CRT=cert1/server.crt PEM=cert1/server.pem", - Option: "environment", - Converter: converters.Shlex, - Scopable: true, - Text: keywords.NewText(fs, "text/kw/environment"), - }, - { - Attr: "RetCodes", - Default: "0:up 1:down", - Example: "0:up 1:down 3:warn 4: n/a 5:undef", - Option: "retcodes", - Required: false, - Scopable: true, - Text: keywords.NewText(fs, "text/kw/retcodes"), - }, - { - Attr: "Umask", - Converter: converters.Umask, - Example: "022", - Option: "umask", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/umask"), - }, + BaseKeywordTimeout, + BaseKeywordStopTimeout, + BaseKeywordSecretsEnv, + BaseKeywordConfigsEnv, + BaseKeywordEnv, + BaseKeywordRetCodes, + BaseKeywordUmask, } ) diff --git a/drivers/resapp/unix_keywords.go b/drivers/resapp/unix_keywords.go index 4bcfdb785..f5fec4ef5 100644 --- a/drivers/resapp/unix_keywords.go +++ b/drivers/resapp/unix_keywords.go @@ -13,157 +13,179 @@ var ( //go:embed text fs embed.FS + UnixKeywordScriptPath = keywords.Keyword{ + Attr: "ScriptPath", + Option: "script", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/script"), + } + UnixKeywordStartCmd = keywords.Keyword{ + Attr: "StartCmd", + Option: "start", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/start"), + } + UnixKeywordStopCmd = keywords.Keyword{ + Attr: "StopCmd", + Option: "stop", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/stop"), + } + UnixKeywordCheckCmd = keywords.Keyword{ + Attr: "CheckCmd", + Option: "check", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/check"), + } + UnixKeywordInfoCmd = keywords.Keyword{ + Attr: "InfoCmd", + Default: "false", + Option: "info", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/info"), + } + UnixKeywordStatusLogKw = keywords.Keyword{ + Attr: "StatusLogKw", + Converter: converters.Bool, + Default: "false", + Option: "status_log", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/status_log"), + } + UnixKeywordCheckTimeout = keywords.Keyword{ + Attr: "CheckTimeout", + Converter: converters.Duration, + Example: "180", + Option: "check_timeout", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/check_timeout"), + } + UnixKeywordInfoTimeout = keywords.Keyword{ + Attr: "InfoTimeout", + Converter: converters.Duration, + Example: "180", + Option: "info_timeout", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/info_timeout"), + } + UnixKeywordCwd = keywords.Keyword{ + Attr: "Cwd", + Option: "cwd", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/cwd"), + } + UnixKeywordUser = keywords.Keyword{ + Attr: "User", + Option: "user", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/user"), + } + UnixKeywordGroup = keywords.Keyword{ + Attr: "Group", + Option: "group", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/group"), + } + UnixKeywordLimitCPU = keywords.Keyword{ + Attr: "Limit.CPU", + Converter: converters.Duration, + Example: "30s", + Option: "limit_cpu", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_cpu"), + } + UnixKeywordLimitCore = keywords.Keyword{ + Attr: "Limit.Core", + Converter: converters.Size, + Option: "limit_core", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_core"), + } + UnixKeywordLimitData = keywords.Keyword{ + Attr: "Limit.Data", + Converter: converters.Size, + Option: "limit_data", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_data"), + } + UnixKeywordLimitFSize = keywords.Keyword{ + Attr: "Limit.FSize", + Converter: converters.Size, + Option: "limit_fsize", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_fsize"), + } + UnixKeywordLimitMemLock = keywords.Keyword{ + Attr: "Limit.MemLock", + Converter: converters.Size, + Option: "limit_memlock", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_memlock"), + } + UnixKeywordLimitNoFile = keywords.Keyword{ + Attr: "Limit.NoFile", + Converter: converters.Size, + Option: "limit_nofile", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_nofile"), + } + UnixKeywordLimitNProc = keywords.Keyword{ + Attr: "Limit.NProc", + Converter: converters.Size, + Option: "limit_nproc", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_nproc"), + } + UnixKeywordLimitRSS = keywords.Keyword{ + Attr: "Limit.RSS", + Converter: converters.Size, + Option: "limit_rss", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_rss"), + } + UnixKeywordLimitStack = keywords.Keyword{ + Attr: "Limit.Stack", + Converter: converters.Size, + Option: "limit_stack", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_stack"), + } + UnixKeywordLimitVmem = keywords.Keyword{ + Attr: "Limit.VMem", + Converter: converters.Size, + Option: "limit_vmem", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_vmem"), + } + UnixKeywordLimitAS = keywords.Keyword{ + Attr: "Limit.AS", + Converter: converters.Size, + Option: "limit_as", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/limit_as"), + } UnixKeywords = []keywords.Keyword{ - { - Attr: "ScriptPath", - Option: "script", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/script"), - }, - { - Attr: "StartCmd", - Option: "start", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/start"), - }, - { - Attr: "StopCmd", - Option: "stop", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/stop"), - }, - { - Attr: "CheckCmd", - Option: "check", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/check"), - }, - { - Attr: "InfoCmd", - Default: "false", - Option: "info", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/info"), - }, - { - Attr: "StatusLogKw", - Converter: converters.Bool, - Default: "false", - Option: "status_log", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/status_log"), - }, - { - Attr: "CheckTimeout", - Converter: converters.Duration, - Example: "180", - Option: "check_timeout", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/check_timeout"), - }, - { - Attr: "InfoTimeout", - Converter: converters.Duration, - Example: "180", - Option: "info_timeout", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/info_timeout"), - }, - { - Attr: "Cwd", - Option: "cwd", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/cwd"), - }, - { - Attr: "User", - Option: "user", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/user"), - }, - { - Attr: "Group", - Option: "group", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/group"), - }, - { - Attr: "Limit.CPU", - Converter: converters.Duration, - Example: "30s", - Option: "limit_cpu", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_cpu"), - }, - { - Attr: "Limit.Core", - Converter: converters.Size, - Option: "limit_core", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_core"), - }, - { - Attr: "Limit.Data", - Converter: converters.Size, - Option: "limit_data", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_data"), - }, - { - Attr: "Limit.FSize", - Converter: converters.Size, - Option: "limit_fsize", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_fsize"), - }, - { - Attr: "Limit.MemLock", - Converter: converters.Size, - Option: "limit_memlock", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_memlock"), - }, - { - Attr: "Limit.NoFile", - Converter: converters.Size, - Option: "limit_nofile", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_nofile"), - }, - { - Attr: "Limit.NProc", - Converter: converters.Size, - Option: "limit_nproc", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_nproc"), - }, - { - Attr: "Limit.RSS", - Converter: converters.Size, - Option: "limit_rss", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_rss"), - }, - { - Attr: "Limit.Stack", - Converter: converters.Size, - Option: "limit_stack", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_stack"), - }, - { - Attr: "Limit.VMem", - Converter: converters.Size, - Option: "limit_vmem", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_vmem"), - }, - { - Attr: "Limit.AS", - Converter: converters.Size, - Option: "limit_as", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/limit_as"), - }, + UnixKeywordScriptPath, + UnixKeywordStartCmd, + UnixKeywordStopCmd, + UnixKeywordCheckCmd, + UnixKeywordInfoCmd, + UnixKeywordStatusLogKw, + UnixKeywordCheckTimeout, + UnixKeywordInfoTimeout, + UnixKeywordCwd, + UnixKeywordUser, + UnixKeywordGroup, + UnixKeywordLimitCPU, + UnixKeywordLimitCore, + UnixKeywordLimitData, + UnixKeywordLimitFSize, + UnixKeywordLimitMemLock, + UnixKeywordLimitNoFile, + UnixKeywordLimitNProc, + UnixKeywordLimitRSS, + UnixKeywordLimitStack, + UnixKeywordLimitVmem, + UnixKeywordLimitAS, } ) diff --git a/drivers/restask/keywords.go b/drivers/restask/keywords.go new file mode 100644 index 000000000..e6818ff60 --- /dev/null +++ b/drivers/restask/keywords.go @@ -0,0 +1,95 @@ +package restask + +import ( + "embed" + + "github.com/opensvc/om3/core/keywords" + "github.com/opensvc/om3/util/converters" +) + +var ( + //go:embed text + fs embed.FS + + Keywords = []keywords.Keyword{ + { + Attr: "Check", + Candidates: []string{"last_run", ""}, + Example: "last_run", + Option: "check", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/check"), + }, + { + Attr: "Confirmation", + Converter: converters.Bool, + Option: "confirmation", + Text: keywords.NewText(fs, "text/kw/confirmation"), + }, + { + Attr: "LogOutputs", + Converter: converters.Bool, + Default: "true", + Option: "log", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/log"), + }, + { + Attr: "MaxParallel", + Converter: converters.Int, + Default: "1", + Example: "2", + Option: "max_parallel", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/max_parallel"), + }, + { + Attr: "OnErrorCmd", + Example: "/srv/{name}/data/scripts/task_on_error.sh", + Option: "on_error", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/on_error"), + }, + { + Attr: "RetCodes", + Default: "0:up 1:down", + Example: "0:up 1:down 3:warn 4: n/a 5:undef", + Option: "retcodes", + Required: false, + Scopable: true, + Text: keywords.NewText(fs, "text/kw/retcodes"), + }, + { + Attr: "RunTimeout", + Converter: converters.Duration, + Example: "1m30s", + Option: "run_timeout", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/run_timeout"), + }, + { + Attr: "Schedule", + DefaultOption: "run_schedule", + Example: "00:00-01:00 mon", + Option: "schedule", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/schedule"), + }, + { + Attr: "Snooze", + Converter: converters.Duration, + Example: "10m", + Option: "snooze", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/snooze"), + }, + { + Attr: "Timeout", + Converter: converters.Duration, + Example: "5m", + Option: "timeout", + Scopable: true, + Text: keywords.NewText(fs, "text/kw/timeout"), + }, + } +) diff --git a/drivers/restask/main.go b/drivers/restask/main.go new file mode 100644 index 000000000..35deb3182 --- /dev/null +++ b/drivers/restask/main.go @@ -0,0 +1,206 @@ +package restask + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/mattn/go-isatty" + "github.com/opensvc/om3/core/actioncontext" + "github.com/opensvc/om3/core/env" + "github.com/opensvc/om3/core/resource" + "github.com/opensvc/om3/core/status" + "github.com/opensvc/om3/util/confirmation" + "github.com/opensvc/om3/util/retcodes" + "github.com/opensvc/om3/util/runfiles" + "github.com/opensvc/om3/util/xsession" +) + +const ( + lockName = "run.lock" +) + +type BaseTask struct { + resource.T + Check string + Confirmation bool + LogOutputs bool + MaxParallel int + OnErrorCmd string + RetCodes string + RunTimeout *time.Duration + Schedule string + Snooze *time.Duration +} + +func (t BaseTask) ScheduleOptions() resource.ScheduleOptions { + return resource.ScheduleOptions{ + Action: "run", + Option: "schedule", + Base: "", + RequireConfirmation: t.Confirmation, + RequireProvisioned: true, + RequireCollector: false, + } +} + +// notifyRunDone is a noop here as for now the daemon api has no support for +// POST /run_done, and may not need one. +func (t BaseTask) notifyRunDone() error { + return nil +} + +func (t BaseTask) handleConfirmation(ctx context.Context) error { + if !t.Confirmation { + return nil + } + if actioncontext.IsConfirm(ctx) { + t.Log().Infof("run confirmed by --confirm command line option") + return nil + } + if actioncontext.IsCron(ctx) { + // as set by the daemon scheduler subsystem + return fmt.Errorf("run aborted (--cron)") + } + if !isatty.IsTerminal(os.Stdin.Fd()) { + return fmt.Errorf("run aborted (stdin is not a tty)") + } + description := fmt.Sprintf(`The resource %s requires a run confirmation. +Please make sure you fully understand its role and effects before confirming the run. +Enter "yes" if you really want to run.`, t.RID()) + s, err := confirmation.ReadLn(description, time.Second*30) + if err != nil { + return fmt.Errorf("read confirmation: %w", err) + } + if s == "yes" { + t.Log().Infof("run confirmed interactively") + return nil + } + return fmt.Errorf("run aborted") +} + +func (t BaseTask) lastRunFile() string { + return filepath.Join(t.VarDir(), "last_run_retcode") +} + +func (t *BaseTask) statusLastRun(ctx context.Context) status.T { + if err := resource.StatusCheckRequires(ctx, t); err != nil { + t.StatusLog().Info("requirements not met") + return status.NotApplicable + } + if i, err := t.readLastRun(); err != nil { + t.StatusLog().Info("never run") + return status.NotApplicable + } else { + s, err := t.ExitCodeToStatus(i) + if err != nil { + t.StatusLog().Info("%s", err) + } + if s != status.Up { + t.StatusLog().Info("last run failed (%d)", i) + } + return s + } +} + +func (t BaseTask) readLastRun() (int, error) { + p := t.lastRunFile() + if b, err := os.ReadFile(p); err != nil { + return 0, err + } else { + return strconv.Atoi(strings.TrimSpace(string(b))) + } +} + +func (t BaseTask) WriteLastRun(retcode int) error { + p := t.lastRunFile() + f, err := os.Create(p) + if err != nil { + return err + } + defer f.Close() + fmt.Fprintf(f, "%d\n", retcode) + return nil +} + +func (t BaseTask) IsRunning() bool { + hasRunning, err := t.RunDir().HasRunning() + if err != nil { + return false + } + return hasRunning +} + +func (t BaseTask) RunDir() runfiles.Dir { + return runfiles.Dir{ + Path: filepath.Join(t.VarDir(), "run"), + Log: t.Log(), + } +} + +func (t BaseTask) RunIf(ctx context.Context, fn func(context.Context) error) error { + runDir := t.RunDir() + canRun := func() error { + disable := actioncontext.IsLockDisabled(ctx) + timeout := actioncontext.LockTimeout(ctx) + unlock, err := t.Lock(disable, timeout, lockName) + if err != nil { + return err + } + n, err := runDir.CountAndClean() + if err != nil { + return err + } + defer unlock() + if n >= t.MaxParallel { + return fmt.Errorf("task is already running %d times", n) + } + if err := runDir.Create(xsession.ID[:]); err != nil { + return err + } + return nil + } + if err := canRun(); err != nil { + return err + } + defer runDir.Remove() + + if !env.HasDaemonOrigin() { + defer t.notifyRunDone() + } + if err := t.handleConfirmation(ctx); err != nil { + return err + } + if err := t.ApplyPGChain(ctx); err != nil { + return err + } + + return fn(ctx) +} + +func (t BaseTask) ExitCodeToStatus(exitCode int) (status.T, error) { + m, err := retcodes.Parse(t.RetCodes) + if err != nil { + return status.Warn, err + } + return m.Status(exitCode), nil +} + +func (t *BaseTask) Status(ctx context.Context) status.T { + n, err := t.RunDir().Count() + if err != nil { + t.StatusLog().Error("%s", err) + } else if n >= t.MaxParallel { + t.StatusLog().Info("%d/%d max parallel runs reached", n, t.MaxParallel) + } + switch t.Check { + case "last_run": + return t.statusLastRun(ctx) + default: + return status.NotApplicable + } +} diff --git a/drivers/restaskdocker/text/kw/check b/drivers/restask/text/kw/check similarity index 100% rename from drivers/restaskdocker/text/kw/check rename to drivers/restask/text/kw/check diff --git a/drivers/restaskdocker/text/kw/confirmation b/drivers/restask/text/kw/confirmation similarity index 100% rename from drivers/restaskdocker/text/kw/confirmation rename to drivers/restask/text/kw/confirmation diff --git a/drivers/restaskdocker/text/kw/log b/drivers/restask/text/kw/log similarity index 100% rename from drivers/restaskdocker/text/kw/log rename to drivers/restask/text/kw/log diff --git a/drivers/restask/text/kw/max_parallel b/drivers/restask/text/kw/max_parallel new file mode 100644 index 000000000..6cf53fe82 --- /dev/null +++ b/drivers/restask/text/kw/max_parallel @@ -0,0 +1,13 @@ +Support limited, concurrent runs of tasks. + +The task#xx.max_parallel=2 setting limits the number of concurrent task runs to 2. + +The default value is 1, ensuring backward compatibility. + +The run count is determined based on PID files created in the /run/ directories. + +The PID file is normally removed when the task execution ends, but if the executor dies abruptly (e.g., due to a SIGKILL), the stale PID file is not considered when computing the resource status. It is removed before the count check of the next run. + +Staleness is evaluated using the condition: (PID file mtime < process birth time). + +A new status log message may appear to indicate that the maximum concurrency limit has been reached. diff --git a/drivers/restaskdocker/text/kw/on_error b/drivers/restask/text/kw/on_error similarity index 100% rename from drivers/restaskdocker/text/kw/on_error rename to drivers/restask/text/kw/on_error diff --git a/drivers/restaskdocker/text/kw/retcodes b/drivers/restask/text/kw/retcodes similarity index 100% rename from drivers/restaskdocker/text/kw/retcodes rename to drivers/restask/text/kw/retcodes diff --git a/drivers/restaskdocker/text/kw/run_timeout b/drivers/restask/text/kw/run_timeout similarity index 100% rename from drivers/restaskdocker/text/kw/run_timeout rename to drivers/restask/text/kw/run_timeout diff --git a/drivers/restaskhost/text/kw/schedule b/drivers/restask/text/kw/schedule similarity index 100% rename from drivers/restaskhost/text/kw/schedule rename to drivers/restask/text/kw/schedule diff --git a/drivers/restaskdocker/text/kw/snooze b/drivers/restask/text/kw/snooze similarity index 100% rename from drivers/restaskdocker/text/kw/snooze rename to drivers/restask/text/kw/snooze diff --git a/drivers/restaskdocker/text/kw/timeout b/drivers/restask/text/kw/timeout similarity index 100% rename from drivers/restaskdocker/text/kw/timeout rename to drivers/restask/text/kw/timeout diff --git a/drivers/restaskdocker/keywords.go b/drivers/restaskdocker/keywords.go index b06fed048..dfb78abd3 100644 --- a/drivers/restaskdocker/keywords.go +++ b/drivers/restaskdocker/keywords.go @@ -4,7 +4,6 @@ import ( "embed" "github.com/opensvc/om3/core/keywords" - "github.com/opensvc/om3/drivers/rescontainer" "github.com/opensvc/om3/util/converters" ) @@ -13,52 +12,6 @@ var ( fs embed.FS Keywords = []keywords.Keyword{ - { - Attr: "Schedule", - DefaultOption: "run_schedule", - Example: "00:00-01:00 mon", - Option: "schedule", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/schedule"), - }, - { - Attr: "Timeout", - Converter: converters.Duration, - Example: "5m", - Option: "timeout", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/timeout"), - }, - { - Attr: "Snooze", - Converter: converters.Duration, - Example: "10m", - Option: "snooze", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/snooze"), - }, - { - Attr: "LogOutputs", - Converter: converters.Bool, - Default: "true", - Option: "log", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/log"), - }, - { - Attr: "Check", - Candidates: []string{"last_run", ""}, - Example: "last_run", - Option: "check", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/check"), - }, - { - Attr: "Confirmation", - Converter: converters.Bool, - Option: "confirmation", - Text: keywords.NewText(fs, "text/kw/confirmation"), - }, { Attr: "Name", DefaultText: keywords.NewText(fs, "text/kw/name.default"), @@ -286,25 +239,5 @@ var ( Scopable: true, Text: keywords.NewText(fs, "text/kw/configs_environment"), }, - { - Attr: "RetCodes", - Default: "0:up 1:down", - Example: "0:up 1:down 3:warn 4: n/a 5:undef", - Option: "retcodes", - Required: false, - Scopable: true, - Text: keywords.NewText(fs, "text/kw/retcodes"), - }, - { - Attr: "RunTimeout", - Converter: converters.Duration, - Example: "1m30s", - Option: "run_timeout", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/run_timeout"), - }, - - rescontainer.KWOsvcRootPath, - rescontainer.KWGuestOS, } ) diff --git a/drivers/restaskdocker/main.go b/drivers/restaskdocker/main.go index 28cad4f71..08b71d39c 100644 --- a/drivers/restaskdocker/main.go +++ b/drivers/restaskdocker/main.go @@ -7,30 +7,22 @@ package restaskdocker import ( "context" "fmt" - "os" - "path/filepath" - "strconv" - "strings" "syscall" "time" "github.com/google/uuid" - "github.com/mattn/go-isatty" - "github.com/opensvc/om3/core/actioncontext" - "github.com/opensvc/om3/core/env" "github.com/opensvc/om3/core/naming" "github.com/opensvc/om3/core/resource" "github.com/opensvc/om3/core/status" "github.com/opensvc/om3/drivers/rescontainerdocker" - "github.com/opensvc/om3/util/confirmation" + "github.com/opensvc/om3/drivers/restask" "github.com/opensvc/om3/util/pg" - "github.com/opensvc/om3/util/retcodes" ) // T is the driver structure. type T struct { - resource.T + restask.BaseTask resource.SCSIPersistentReservation Detach bool `json:"detach"` PG pg.Config `json:"pg"` @@ -69,30 +61,17 @@ type T struct { IPCNS string `json:"ipcns"` UTSNS string `json:"utsns"` RegistryCreds string `json:"registry_creds"` - RunTimeout *time.Duration `json:"run_timeout"` PullTimeout *time.Duration `json:"pull_timeout"` Timeout *time.Duration `json:"timeout"` - - RetCodes string `json:"retcodes"` - - Check string - Schedule string - Confirmation bool - LogOutputs bool - Snooze *time.Duration } -const ( - lockName = "run" -) - func New() resource.Driver { return &T{} } func (t T) Container() *rescontainerdocker.T { return &rescontainerdocker.T{ - T: t.T, + T: t.BaseTask.T, Detach: false, SCSIPersistentReservation: t.SCSIPersistentReservation, PG: t.PG, @@ -135,44 +114,11 @@ func (t T) Container() *rescontainerdocker.T { } } -func (t T) IsRunning() bool { - unlock, err := t.Lock(false, time.Second*0, lockName) - if err != nil { - return true - } - defer unlock() - return false -} - func (t T) Run(ctx context.Context) error { - disable := actioncontext.IsLockDisabled(ctx) - timeout := actioncontext.LockTimeout(ctx) - unlock, err := t.Lock(disable, timeout, lockName) - if err != nil { - return err - } - defer unlock() - return t.lockedRun(ctx) -} - -func (t T) ExitCodeToStatus(exitCode int) (status.T, error) { - m, err := retcodes.Parse(t.RetCodes) - if err != nil { - return status.Warn, err - } - return m.Status(exitCode), nil + return t.RunIf(ctx, t.lockedRun) } func (t T) lockedRun(ctx context.Context) (err error) { - if !env.HasDaemonOrigin() { - defer t.notifyRunDone() - } - if err := t.handleConfirmation(ctx); err != nil { - return err - } - if err := t.ApplyPGChain(ctx); err != nil { - return err - } // TODO: if t.LogOutputs {} container := t.Container() if err := container.Start(ctx); err != nil { @@ -183,11 +129,11 @@ func (t T) lockedRun(ctx context.Context) (err error) { if err != nil { return err } - if err := t.writeLastRun(inspect.State.ExitCode); err != nil { + if err := t.WriteLastRun(inspect.State.ExitCode); err != nil { t.Log().Errorf("write last run: %s", err) return err } - if s, err := t.ExitCodeToStatus(inspect.State.ExitCode); err != nil { + if s, err := t.BaseTask.ExitCodeToStatus(inspect.State.ExitCode); err != nil { return err } else if s != status.Up { return fmt.Errorf("command exited with code %d", inspect.State.ExitCode) @@ -199,59 +145,6 @@ func (t *T) Kill(ctx context.Context) error { return t.Container().Signal(syscall.SIGKILL) } -func (t *T) Status(ctx context.Context) status.T { - switch t.Check { - case "last_run": - return t.statusLastRun(ctx) - default: - return status.NotApplicable - } -} - -func (t T) writeLastRun(retcode int) error { - p := t.lastRunFile() - f, err := os.Create(p) - if err != nil { - return err - } - defer f.Close() - fmt.Fprintf(f, "%d\n", retcode) - return nil -} - -func (t T) readLastRun() (int, error) { - p := t.lastRunFile() - if b, err := os.ReadFile(p); err != nil { - return 0, err - } else { - return strconv.Atoi(strings.TrimSpace(string(b))) - } -} - -func (t T) lastRunFile() string { - return filepath.Join(t.VarDir(), "last_run_retcode") -} - -func (t *T) statusLastRun(ctx context.Context) status.T { - if err := resource.StatusCheckRequires(ctx, t); err != nil { - t.StatusLog().Info("requirements not met") - return status.NotApplicable - } - if i, err := t.readLastRun(); err != nil { - t.StatusLog().Info("never run") - return status.NotApplicable - } else { - s, err := t.ExitCodeToStatus(i) - if err != nil { - t.StatusLog().Info("%s", err) - } - if s != status.Up { - t.StatusLog().Info("last run failed (%d)", i) - } - return s - } -} - func (t *T) running(ctx context.Context) bool { inspect, err := t.Container().Inspect(ctx) if err != nil { @@ -264,49 +157,3 @@ func (t *T) running(ctx context.Context) bool { func (t T) Label() string { return "" } - -func (t T) handleConfirmation(ctx context.Context) error { - if !t.Confirmation { - return nil - } - if actioncontext.IsConfirm(ctx) { - t.Log().Infof("run confirmed by --confirm command line option") - return nil - } - if actioncontext.IsCron(ctx) { - // as set by the daemon scheduler subsystem - return fmt.Errorf("run aborted (--cron)") - } - if !isatty.IsTerminal(os.Stdin.Fd()) { - return fmt.Errorf("run aborted (stdin is not a tty)") - } - description := fmt.Sprintf(`The resource %s requires a run confirmation. -Please make sure you fully understand its role and effects before confirming the run. -Enter "yes" if you really want to run.`, t.RID()) - s, err := confirmation.ReadLn(description, time.Second*30) - if err != nil { - return fmt.Errorf("read confirmation: %w", err) - } - if s == "yes" { - t.Log().Infof("run confirmed interactively") - return nil - } - return fmt.Errorf("run aborted") -} - -// notifyRunDone is a noop here as for now the daemon api has no support for -// POST /run_done, and may not need one. -func (t T) notifyRunDone() error { - return nil -} - -func (t T) ScheduleOptions() resource.ScheduleOptions { - return resource.ScheduleOptions{ - Action: "run", - Option: "schedule", - Base: "", - RequireConfirmation: t.Confirmation, - RequireProvisioned: true, - RequireCollector: false, - } -} diff --git a/drivers/restaskdocker/manifest.go b/drivers/restaskdocker/manifest.go index faf3d9c39..8bce50281 100644 --- a/drivers/restaskdocker/manifest.go +++ b/drivers/restaskdocker/manifest.go @@ -4,6 +4,8 @@ import ( "github.com/opensvc/om3/core/driver" "github.com/opensvc/om3/core/manifest" "github.com/opensvc/om3/core/naming" + "github.com/opensvc/om3/drivers/rescontainer" + "github.com/opensvc/om3/drivers/restask" ) var ( @@ -26,7 +28,10 @@ func (t T) Manifest() *manifest.T { manifest.ContextObjectID, manifest.ContextObjectID, manifest.ContextDNS, + rescontainer.KWOsvcRootPath, + rescontainer.KWGuestOS, ) + m.AddKeywords(restask.Keywords...) m.AddKeywords(Keywords...) return m } diff --git a/drivers/restaskhost/keywords.go b/drivers/restaskhost/keywords.go index a8d1bc13f..28ff22500 100644 --- a/drivers/restaskhost/keywords.go +++ b/drivers/restaskhost/keywords.go @@ -4,7 +4,6 @@ import ( "embed" "github.com/opensvc/om3/core/keywords" - "github.com/opensvc/om3/util/converters" ) var ( @@ -12,72 +11,11 @@ var ( fs embed.FS Keywords = []keywords.Keyword{ - { - Option: "schedule", - DefaultOption: "run_schedule", - Attr: "Schedule", - Scopable: true, - Text: keywords.NewText(fs, "text/kw/schedule"), - Example: "00:00-01:00 mon", - }, - { - Option: "timeout", - Attr: "Timeout", - Converter: converters.Duration, - Scopable: true, - Text: keywords.NewText(fs, "text/kw/timeout"), - Example: "5m", - }, - { - Option: "snooze", - Attr: "Snooze", - Converter: converters.Duration, - Scopable: true, - Example: "10m", - Text: keywords.NewText(fs, "text/kw/snooze"), - }, - { - Option: "log", - Attr: "LogOutputs", - Default: "true", - Converter: converters.Bool, - Scopable: true, - Text: keywords.NewText(fs, "text/kw/log"), - }, { Option: "command", Attr: "RunCmd", Scopable: true, Text: keywords.NewText(fs, "text/kw/command"), }, - { - Option: "on_error", - Attr: "OnErrorCmd", - Scopable: true, - Example: "/srv/{name}/data/scripts/task_on_error.sh", - Text: keywords.NewText(fs, "text/kw/on_error"), - }, - { - Option: "check", - Attr: "Check", - Candidates: []string{"last_run", ""}, - Scopable: true, - Example: "last_run", - Text: keywords.NewText(fs, "text/kw/check"), - }, - { - Option: "confirmation", - Attr: "Confirmation", - Converter: converters.Bool, - Text: keywords.NewText(fs, "text/kw/confirmation"), - }, - { - Option: "run_timeout", - Attr: "RunTimeout", - Converter: converters.Duration, - Scopable: true, - Example: "1m30s", - Text: keywords.NewText(fs, "text/kw/run_timeout"), - }, } ) diff --git a/drivers/restaskhost/main.go b/drivers/restaskhost/main.go index a774d7316..860d11169 100644 --- a/drivers/restaskhost/main.go +++ b/drivers/restaskhost/main.go @@ -8,66 +8,85 @@ import ( "context" "fmt" "os" - "path/filepath" - "strconv" - "strings" "syscall" "time" - "github.com/mattn/go-isatty" + "github.com/google/uuid" "github.com/rs/zerolog" - "github.com/opensvc/om3/core/actioncontext" - "github.com/opensvc/om3/core/env" + "github.com/opensvc/om3/core/naming" "github.com/opensvc/om3/core/resource" "github.com/opensvc/om3/core/status" "github.com/opensvc/om3/drivers/resapp" + "github.com/opensvc/om3/drivers/restask" "github.com/opensvc/om3/util/command" - "github.com/opensvc/om3/util/confirmation" "github.com/opensvc/om3/util/funcopt" + "github.com/opensvc/om3/util/pg" "github.com/opensvc/om3/util/plog" "github.com/opensvc/om3/util/proc" + "github.com/opensvc/om3/util/ulimit" ) // T is the driver structure. type T struct { - resapp.T - Check string - Confirmation bool - LogOutputs bool - OnErrorCmd string - RunCmd string - RunTimeout *time.Duration - Schedule string - Snooze *time.Duration + restask.BaseTask + + // From resapp.BaseT + RetCodes string `json:"retcodes"` + SecretsEnv []string `json:"secret_environment"` + ConfigsEnv []string `json:"configs_environment"` + Env []string `json:"environment"` + Timeout *time.Duration `json:"timeout"` + //StartTimeout *time.Duration `json:"start_timeout"` + StopTimeout *time.Duration `json:"stop_timeout"` + Umask *os.FileMode `json:"umask"` + ObjectID uuid.UUID `json:"objectID"` + + // From resapp.T + Path naming.Path `json:"path"` + Nodes []string `json:"nodes"` + Cwd string `json:"cwd"` + User string `json:"user"` + Group string `json:"group"` + PG pg.Config `json:"pg"` + Limit ulimit.Config `json:"limit"` + StopCmd string + + RunCmd string } -const ( - lockName = "run" -) - func New() resource.Driver { return &T{} } -func (t T) IsRunning() bool { - unlock, err := t.Lock(false, time.Second*0, lockName) - if err != nil { - return true +func (t T) App() *resapp.T { + return &resapp.T{ + BaseT: resapp.BaseT{ + T: t.BaseTask.T, + RetCodes: t.RetCodes, + Path: t.Path, + Nodes: t.Nodes, + SecretsEnv: t.SecretsEnv, + ConfigsEnv: t.ConfigsEnv, + Env: t.Env, + Timeout: t.Timeout, + Umask: t.Umask, + ObjectID: t.ObjectID, + StopTimeout: t.StopTimeout, + }, + Path: t.Path, + Nodes: t.Nodes, + Cwd: t.Cwd, + User: t.User, + Group: t.Group, + PG: t.PG, + Limit: t.Limit, + StopCmd: t.StopCmd, } - defer unlock() - return false } func (t T) Run(ctx context.Context) error { - disable := actioncontext.IsLockDisabled(ctx) - timeout := actioncontext.LockTimeout(ctx) - unlock, err := t.Lock(disable, timeout, lockName) - if err != nil { - return err - } - defer unlock() - return t.lockedRun(ctx) + return t.RunIf(ctx, t.lockedRun) } func (t T) loggerWithProc(p proc.T) *plog.Logger { @@ -79,22 +98,14 @@ func (t T) loggerWithCmd(cmd *command.T) *plog.Logger { } func (t T) lockedRun(ctx context.Context) (err error) { - if !env.HasDaemonOrigin() { - defer t.notifyRunDone() - } var opts []funcopt.O - if err := t.handleConfirmation(ctx); err != nil { - return err - } - if opts, err = t.GetFuncOpts(t.RunCmd, "run"); err != nil { + app := t.App() + if opts, err = app.GetFuncOpts(t.RunCmd, "run"); err != nil { return err } if len(opts) == 0 { return nil } - if err := t.ApplyPGChain(ctx); err != nil { - return err - } if t.LogOutputs { opts = append(opts, command.WithLogger(t.Log()), @@ -103,13 +114,13 @@ func (t T) lockedRun(ctx context.Context) (err error) { ) } opts = append(opts, - command.WithTimeout(t.GetTimeout("run")), + command.WithTimeout(app.GetTimeout("run")), command.WithIgnoredExitCodes(), ) cmd := command.New(opts...) t.loggerWithCmd(cmd).Infof("run %s", cmd) err = cmd.Run() - if err := t.writeLastRun(cmd.ExitCode()); err != nil { + if err := t.WriteLastRun(cmd.ExitCode()); err != nil { return err } if err != nil { @@ -118,7 +129,7 @@ func (t T) lockedRun(ctx context.Context) (err error) { t.Log().Warnf("on error: %s", err) } } - if s, err := t.ExitCodeToStatus(cmd.ExitCode()); err != nil { + if s, err := t.BaseTask.ExitCodeToStatus(cmd.ExitCode()); err != nil { return err } else if s != status.Up { return fmt.Errorf("command exited with code %d", cmd.ExitCode()) @@ -127,7 +138,8 @@ func (t T) lockedRun(ctx context.Context) (err error) { } func (t T) onError() error { - opts, err := t.GetFuncOpts(t.OnErrorCmd, "on_error") + app := t.App() + opts, err := app.GetFuncOpts(t.OnErrorCmd, "on_error") if err != nil { return err } @@ -140,14 +152,16 @@ func (t T) onError() error { } func (t *T) Kill(ctx context.Context) error { - if t.StopCmd != "" { - return t.CommonStop(ctx, t) + app := t.App() + if app.StopCmd != "" { + return app.CommonStop(ctx, t) } return t.stop(ctx) } func (t *T) stop(ctx context.Context) error { - cmdArgs, err := t.BaseCmdArgs(t.StartCmd, "stop") + app := t.App() + cmdArgs, err := app.BaseCmdArgs(app.StartCmd, "stop") if err != nil { return err } @@ -183,96 +197,11 @@ func (t *T) stop(ctx context.Context) error { return fmt.Errorf("waited too long for process %s to disappear", procs) } -func (t *T) Status(ctx context.Context) status.T { - switch t.Check { - case "last_run": - return t.statusLastRun(ctx) - default: - return status.NotApplicable - } -} - -func (t T) writeLastRun(retcode int) error { - p := t.lastRunFile() - f, err := os.Create(p) - if err != nil { - return err - } - defer f.Close() - fmt.Fprintf(f, "%d\n", retcode) - return nil -} - -func (t T) readLastRun() (int, error) { - p := t.lastRunFile() - if b, err := os.ReadFile(p); err != nil { - return 0, err - } else { - return strconv.Atoi(strings.TrimSpace(string(b))) - } -} - -func (t T) lastRunFile() string { - return filepath.Join(t.VarDir(), "last_run_retcode") -} - -func (t *T) statusLastRun(ctx context.Context) status.T { - if err := resource.StatusCheckRequires(ctx, t); err != nil { - t.StatusLog().Info("requirements not met") - return status.NotApplicable - } - if i, err := t.readLastRun(); err != nil { - t.StatusLog().Info("never run") - return status.NotApplicable - } else { - s, err := t.ExitCodeToStatus(i) - if err != nil { - t.StatusLog().Info("%s", err) - } - if s != status.Up { - t.StatusLog().Info("last run failed (%d)", i) - } - return s - } -} - -func (t *T) running(ctx context.Context) bool { - var s status.T - if t.CheckCmd != "" { - s = t.CommonStatus(ctx) - } else { - s = t.status() - } - return s == status.Up -} - // Label returns a formatted short description of the Resource func (t T) Label() string { return "" } -func (t *T) status() status.T { - cmdArgs, err := t.BaseCmdArgs(t.StartCmd, "start") - if err != nil { - t.StatusLog().Error("%s", err) - return status.Undef - } - procs, err := t.getRunning(cmdArgs) - if err != nil { - t.StatusLog().Error("%s", err) - return status.Undef - } - switch procs.Len() { - case 0: - return status.Down - case 1: - return status.Up - default: - t.StatusLog().Warn("too many process (%d)", procs.Len()) - return status.Up - } -} - func (t T) getRunning(cmdArgs []string) (proc.L, error) { procs, err := proc.All() if err != nil { @@ -282,49 +211,3 @@ func (t T) getRunning(cmdArgs []string) (proc.L, error) { procs = procs.FilterByEnv("OPENSVC_RID", t.RID()) return procs, nil } - -func (t T) handleConfirmation(ctx context.Context) error { - if !t.Confirmation { - return nil - } - if actioncontext.IsConfirm(ctx) { - t.Log().Infof("run confirmed by --confirm command line option") - return nil - } - if actioncontext.IsCron(ctx) { - // as set by the daemon scheduler subsystem - return fmt.Errorf("run aborted (--cron)") - } - if !isatty.IsTerminal(os.Stdin.Fd()) { - return fmt.Errorf("run aborted (stdin is not a tty)") - } - description := fmt.Sprintf(`The resource %s requires a run confirmation. -Please make sure you fully understand its role and effects before confirming the run. -Enter "yes" if you really want to run.`, t.RID()) - s, err := confirmation.ReadLn(description, time.Second*30) - if err != nil { - return fmt.Errorf("read confirmation: %w", err) - } - if s == "yes" { - t.Log().Infof("run confirmed interactively") - return nil - } - return fmt.Errorf("run aborted") -} - -// notifyRunDone is a noop here as for now the daemon api has no support for -// POST /run_done, and may not need one. -func (t T) notifyRunDone() error { - return nil -} - -func (t T) ScheduleOptions() resource.ScheduleOptions { - return resource.ScheduleOptions{ - Action: "run", - Option: "schedule", - Base: "", - RequireConfirmation: t.Confirmation, - RequireProvisioned: true, - RequireCollector: false, - } -} diff --git a/drivers/restaskhost/manifest.go b/drivers/restaskhost/manifest.go index 35592e132..89ec85ec0 100644 --- a/drivers/restaskhost/manifest.go +++ b/drivers/restaskhost/manifest.go @@ -5,6 +5,7 @@ import ( "github.com/opensvc/om3/core/manifest" "github.com/opensvc/om3/core/naming" "github.com/opensvc/om3/drivers/resapp" + "github.com/opensvc/om3/drivers/restask" ) var ( @@ -23,9 +24,30 @@ func (t T) Manifest() *manifest.T { manifest.ContextObjectPath, manifest.ContextNodes, manifest.ContextObjectID, + resapp.BaseKeywordTimeout, + resapp.BaseKeywordStopTimeout, + resapp.BaseKeywordSecretsEnv, + resapp.BaseKeywordConfigsEnv, + resapp.BaseKeywordEnv, + resapp.BaseKeywordRetCodes, + resapp.BaseKeywordUmask, + resapp.UnixKeywordStopCmd, + resapp.UnixKeywordCwd, + resapp.UnixKeywordUser, + resapp.UnixKeywordGroup, + resapp.UnixKeywordLimitCPU, + resapp.UnixKeywordLimitCore, + resapp.UnixKeywordLimitData, + resapp.UnixKeywordLimitFSize, + resapp.UnixKeywordLimitMemLock, + resapp.UnixKeywordLimitNoFile, + resapp.UnixKeywordLimitNProc, + resapp.UnixKeywordLimitRSS, + resapp.UnixKeywordLimitStack, + resapp.UnixKeywordLimitVmem, + resapp.UnixKeywordLimitAS, ) - m.AddKeywords(resapp.BaseKeywords...) - m.AddKeywords(resapp.UnixKeywords...) + m.AddKeywords(restask.Keywords...) m.AddKeywords(Keywords...) return m } diff --git a/drivers/restaskhost/text/kw/check b/drivers/restaskhost/text/kw/check deleted file mode 100644 index b661ecbf9..000000000 --- a/drivers/restaskhost/text/kw/check +++ /dev/null @@ -1,4 +0,0 @@ -If set to `last_run`, the last run retcode is used to report a task resource -status. - -If not set (default), the status of a task is always n/a. diff --git a/drivers/restaskhost/text/kw/confirmation b/drivers/restaskhost/text/kw/confirmation deleted file mode 100644 index 4d6055128..000000000 --- a/drivers/restaskhost/text/kw/confirmation +++ /dev/null @@ -1,3 +0,0 @@ -If set to `true`, ask for an interactive confirmation to run the task. - -This flag can be used for dangerous tasks like data restoration. diff --git a/drivers/restaskhost/text/kw/log b/drivers/restaskhost/text/kw/log deleted file mode 100644 index eb0198f96..000000000 --- a/drivers/restaskhost/text/kw/log +++ /dev/null @@ -1 +0,0 @@ -Log the task outputs in the service log. diff --git a/drivers/restaskhost/text/kw/on_error b/drivers/restaskhost/text/kw/on_error deleted file mode 100644 index ffcbe5af1..000000000 --- a/drivers/restaskhost/text/kw/on_error +++ /dev/null @@ -1 +0,0 @@ -A command to execute on `run` action if `command` returned an error. diff --git a/drivers/restaskhost/text/kw/run_timeout b/drivers/restaskhost/text/kw/run_timeout deleted file mode 100644 index 36c14b5af..000000000 --- a/drivers/restaskhost/text/kw/run_timeout +++ /dev/null @@ -1,3 +0,0 @@ -Wait for `` before declaring the action a failure. - -Takes precedence over `timeout`. diff --git a/drivers/restaskhost/text/kw/snooze b/drivers/restaskhost/text/kw/snooze deleted file mode 100644 index 9fc11cb99..000000000 --- a/drivers/restaskhost/text/kw/snooze +++ /dev/null @@ -1,3 +0,0 @@ -Snooze the service before running the task, so if the command is cause a -status degradation the user can decide to snooze alarms for the duration -set as value. diff --git a/drivers/restaskhost/text/kw/timeout b/drivers/restaskhost/text/kw/timeout deleted file mode 100644 index 03b172b1b..000000000 --- a/drivers/restaskhost/text/kw/timeout +++ /dev/null @@ -1,3 +0,0 @@ -Wait for `` before declaring the task `run` action a failure. - -If no timeout is set, the agent waits indefinitely for the task command to exit. From cdf225511ddac95d82b6084b3f920f29e54fa272 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 5 Sep 2024 11:02:35 +0200 Subject: [PATCH 09/16] Refine the CHANGELOG entry for the relay heartbeat --- CHANGELOG.md | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5ad813e..db58b6710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,7 +121,7 @@ - - - - - - =============== =============== =============== =============== ================ ================= -* **breaking change:** The raw protocol is dropped. `echo | socat - /var/lib/opensvc/lsnr/lsnr.sock` +* **breaking change:** The raw protocol is dropped. `echo | socat - /var/lib/opensvc/lsnr/lsnr.sock` * **breaking change:** Task and sync resources are now non-optional by default, but their status is never aggregated in the instance availability status. Errors in the run produce a non-zero exitcode if optional=false, zero if optional=true. @@ -216,13 +216,44 @@ In 2.1 the instance status resources was a dict of rid to exposed status now it is a list of exposed status, rid is now a property of exposed status -* **breaking change:** replace relay heartbeat secret keyword with username and password. - - The password value is the sec object path containing the actual relay password encoded in the password key. +* **breaking change:** relay heartbeat changes. + + The v3 agent needs to address a v3 relay. + + The v3 relay must have a user with the `heartbeat` grant that the client will need to use. + ``` + om system/usr/relayuser create --kw grant=heartbeat + om system/usr/relayuser add --key password --value $PASSWORD + ``` + + On the cluster nodes, store the relay password in a secret: + ``` + om system/sec/relay-v3 create + om system/sec/relay-v3 add --key password --value $PASSWORD + ``` + + And the heartbeat configuration: + ``` + [hb#1] + type = relay + relay = relay-v2 + secret = 3aaf0dae606212349b7123eb8cc7e89b + ``` + + Becomes: + ``` + [hb#1] + type = relay + relay = relay1-v3 + username = relayuser + password = system/sec/relay-v3 + ``` + + Where the password is the value of the `þassword` key in `system/sec/relay-v3`. #### logging -* **breaking change** OpenSVC no longer logs to private log files. It logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. +* **breaking change:** OpenSVC no longer logs to private log files. It logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. * The **sc** log entries attribute is replaced with **origin=daemon/scheduler**. From a762fdb05fc9bc3a48752db9fd7961990084f7de Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 5 Sep 2024 11:28:45 +0200 Subject: [PATCH 10/16] Move the relay hb note in the CHANGELOG Closer to the arbitrator note. --- CHANGELOG.md | 73 ++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db58b6710..d62f33225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -216,41 +216,6 @@ In 2.1 the instance status resources was a dict of rid to exposed status now it is a list of exposed status, rid is now a property of exposed status -* **breaking change:** relay heartbeat changes. - - The v3 agent needs to address a v3 relay. - - The v3 relay must have a user with the `heartbeat` grant that the client will need to use. - ``` - om system/usr/relayuser create --kw grant=heartbeat - om system/usr/relayuser add --key password --value $PASSWORD - ``` - - On the cluster nodes, store the relay password in a secret: - ``` - om system/sec/relay-v3 create - om system/sec/relay-v3 add --key password --value $PASSWORD - ``` - - And the heartbeat configuration: - ``` - [hb#1] - type = relay - relay = relay-v2 - secret = 3aaf0dae606212349b7123eb8cc7e89b - ``` - - Becomes: - ``` - [hb#1] - type = relay - relay = relay1-v3 - username = relayuser - password = system/sec/relay-v3 - ``` - - Where the password is the value of the `þassword` key in `system/sec/relay-v3`. - #### logging * **breaking change:** OpenSVC no longer logs to private log files. It logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. @@ -260,7 +225,43 @@ * The **origin=daemon** log entries attribute is replaced with **origin=daemon/monitor** ### cluster config -#### arbitrator + +#### Relay Heartbeat + +The v3 agent needs to address a v3 relay. + +The v3 relay must have a user with the `heartbeat` grant that the client will need to use. +``` +om system/usr/relayuser create --kw grant=heartbeat +om system/usr/relayuser add --key password --value $PASSWORD +``` + +On the cluster nodes, store the relay password in a secret: +``` +om system/sec/relay-v3 create +om system/sec/relay-v3 add --key password --value $PASSWORD +``` + +And the heartbeat configuration: +``` +[hb#1] +type = relay +relay = relay-v2 +secret = 3aaf0dae606212349b7123eb8cc7e89b +``` + +Becomes: +``` +[hb#1] +type = relay +relay = relay1-v3 +username = relayuser +password = system/sec/relay-v3 +``` + +Where the password is the value of the `þassword` key in `system/sec/relay-v3`. + +#### Arbitrator * The new keyword **uri** replaces **name**. From a9883d4004f648e736dd95d535e05918a4b0c199 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 5 Sep 2024 15:11:58 +0200 Subject: [PATCH 11/16] Reorg the CHANGELOG --- CHANGELOG.md | 351 +++++++++++++++++++++++++++++---------------------- 1 file changed, 197 insertions(+), 154 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d62f33225..137aeb67e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,232 +1,259 @@ -# opensvc agent Changelog +# OpenSVC agent v3 Changelog -## v3.0.0 +## Breaking Changes -### core +### Core -* **breaking change:** Drop the constraints svc keyword. Use host label selectors instead. +* Switch to RFC3389 time format in all internal and exposed data. -* **breaking change:** The "om daemon dns dump" command is deprecated (with backward compatibility) in favour of "om dns dump". As a consequence, the "dns" object path, if used, is now masked. The root/svc/dns identifier can still be used to help with the transition to a new object name. + A unix timestamp was previously used, but it was tedious for users to understand the json data. And go makes the time.Time type unavoidable anyway, so the performance argument for timestamps doesn't stand anymore. -* The set and unset commands are superseded by update --set ... --unset ... --delete. This new command allow to have a single commit for different kind of changes. The set and unset commands are now hidden so users don't get tempted to use them anymore. +* The keyword `cluster.name` has no default value. -* **breaking change:** set/unset/get/eval now need --local to operate on the local node without api calls. + In v2.1, the default cluster name was `default`. + + In v3, the startup will automatically replace the undefined `cluster.name` with a random human-readable value. -* New placement policy `last start`. Use the mtime of `/last_start` as the candidate sort key. More recent has higher priority. +* The keyword `cluster.name` is no longer scopable. -* **breaking change:** Drop the --dry-run flag. +* Drop the `constraints` svc keyword. Use host label selectors instead. -* **breaking change:** Drop the `default_mon_format` node keyword. It should be a user-level setting, not a node-level config. +* The "om daemon dns dump" command is deprecated (with backward compatibility) in favour of "om dns dump". As a consequence, the "dns" object path, if used, is now masked. The root/svc/dns identifier can still be used to help with the transition to a new object name. -* **breaking change:** Drop the `reboot` node command and associated keywords: `reboot.schedule`, `reboot.pre`, `reboot.once`, `reboot.blocking_pre` +* `set`, `unset`, `get`, `eval` now need `--local` to operate on the local node without api calls. -* **breaking change:** Drop the `rotate root password` node command and associated keywords: `rotate_root_pw.schedule` +* Drop the --dry-run flag. -* **breaking change:** Drop the `pushstats` node command and associated keywords: `stats_collection.schedule`, `stats.schedule`, `stats.disable` +* Drop the `default_mon_format` node keyword. It should be a user-level setting, not a node-level config. -* **breaking change:** Deny object path name and namespaces longer than 63 character. +* Drop the `reboot` node command and associated keywords: `reboot.schedule`, `reboot.pre`, `reboot.once`, `reboot.blocking_pre` -* **breaking change:** replace the --debug flag with --log debug|info|warn|error|fatal|panic +* Drop the `rotate root password` node command and associated keywords: `rotate_root_pw.schedule` -* Add --quiet to disable both the progress renderer and the console logging +* Drop the `pushstats` node command and associated keywords: `stats_collection.schedule`, `stats.schedule`, `stats.disable` + +* Deny object path name and namespaces longer than 63 character. + +* Replace the `--debug` flag with --log debug|info|warn|error|fatal|panic -* **breaking change:** remove the --eval flag of the get command. +* Remove the `--eval` flag of the get command. - users need to use the "eval" command instead. + Users need to use the `eval` command instead. -* **breaking change:** remove the --unprovision flag of the delete command. +* Remove the `--unprovision` flag of the `delete` command. - users need to use the "unprovision && delete" sequence instead. + Users need to use the `unprovision` and `delete` sequence instead, or `purge`. -* **breaking change:** Remove the --rid flag of the delete command. +* Remove the `--rid` flag of the `delete` command. - Users can use the "unset --section " command instead. + Users can use the `unset --section ` command instead. -* **breaking change:** command flags that accept a duration now require a unit. +* Command flags that accept a duration now require a unit. change --waitlock=60 to --waitlock=1m change --time=10 to --time=10s -* **breaking change:** drop support for deprecated driver group names: - - drbd: disk.drbd - vdisk: disk.vdisk - vmdg: disk.ldom - pool: disk.zpool - zpool: disk.zpool - loop: disk.loop - md: disk.md - zvol: disk.zvol - lv: disk.lv - raw: disk.raw - vxdg: disk.vxdg - vxvol: disk.vxvol +* Drop support for driver group names already deprecated in v2.1: + + ``` + drbd disk.drbd + vdisk disk.vdisk + vmdg disk.ldom + pool disk.zpool + zpool disk.zpool + loop disk.loop + md disk.md + zvol disk.zvol + lv disk.lv + raw disk.raw + vxdg disk.vxdg + vxvol disk.vxvol + ``` For example, a [md#1] section needs reformatting as: [disk#1] type = md -* **breaking change:** stop matching DEFAULT. for ":" object selector expressions. Match only sections basename (like in [#]). +* Stop matching `DEFAULT.foo` with the `om foo: ls`. -* **breaking change:** drop backward compatibility for the always_on= keyword. + Match only objects with `foo` as a section basename (eg. `[foo#bar]`). -* New fields in print schedule json format: node, path +* Drop backward compatibility for the `always_on=` keyword. + + The `standby=true` keyword is the target since v2.1. + +* New cgroup layout. -* **breaking change:** new cgroup layout. The previous organization allowed conflicts between different object types, and was hard to read. + The previous layout allowed conflicts between different object types (eg. `vol` and `svc`). -* Change the "print status" instance-level errors and warnings (to no-whitespace words): +* Change the `print status` instance-level errors and warnings (to no-whitespace words): - part provisioned => mix-provisioned - not provisioned => not-provisioned - node frozen => node-frozen - daemon down => daemon-down + ``` + part provisioned -> mix-provisioned + not provisioned -> not-provisioned + node frozen -> node-frozen + daemon down -> daemon-down + ``` -* **breaking change:** Rename the create --config flag to --from, and merge --template into --from. +* Simplify the `om create` flags + + ``` + --config -> --from + --template -> --from + ``` Support the following template selector syntaxes: - --from 111 - --from template://111 - --from "template://my tmpl 111" + ``` + --from 111 + --from template://111 + --from "template://my tmpl 111" + ``` -* **breaking change:** Rename commands +* Rename commands - node scan capabilities => node capabilities scan - node print capabilities => node capabilities list + ``` + node scan capabilities -> node capabilities scan + node print capabilities -> node capabilities list + ``` -* **breaking change:** In previous releases, om node get --kw node.env returned the keyword's raw string value from cluster.conf if it is not defined in node.conf. In this release, this get command returns the empty string. The eval command is unchanged though: it still falls back to cluster.conf. +* In previous releases, `om node get --kw node.env` returned the keyword's raw string value from `cluster.conf` if it is not defined in `node.conf`. - In v2: + In this release, this command returns the empty string. The `eval` command is unchanged though: it still falls back to `cluster.conf`. - =============== =============== =============== =============== ================ ================= - node.conf cluster.conf om node get om node eval om cluster get om cluster eval - =============== =============== =============== =============== ================ ================= - fr kr fr fr kr kr - fr - fr fr - - - - kr kr kr kr kr - - - - - - - - =============== =============== =============== =============== ================ ================= + In v2: + ``` + node.conf cluster.conf om node get om node eval om cluster get om cluster eval + --------- ------------ ----------- ------------ -------------- --------------- + fr kr fr fr kr kr + fr - fr fr - - + - kr kr kr kr kr + - - - - - - + ``` In v3: - =============== =============== =============== =============== ================ ================= - node.conf cluster.conf om node get om node eval om cluster get om cluster eval - =============== =============== =============== =============== ================ ================= - fr kr fr fr kr kr - fr - fr fr - - - - kr - kr kr kr - - - - - - - - =============== =============== =============== =============== ================ ================= + ``` + node.conf cluster.conf om node get om node eval om cluster get om cluster eval + --------- ------------ ----------- ------------ -------------- --------------- + fr kr fr fr kr kr + fr - fr fr - - + - kr - kr kr kr + - - - - - - + ``` -* **breaking change:** The raw protocol is dropped. `echo | socat - /var/lib/opensvc/lsnr/lsnr.sock` +* The `raw` jsonrpc protocol API is dropped. -* **breaking change:** Task and sync resources are now non-optional by default, but their status is never aggregated in the instance availability status. Errors in the run produce a non-zero exitcode if optional=false, zero if optional=true. + For example, this v2.1 API call is no longer supported: + ``` + echo '{"action": "daemon_status"}' | socat - /var/lib/opensvc/lsnr/lsnr.sock + ``` + + To keep using a root Unix Socket in v3, you can switch to: + ``` + curl -o- -X GET -H "Content-Type: application/json" --unix-socket /var/lib/opensvc/lsnr/http.sock http://localhost/daemon/status + ``` -### objects +* Propagate the task run and sync errors to a non-zero exitcode. + + The `task` and `sync` resources are now `optional=false` by default, but their status is not aggregated in the instance availability status whatever the `optional` value. Errors in the run produce a non-zero exitcode if optional=false, zero if optional=true. + +* Drop support of some `DEFAULT` section keywords: -* **breaking change:** drop support of some DEFAULT keywords: * `svc_flex_cpu_low_threshold` * `svc_flex_cpu_high_threshold` -* **breaking change:** key-value stores (cfg, sec, usr kinded objects) `change` action is no longer failing if the key does not exist. The key is added instead. - -### commands +* Key-Value stores (cfg, sec, usr kinded objects) `change` action is no longer failing if the key does not exist. The key is added instead. -* **breaking change:** "om node freeze" is now local only. Use "om cluster freeze" for the orchestrated freeze of all nodes. Same applies to "unfreeze" and its hidden alias "thaw". +* `om node freeze` is now local only. Use `om cluster freeze` for the orchestrated freeze of all nodes. Same applies to `unfreeze` and its hidden alias `thaw`. -* **breaking change:** "om cluster abort" replaces "om node abort" to abort the pending cluster action orchestration. +* `om cluster abort` replaces `om node abort` to abort the pending cluster action orchestration. -* **breaking change:** "om ... set|unset" no longer accept --param and --value. Use --kw instead, which was also supported in v2. +* `om ... set|unset` no longer accept ``--param`` and ``--value``. Use ``--kw`` instead, which was also supported in v2. -* **breaking change:** "om node logs" now display only local logs. A new "om cluster logs" command displays all cluster nodes logs. +* `om node logs` now display only local logs. A new `om cluster logs` command displays all cluster nodes logs. -* "unset" now accepts "--section " to remove an cluster, node or object configuration section. +* `om unset` now accepts `--section ` to remove a cluster, node or object configuration section. -* "om monitor" instance availability icons changes: +* `om monitor` instance availability icons changes: + ``` standby down: s => x standby up: S => o + ``` + +### Driver: ip -### driver ip +* Drop the `dns_name_suffix`, `provisioner`, `dns_update` keywords. The zone management feature of the collector will be dropped in the collector too. -* **breaking change:** Drop the `dns_name_suffix`, `provisioner`, `dns_update` keywords. The zone management feature of the collector will be dropped in the collector too. +### Driver: fs -### driver fs +* Keywords `size` and `vg` are no longer supported, and a logical volume can no longer be created by the fs provisioner. Use a proper disk.lv to do that. -* **breaking change:** keywords `size` and `vg` are no longer supported, and a logical volume can no longer be created by the fs provisioner. Use a proper disk.lv to do that. +### Driver: sync -### driver sync +* The `sync drp` action is removed. Use `sync update --target drpnodes` instead. -* **breaking change:** The "sync drp" action is removed. Use "sync update --target drpnodes" instead. +* The `sync nodes` action is removed. Use `sync update --target nodes` instead. -* **breaking change:** The "sync nodes" action is removed. Use "sync update --target nodes" instead. +* The `sync all` action is deprecated. Use `sync update` with no `--target` flag instead. -* The "sync all" action is deprecated. Use "sync update" with no --target flag instead. +* The `sync full` and `sync update` now both accept a `--target nodes|drpnodes|node_selector_expr` flag -* The "sync full" and "sync update" now both accept a "--target nodes|drpnodes|node_selector_expr" flag +### Driver: app -### driver app - -* **breaking change:** keyword `environment` now keep var name unchanged (respect mixedCase) +* The keyword `environment` now keeps the variable names unchanged and accepts mixedCase. - environment = Foo=one bar=2 Z=u - => - Foo=one was previsouly changed to FOO=one - bar=2 was previsouly changed to BAR=2 - Zoo=u was previously changed to ZOO=u - -* **breaking change:** Remove support on some deprecated env var - - The following env var are not anymore added to process env var during actions: - + ``` + With: + environment = Foo=one bar=2 Z=u + + Foo=one was previsouly changed to FOO=one + bar=2 was previsouly changed to BAR=2 + Zoo=u was previously changed to ZOO=u + ``` + +* Remove support of some deprecated environment variables. + + The following variables are no longer added to process environment during actions: + + ``` OPENSVC_SVCNAME OPENSVC_SVC_ID + ``` -* **breaking change:** Fix OPENSVC_ID var value on app resources +* Fix `OPENSVC_ID` environment variable value in `app` resources - In the app drivers, the object id is now exposed as the OPENSVC_ID environment variable. - In 2.1, OPENSVC_ID was set to the object path name (for example "foo" from "test/svc/foo"). + In the `app` drivers, the object id is now exposed as the `OPENSVC_ID` environment variable. -* The kill keyword is removed. The default behaviour is now to kill all processes with the matching OPENSVC_ID and OPENSVC_RID variables in their environment. - In 2.1 the default behaviour was to try to identify the topmost process matching the start command in the process command line, and having the matching env vars, but this guess is not accurate enough as processes can change their cmdline via PRCTL or via execv. - If the new behaviour is not acceptable, users can provide their own stopper via the "stop" keyword. - -### object sec - -* **breaking change:** Remove the `fullpem` action. Add the `fullpem` key on `gencert` action. - -### daemon - -* Add a 60 seconds timeout to pre_monitor_action. The 2.1 daemon waits forever for this callout to terminate. - -* Earlier local object instance orchestration after node boot - - * In 2.1 local object instance orchestration waits for all local object instances boot action done - * Now object instance orchestration only waits for boot action completed. Each instance has a last boot id. - -* **breaking change:** switch to time.Time in RFC3389 format in all internal and exposed data + In 2.1, `OPENSVC_ID` was set to the object name (for example `foo` in `test/svc/foo`). + +* The `kill` keyword is removed. - A unix timestamp was previously used, but it was tedious for users to understand the json data. And go makes the time.Time type unavoidable anyway, so the performance argument for timestamps doesn't stand anymore. + The default behaviour is now to kill all processes with the matching `OPENSVC_ID` and `OPENSVC_RID` variables in their environment. + + In 2.1 the default behaviour was to try to identify the topmost process matching the start command in the process command line, and having the matching env vars, but this guess is not accurate enough as processes can change their cmdline via PRCTL or via execv. + + If the new behaviour is not acceptable, users can provide their own stopper via the "stop" keyword. -* **breaking change:** change instance status resources type +### Object: sec - In 2.1 the instance status resources was a dict of rid to exposed status - now it is a list of exposed status, rid is now a property of exposed status +* Remove the `fullpem` action. Add the `fullpem` key on `gencert` action. -#### logging +### Logging -* **breaking change:** OpenSVC no longer logs to private log files. It logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. +* OpenSVC no longer logs to private log files. -* The **sc** log entries attribute is replaced with **origin=daemon/scheduler**. + It logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. -* The **origin=daemon** log entries attribute is replaced with **origin=daemon/monitor** +* The `sc` log entries attribute is replaced with `origin=daemon/scheduler`. -### cluster config +* The `origin=daemon` log entries attribute is replaced with `origin=daemon/monitor`. -#### Relay Heartbeat +### Heartbeat: relay The v3 agent needs to address a v3 relay. @@ -261,9 +288,9 @@ password = system/sec/relay-v3 Where the password is the value of the `þassword` key in `system/sec/relay-v3`. -#### Arbitrator +### Arbitrator -* The new keyword **uri** replaces **name**. +* The new keyword `uri` replaces `name`. * When the uri scheme is http or https, the vote checker is based on a GET request, else it is based on a TCP connect. When the port is not specified in a TCP connect uri, the 1215 port is implied. @@ -274,30 +301,46 @@ Where the password is the value of the `þassword` key in `system/sec/relay-v3`. uri = arbitrator1.opensvc.com:1215 uri = arbitrator1.opensvc.com # implicitly port 1215 -* The new keyword **insecure** disables the server certificate validation when the uri scheme is https, the default is false. +* The new keyword `insecure` disables the server certificate validation when the uri scheme is https, the default is false. -* The **name** keyword is deprecated. Aliased to **uri** to ease transition. +* The `name* keyword is deprecated. Aliased to `uri` to ease transition. -* The **timeout** keyword is removed to avoid users setting a value greater than the ready period, +* The `timeout` keyword is removed to avoid users setting a value greater than the ready period, which would let the service double start before the end of the vote. The internal timeout value is now set to a third of the ready period. -* The **secret** keyword is now ignored. +* The `secret` keyword is now ignored. + +## Enhancements -#### cluster section +### Core + +* The `set` and `unset` commands are complemented by `update --set ... --unset ... --delete`. This new command allow to have a single commit for different kind of changes. The set and unset commands are now hidden so users don't get tempted to use them anymore. + +* New placement policy `last start`. Use the mtime of `/last_start` as the candidate sort key. More recent has higher priority. + +* Add --quiet to disable both the progress renderer and the console logging + +* New fields in print schedule json format: node, path + +### Daemon + +* Add a 60 seconds timeout to `pre_monitor_action`. The 2.1 daemon waits forever for this callout to terminate. + +* Earlier local object instance orchestration after node boot -##### cluster.name + In 2.1 local object instance orchestration waits for all local object instances boot action done + + Now object instance orchestration only waits for boot action completed. Each instance has a last boot id. -* **breaking change:** keyword `cluster.name` has no default value. It has - previously the default value *default*. Now daemon startup will automatically - replace undefined cluster.name with a random value. +## Upgrade from b2.1 -* **breaking change:** keyword `cluster.name` is not anymore scopable. +### Cluster Config -## upgrade from b2.1 -### cluster config +* Need to set explicitely the `cluster.name` because the v3 daemon will generate a random cluster name if none is set: -* Need explicit cluster.name because of v3 random cluster name: + ``` + # Ensure cluster.name is defined before upgrade to v3 + om cluster set --kw cluster.name=$(om cluster eval --kw cluster.name) + ``` - # Ensure cluster.name is defined before upgrade to v3 - om cluster set --kw cluster.name=$(om cluster eval --kw cluster.name) From 713f135990e398a2b764b36f502907c3269175cb Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Thu, 5 Sep 2024 19:52:57 +0200 Subject: [PATCH 12/16] Update CHANGELOG --- CHANGELOG.md | 377 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 230 insertions(+), 147 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 137aeb67e..a5d1f2204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,94 +1,162 @@ -# OpenSVC agent v3 Changelog +# OpenSVC Agent v3 Changelog ## Breaking Changes -### Core - -* Switch to RFC3389 time format in all internal and exposed data. - - A unix timestamp was previously used, but it was tedious for users to understand the json data. And go makes the time.Time type unavoidable anyway, so the performance argument for timestamps doesn't stand anymore. +### Cluster and Node Configuration -* The keyword `cluster.name` has no default value. +* **Time format change:** + OpenSVC now uses RFC3339 time format for all internal and exposed data, replacing the Unix timestamps. +* **`cluster.name` default value:** In v2.1, the default cluster name was `default`. - - In v3, the startup will automatically replace the undefined `cluster.name` with a random human-readable value. - -* The keyword `cluster.name` is no longer scopable. - -* Drop the `constraints` svc keyword. Use host label selectors instead. + In v3, if `cluster.name` is undefined at startup, it will be automatically replaced with a randomly generated human-readable value. -* The "om daemon dns dump" command is deprecated (with backward compatibility) in favour of "om dns dump". As a consequence, the "dns" object path, if used, is now masked. The root/svc/dns identifier can still be used to help with the transition to a new object name. - -* `set`, `unset`, `get`, `eval` now need `--local` to operate on the local node without api calls. +* **`cluster.name` scope:** + This keyword is no longer scopable. + +* **`node.default_mon_format` removed:** + It should be a user-level setting, not a node-level config. -* Drop the --dry-run flag. +* **`reboot` section removed:** + * `reboot.schedule` + + * `reboot.pre` + + * `reboot.once` + + * `reboot.blocking_pre` -* Drop the `default_mon_format` node keyword. It should be a user-level setting, not a node-level config. +* **`rotate_root_pw` section removed:** + * `rotate_root_pw.schedule` + +* **`stats_collection` section removed:** + * `stats_collection.schedule` + + * `stats.schedule` + + * `stats.disable` + +### Object Configuration -* Drop the `reboot` node command and associated keywords: `reboot.schedule`, `reboot.pre`, `reboot.once`, `reboot.blocking_pre` +* **Keywords removed:** + * `svc_flex_cpu_low_threshold` + + * `svc_flex_cpu_high_threshold` + + * `constraints` + Replaced by host label selectors in `nodes`. + Example: + ``` + [DEFAULT] + nodes = az=fr1 az=us1 + ``` + + * `always_on=` + Replaced by `standby=true`. + This keyword was already marked deprecated in v2.1. + +* **Driver Group Names Removed:** + + Drop support for driver group names:** + + * `drbd` + Replaced by `disk#foo.type=drbd` + + * `vdisk` + Replaced by `disk#foo.type=vdisk` + + * `vmdg` + Replaced by `disk#foo.type=ldom` + + * `pool` + Replaced by `disk#foo.type=zpool` + + * `zpool` + Replaced by `disk#foo.type=zpool` + + * `loop` + Replaced by `disk#foo.type=loop` + + * `md` + Replaced by `disk#foo.type=md` + + * `zvol` + Replaced by `disk#foo.type=zvol` + + * `lv` + Replaced by `disk#foo.type=lv` + + * `raw` + Replaced by `disk#foo.type=raw` + + * `vxdg` + Replaced by `disk#foo.type=vxdg` + + * `vxvol` + Replaced by `disk#foo.type=vxvol` + + For example, a `[md#1]` section needs reformatting as: + ``` + [disk#1] + type = md + ``` + + These driver group names were already deprecated in v2.1. -* Drop the `rotate root password` node command and associated keywords: `rotate_root_pw.schedule` +### Commands -* Drop the `pushstats` node command and associated keywords: `stats_collection.schedule`, `stats.schedule`, `stats.disable` +* **Deprecated:** + * `om daemon dns dump` + Replaced by `om dns dump`. + As a consequence, the `dns` object path is masked. The `root/svc/dns` path can still be used to help with the transition to a new object name. -* Deny object path name and namespaces longer than 63 character. +* **Configuration updates use the daemon api by default:** + `om set`, `om unset`, `om get`, `om eval` now need `--local` to operate on the local configurations without api calls. -* Replace the `--debug` flag with --log debug|info|warn|error|fatal|panic +* **Removed:** + * `om node reboot` -* Remove the `--eval` flag of the get command. + * `om node rotate root password` - Users need to use the `eval` command instead. + * `om node pushstats` -* Remove the `--unprovision` flag of the `delete` command. + * `node scan capabilities` + Replaced by `node capabilities scan` + + * `node print capabilities` + Replaced by `node capabilities list` + + * `om node abort` + Replaced by `om cluster abort` to abort the pending cluster action orchestration. + - Users need to use the `unprovision` and `delete` sequence instead, or `purge`. + * **Flags Removed:** -* Remove the `--rid` flag of the `delete` command. + * `--debug` + Replaced by `--log debug|info|warn|error|fatal|panic`. - Users can use the `unset --section ` command instead. + * `om get --eval` + Replaced by `om eval` -* Command flags that accept a duration now require a unit. + * `om foo set|unset --param ... --value` + Replaced by `--kw`, which was also supported in v2. - change --waitlock=60 to --waitlock=1m - change --time=10 to --time=10s + * `om delete --unprovision` + Replaced by the `om unprovision` and `om delete` sequence or by `om purge`. -* Drop support for driver group names already deprecated in v2.1: + * `om delete --rid` + Replaced by `om unset --section `. + + * `om --dry-run` +* **Duration flags now require a unit:** ``` - drbd disk.drbd - vdisk disk.vdisk - vmdg disk.ldom - pool disk.zpool - zpool disk.zpool - loop disk.loop - md disk.md - zvol disk.zvol - lv disk.lv - raw disk.raw - vxdg disk.vxdg - vxvol disk.vxvol + --waitlock=60 -> --waitlock=1m + --time=10 -> --time=10s ``` - - For example, a [md#1] section needs reformatting as: - - [disk#1] - type = md - -* Stop matching `DEFAULT.foo` with the `om foo: ls`. - - Match only objects with `foo` as a section basename (eg. `[foo#bar]`). - -* Drop backward compatibility for the `always_on=` keyword. - - The `standby=true` keyword is the target since v2.1. - -* New cgroup layout. - - The previous layout allowed conflicts between different object types (eg. `vol` and `svc`). - -* Change the `print status` instance-level errors and warnings (to no-whitespace words): - + +* **`print status`**: + Change the instance-level errors and warnings (to no-whitespace words): ``` part provisioned -> mix-provisioned not provisioned -> not-provisioned @@ -96,34 +164,23 @@ daemon down -> daemon-down ``` -* Simplify the `om create` flags - - ``` - --config -> --from - --template -> --from - ``` - - Support the following template selector syntaxes: - - ``` - --from 111 - --from template://111 - --from "template://my tmpl 111" - ``` - -* Rename commands - - ``` - node scan capabilities -> node capabilities scan - node print capabilities -> node capabilities list - ``` - +* **`om create`:** + * Simplify the flags + ``` + --config -> --from + --template -> --from + ``` -* In previous releases, `om node get --kw node.env` returned the keyword's raw string value from `cluster.conf` if it is not defined in `node.conf`. + * Support the following template selector syntaxes: + ``` + --from 111 + --from template://111 + --from "template://my tmpl 111" + ``` - In this release, this command returns the empty string. The `eval` command is unchanged though: it still falls back to `cluster.conf`. +* **`om node get|eval`:** + In previous releases, `om node get --kw node.env` returned the keyword's raw string value from `cluster.conf` if it is not defined in `node.conf`: - In v2: ``` node.conf cluster.conf om node get om node eval om cluster get om cluster eval --------- ------------ ----------- ------------ -------------- --------------- @@ -134,7 +191,7 @@ ``` - In v3: + In this release, this command returns the empty string. The `eval` command is unchanged though (it still falls back to `cluster.conf`): ``` node.conf cluster.conf om node get om node eval om cluster get om cluster eval @@ -145,68 +202,96 @@ - - - - - - ``` -* The `raw` jsonrpc protocol API is dropped. - - For example, this v2.1 API call is no longer supported: - ``` - echo '{"action": "daemon_status"}' | socat - /var/lib/opensvc/lsnr/lsnr.sock - ``` +* **`om foo run` and `om foo sync *`:** + Propagate the task run and sync errors to a non-zero exitcode. - To keep using a root Unix Socket in v3, you can switch to: - ``` - curl -o- -X GET -H "Content-Type: application/json" --unix-socket /var/lib/opensvc/lsnr/http.sock http://localhost/daemon/status - ``` - -* Propagate the task run and sync errors to a non-zero exitcode. - The `task` and `sync` resources are now `optional=false` by default, but their status is not aggregated in the instance availability status whatever the `optional` value. Errors in the run produce a non-zero exitcode if optional=false, zero if optional=true. -* Drop support of some `DEFAULT` section keywords: - * `svc_flex_cpu_low_threshold` - * `svc_flex_cpu_high_threshold` +* **`om change`:** + This action is no longer failing if the key does not exist. The key is added instead. -* Key-Value stores (cfg, sec, usr kinded objects) `change` action is no longer failing if the key does not exist. The key is added instead. +* **`om node freeze`:** + This command is now local only. + Use `om cluster freeze` for the orchestrated freeze of all nodes. + Same applies to `om node unfreeze` and its hidden alias `om node thaw`. -* `om node freeze` is now local only. Use `om cluster freeze` for the orchestrated freeze of all nodes. Same applies to `unfreeze` and its hidden alias `thaw`. +* **`om node logs`:** + Now display only local logs. + A new `om cluster logs` command displays all cluster nodes logs. -* `om cluster abort` replaces `om node abort` to abort the pending cluster action orchestration. +* **`om unset`:** + Now accepts `--section ` to remove a cluster, node or object configuration section. -* `om ... set|unset` no longer accept ``--param`` and ``--value``. Use ``--kw`` instead, which was also supported in v2. +* **`om monitor`:** + Instance availability icons changes: + ``` + standby down: s => x + standby up: S => o + ``` + +### Core -* `om node logs` now display only local logs. A new `om cluster logs` command displays all cluster nodes logs. +* **Object Names policy change:** + Deny names and namespaces longer than 63 character. -* `om unset` now accepts `--section ` to remove a cluster, node or object configuration section. +* **Object selector policy:** + Stop matching `DEFAULT.foo` with the `om foo: ls`. + Match only objects with `foo` as a section basename (eg. `[foo#bar]`). -* `om monitor` instance availability icons changes: +* **New cgroup layout:** + The previous layout allowed conflicts between different object types (eg. `vol` and `svc`). +* **The `raw` jsonrpc protocol socket is dropped.** + For example, this v2.1 API call is no longer supported: ``` - standby down: s => x - standby up: S => o + echo '{"action": "daemon_status"}' | socat - /var/lib/opensvc/lsnr/lsnr.sock ``` + To keep using a root Unix Socket in v3, you can switch to: + ``` + curl -o- -X GET -H "Content-Type: application/json" --unix-socket /var/lib/opensvc/lsnr/http.sock http://localhost/daemon/status + ``` + + ### Driver: ip -* Drop the `dns_name_suffix`, `provisioner`, `dns_update` keywords. The zone management feature of the collector will be dropped in the collector too. +* **Removed keywords:** + * `dns_name_suffix` + * `provisioner` + * `dns_update` + +* **Collector DNS zone:** + This feature of the collector, used by the ip driver for one of its provisioning methods, is deprecated. ### Driver: fs -* Keywords `size` and `vg` are no longer supported, and a logical volume can no longer be created by the fs provisioner. Use a proper disk.lv to do that. - +* **Removed keywords:** + * `size` + Configure a disk.lv resource + + * `vg` + Configure a disk.lv resource + ### Driver: sync -* The `sync drp` action is removed. Use `sync update --target drpnodes` instead. +* **Removed actions:** + * `om foo sync drp` + Replaced by `om foo sync update --target drpnodes`. -* The `sync nodes` action is removed. Use `sync update --target nodes` instead. + * `om foo sync nodes` + Replaced by `om foo sync update --target nodes`. -* The `sync all` action is deprecated. Use `sync update` with no `--target` flag instead. + * `om foo sync all` + Replaced by `om foo sync update`. -* The `sync full` and `sync update` now both accept a `--target nodes|drpnodes|node_selector_expr` flag +* **`sync full` and `sync update`:** + Now both accept a `--target nodes|drpnodes|node_selector_expr` flag. ### Driver: app -* The keyword `environment` now keeps the variable names unchanged and accepts mixedCase. - +* **`environment`** + Now keeps the variable names unchanged and accepts mixedCase. ``` With: environment = Foo=one bar=2 Z=u @@ -216,42 +301,40 @@ Zoo=u was previously changed to ZOO=u ``` -* Remove support of some deprecated environment variables. - +* **Removed environment variables:** The following variables are no longer added to process environment during actions: + * `OPENSVC_SVCNAME` - ``` - OPENSVC_SVCNAME - OPENSVC_SVC_ID - ``` - -* Fix `OPENSVC_ID` environment variable value in `app` resources + * `OPENSVC_SVC_ID` - In the `app` drivers, the object id is now exposed as the `OPENSVC_ID` environment variable. - - In 2.1, `OPENSVC_ID` was set to the object name (for example `foo` in `test/svc/foo`). +* **Changed environment variables:** + * `OPENSVC_ID` + In 2.1, `OPENSVC_ID` was set to the object name (for example `foo` in `test/svc/foo`). + In v3 , `OPENSVC_ID` is set to the `DEFAULT.id` value. -* The `kill` keyword is removed. - - The default behaviour is now to kill all processes with the matching `OPENSVC_ID` and `OPENSVC_RID` variables in their environment. +* **Removed keywords:** + * `kill` + The default behaviour is now to kill all processes with the matching `OPENSVC_ID` and `OPENSVC_RID` variables in their environment. - In 2.1 the default behaviour was to try to identify the topmost process matching the start command in the process command line, and having the matching env vars, but this guess is not accurate enough as processes can change their cmdline via PRCTL or via execv. + In 2.1 the default behaviour was to try to identify the topmost process matching the start command in the process command line, and having the matching env vars, but this guess is not accurate enough as processes can change their cmdline via PRCTL or via execv. - If the new behaviour is not acceptable, users can provide their own stopper via the "stop" keyword. + If the new behaviour is not acceptable, users can provide their own stopper via the "stop" keyword. ### Object: sec -* Remove the `fullpem` action. Add the `fullpem` key on `gencert` action. +* **Removed actions:** + * `om sec fullpem` + The `fullpem` key is added to the sec on `gencert` action. ### Logging -* OpenSVC no longer logs to private log files. - - It logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. +* **No more private log files:** + The agent logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. -* The `sc` log entries attribute is replaced with `origin=daemon/scheduler`. +* **Log entries key changes:** + * The `sc` log entries attribute is replaced with `origin=daemon/scheduler`. -* The `origin=daemon` log entries attribute is replaced with `origin=daemon/monitor`. + * The `origin=daemon` log entries attribute is replaced with `origin=daemon/monitor`. ### Heartbeat: relay From 59e6690842b7e91caf2d41ed8b718014bed70d55 Mon Sep 17 00:00:00 2001 From: Cyril Galibern Date: Fri, 6 Sep 2024 08:30:28 +0200 Subject: [PATCH 13/16] CHANGELOG fixes --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5d1f2204..38327cd55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,7 @@ Replaced by `disk#foo.type=vdisk` * `vmdg` - Replaced by `disk#foo.type=ldom` + Replaced by `disk#foo.type=vmdg` * `pool` Replaced by `disk#foo.type=zpool` @@ -329,7 +329,7 @@ ### Logging * **No more private log files:** - The agent logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=svc1 to filter only logs relevant to an object. + The agent logs to journald instead. So the log entries attributes are indexed and can be used to filter logs very fast. Use `journalctl _COMM=om3` to extract all OpenSVC logs. Add OBJ_PATH=foo/svc/svc1 to filter only logs relevant to an object. * **Log entries key changes:** * The `sc` log entries attribute is replaced with `origin=daemon/scheduler`. From 91e8c6b3847fba7a311f1537813c25ecdcf5a42f Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 6 Sep 2024 08:50:33 +0200 Subject: [PATCH 14/16] Use the pid 0 to test a runfile of stale process Instead of 2, which is a like existing pid, especially in a containerized go test run. --- util/runfiles/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/runfiles/main_test.go b/util/runfiles/main_test.go index 5e780b45d..4d11db797 100644 --- a/util/runfiles/main_test.go +++ b/util/runfiles/main_test.go @@ -36,7 +36,7 @@ func TestRunFiles(t *testing.T) { t.Run("CreateStale", func(t *testing.T) { content := "foo" - err := d.create(2, []byte(content)) + err := d.create(0, []byte(content)) assert.NoError(t, err) }) From ea58be3113d96e4d1c6018bef90ac9538e21ba8a Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 6 Sep 2024 08:51:50 +0200 Subject: [PATCH 15/16] Fix two go vet issues in the kvstore handlers daemon/daemonapi/get_object_kvstore_keys.go:51:40: github.com/opensvc/om3/util/key.T struct literal uses unkeyed fields daemon/daemonapi/get_object_kvstore_entry.go:53:3: unreachable code --- daemon/daemonapi/get_object_kvstore_entry.go | 1 - daemon/daemonapi/get_object_kvstore_keys.go | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/daemon/daemonapi/get_object_kvstore_entry.go b/daemon/daemonapi/get_object_kvstore_entry.go index 2129b7755..758b0e403 100644 --- a/daemon/daemonapi/get_object_kvstore_entry.go +++ b/daemon/daemonapi/get_object_kvstore_entry.go @@ -50,7 +50,6 @@ func (a *DaemonAPI) GetObjectKVStoreEntry(ctx echo.Context, namespace string, ki } return ctx.Blob(http.StatusOK, contentType, b) } - return ctx.NoContent(http.StatusNoContent) } for nodename := range instanceConfigData { diff --git a/daemon/daemonapi/get_object_kvstore_keys.go b/daemon/daemonapi/get_object_kvstore_keys.go index 9a0cf8a49..e5b27f772 100644 --- a/daemon/daemonapi/get_object_kvstore_keys.go +++ b/daemon/daemonapi/get_object_kvstore_keys.go @@ -44,11 +44,16 @@ func (a *DaemonAPI) GetObjectKVStoreKeys(ctx echo.Context, namespace string, kin } else { items := make(api.KVStoreKeyListItems, 0) for _, name := range names { + configKey := key.T{ + Section: "data", + Option: name, + } + size := len(ks.Config().GetString(configKey)) items = append(items, api.KVStoreKeyListItem{ Object: p.String(), Node: a.localhost, Key: name, - Size: len(ks.Config().GetString(key.T{"data", name})), + Size: size, }) } return ctx.JSON(http.StatusOK, api.KVStoreKeyList{ From 3ae85fb3c0e5ee2a19e4939e776dbc0f0da42467 Mon Sep 17 00:00:00 2001 From: Christophe Varoqui Date: Fri, 6 Sep 2024 09:49:33 +0200 Subject: [PATCH 16/16] Return typed errors in object.Keystore.Decode func And map them to http status codes in the GET /object/path/.../kvstore/entry api handler. KeystoreErrNotExist => 404 KeystoreErrKeyEmpty => 400 --- core/object/keystore_decode.go | 4 +- core/oxcmd/keystore_decode.go | 2 + daemon/api/api.yaml | 2 + daemon/api/codegen_client_gen.go | 8 + daemon/api/codegen_server_gen.go | 356 +++++++++---------- daemon/daemonapi/get_object_kvstore_entry.go | 12 +- 6 files changed, 201 insertions(+), 183 deletions(-) diff --git a/core/object/keystore_decode.go b/core/object/keystore_decode.go index dcae229e7..fc32a1c1d 100644 --- a/core/object/keystore_decode.go +++ b/core/object/keystore_decode.go @@ -11,10 +11,10 @@ func (t *keystore) decode(keyname string) ([]byte, error) { err error ) if keyname == "" { - return []byte{}, fmt.Errorf("key name can not be empty") + return []byte{}, KeystoreErrKeyEmpty } if !t.HasKey(keyname) { - return []byte{}, fmt.Errorf("key does not exist: %s", keyname) + return []byte{}, fmt.Errorf("%w: %s", KeystoreErrNotExist, keyname) } k := keyFromName(keyname) if s, err = t.config.GetStrict(k); err != nil { diff --git a/core/oxcmd/keystore_decode.go b/core/oxcmd/keystore_decode.go index 85f4319ec..dd5a786d2 100644 --- a/core/oxcmd/keystore_decode.go +++ b/core/oxcmd/keystore_decode.go @@ -66,6 +66,8 @@ func (t *CmdKeystoreDecode) RunForPath(ctx context.Context, c *client.T, path na return fmt.Errorf("%s: %s", path, *response.JSON403) case http.StatusInternalServerError: return fmt.Errorf("%s: %s", path, *response.JSON500) + case http.StatusNotFound: + return fmt.Errorf("%s: %s", path, *response.JSON404) default: return fmt.Errorf("%s: unexpected response: %s", path, response.Status()) } diff --git a/daemon/api/api.yaml b/daemon/api/api.yaml index 5d0ce271b..ac97ca32e 100644 --- a/daemon/api/api.yaml +++ b/daemon/api/api.yaml @@ -2400,6 +2400,8 @@ paths: $ref: '#/components/responses/401' 403: $ref: '#/components/responses/403' + 404: + $ref: '#/components/responses/403' 500: $ref: '#/components/responses/500' diff --git a/daemon/api/codegen_client_gen.go b/daemon/api/codegen_client_gen.go index 4d5a7dd73..81127791b 100644 --- a/daemon/api/codegen_client_gen.go +++ b/daemon/api/codegen_client_gen.go @@ -11542,6 +11542,7 @@ type GetObjectKVStoreEntryResponse struct { JSON400 *N400 JSON401 *N401 JSON403 *N403 + JSON404 *N403 JSON500 *N500 } @@ -18143,6 +18144,13 @@ func ParseGetObjectKVStoreEntryResponse(rsp *http.Response) (*GetObjectKVStoreEn } response.JSON403 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest N403 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: var dest N500 if err := json.Unmarshal(bodyBytes, &dest); err != nil { diff --git a/daemon/api/codegen_server_gen.go b/daemon/api/codegen_server_gen.go index b17e7816e..4e8c10582 100644 --- a/daemon/api/codegen_server_gen.go +++ b/daemon/api/codegen_server_gen.go @@ -4373,7 +4373,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL var swaggerSpec = []string{ "H4sIAAAAAAAC/+x9e3Mbt5LvV0HxbFWSvdTLdrKJb6W2HCvO8Ylj60j22aqNfFXgTJPE0QwwATCUlZS/", - "+y285gkMZ0jqYWn+iSMOHo3GrxsNoLvx1yRiacYoUCkmz/+aZJjjFCRw/dfx6U/HLxmdk8VbnIL6JQYR", + "+y285gkMZ0jqYWn+iSMOHo3GrxuNRqPx1yRiacYoUCkmz/+aZJjjFCRw/dfx6U/HLxmdk8VbnIL6JQYR", "cZJJwujk+UQuAc3zJEEZlkvE5kj/QBJARKAY4jyCGM05S/UHqtqYToiq+UcO/Hoynejfnk/sJw5/5IRD", "PHkueQ7TiYiWkGLVr7zOVDkhOaGLyefP08lxzrEho0lVij+h2H3191f5XPYBn3CaJerzt2Iy9XT58won", "OZYeRoD74u+u8rk1pBljCWBqOwAqX5FEAm/3kRAhFY9BFVJcVqX8/RUfy96IhFS0GzUlEXzKOAhBGH2O", @@ -4392,183 +4392,183 @@ var swaggerSpec = []string{ "Mx5Bz9E1tgJD7HrP5CtjRyNAMlTdR1wtgRYbCbpA2I13H52B1D/VilucONb+qBCOOMicU4Ew+gnH6BT+", "yEFIBJwzvt+F71/hOjS0S7julKT6EF+gy5WQjOvZchvqrm5Fd79rBapPh02xqK/NmogaTYrrYbquepJl", "0SoZ4pCyFdQlGehqfxNBfgM4Nnsyr+FvvvbDtYUH8DMShxrkrsyFaNg7xR4yz0l7BHUWlz0Zy0HZQRU6", - "OrrvNLIandRbPQMZnEMBctgksgzMeYwzfyBWO/3z/PDwaXR5pf+F382fhMbwyfzy0fzCMvOn+UurLvOD", - "UfeIZSghl4B+RP/nR7T3YxsogOWPc54TKYZA5SyfqYGGeGC+rlvfdUvv8SLUjMSLnm2wYBOsXwsfqOiY", - "05z2nNXqEqwt6nIRNoK660X4s1KfImNUGAviyeGh+idiVALV84OzLCGRBtjBv4U5Muxn3Z1wNksgNb3U", - "x/nuV0XLk8NnbRa8Zeil7f3zdPLsduiprEim16Pb6PUDxblcMk7+hNh0+/Q2un3F+IzEMVDT57Pb6PMt", - "k+gVy6kd5/e30aczMd6TFFhuJ/aH2+j5JaPzhES6y29vB8GvqQROcYLOgK+Ao5+VXWX6vxVQqW5JBOgD", - "xStMEmWpa/1oq6qWX/AZkRxLxs3Zt74y4Wr5ksRoH1H83kWFrf15Osl54tfKpUn4uy40dU1/LDSgOdxR", - "rbzI5fI1nbM2PSnIJbPmllPYQPNUNcsyoNoCmGFBIrXef3v4g+rImBGVnsKmnm2j1W+UEKDywnxqtXIF", - "SXJxSdkVvcg5Wc+ARvlppfmPzbJuxCE+vWeXQNsEw6dMtXCBZc38irGEPUkCdq9rqpv6StOujo+4lzjD", - "M5IQed2mzp1ldXekS3U3/VpC2m4+xnKt5FTI+zw1hyYVLDV68EEnhfWdvGUx/KbKNYdmD2l0G1ND7/qB", - "it6nVA3yPUAvS7whQrZZuEE3opuRup+P0zVzbhljuveyxByOekRU3+yuJdlUN9fAqj19P9CvkppNrSQM", - "Mf0qvSso76dLbTWnUhvssYOcumsNS0qnNq0PuZS6Vom3lhWh7++KcYdKlMtIuwRLUyKVDdzGmoiWmC4g", - "DuxAqwwoy/qGevz27BQixr0aHAvh1d5OW7Q+BLTUdCJl4rsmcwT10mu28NQSZhrtUAXHb8/+l1HoLZsl", - "KzzSf3z60/GLJGFR4XOw/eph1sYhG31zwZESyrifnRnjMnAfWeWnLuYamtbXJxIASuEKEl49iqHMrqX/", - "YKpKRHjiMKSMvmGRuRNoLISVi50WRznLpbv0XcOC6jG2qxUm5sRn4mQ14ybI6xBDdcMhMzIqlXYP3WeE", - "UrW3FuQV3rZUpW2saMtLNhGXHgTAKrO3vwGY94A1iyHxTyssCKP9V9lTXd4nxoL8CXWhC7loBFXTdLIC", - "GtcEMABwLc6OM7bvorYbb6HV3CBDTN/ccNNT5rE0ilZvyljTxNmmuobVf2YLkn0qmojLLUyzkpgAq3Zk", - "jh1zsvJZY1ta+KbZLUBiyPKNXX8Rd4uUYnQDJrTkiA8t+us2eKmQFObajkCjHQMdsS0XAn32WRRBRCCM", - "3J2+EEq5lCszoVgf67am8RfO8szDC98a51Pf/fCrdWIQxJqGzTFshuCZjLLduwJwQUF/gJVEe+CrP26B", - "3go9IX7tCLp/xzy+whwGbTCqCPd9L3Ro61PQDOm307BrdZWAcsNhu7VtdQ12cwwX7PJMS631u0JylYj+", - "eKuR7sGz+74FpOuEdbBvR8B+ffIijjkIj/WOyw+tOZoneBFDxiHC0ruBryvXVwleHJfF9XWdnHtbTnEU", - "+N2Y7BuKhGp2WgypNQBLkO2mQzYKfm0uHCXLPdNbb/+uxKNGRX/w1on3CEhRYAsJadDm4+FxtZcdyAgV", - "EtMINj18dPXL08eUUSIZ71vxN1u892miq1g5TgyO6oW+2X4RRZB5j+nsPcrF8IOeuiNHleWVNrsYHjqq", - "wVnmVQXREqJLkaeBjySJubnpKIC29mYo5pnveHI6AboKaEb4dJHiT/6jLfOV0I6vEvMFSH8Bi5sLHAWt", - "is6jJcajJQjJrZdaF4TeVYpqi4S76Kr+zAuaMVmCI0iByouMJSS6Xnvf6cqfmOL6kJD5z1kyDhc9+JRx", - "wri9pWoz2rknu4WQGO/ekxoMu09vTAOlzLdQrh0YhzG0dfITPvgxrjvr73BNsQqZLGMJW6ydkveu3Ofp", - "JDfBUwPOjRv6QAl0RXwrwmok0IhbRbgqklQXm5aMeAExrZ6cVoVi6mxnh3cPVivYqQLFTWjJ+gozazxq", - "qTyrNotJNNpv/6W79Sm+7pHUHY4b2ZosiFzms/2IpQcsAypW0QFLnx5EjMOBa8jEqtk/trBbiuY8S261", - "9U2tlmK52+LqtErIAJuiSr7PbrHftzFbaoR1sLCf0WL6tK10MeI3nG2qw6oTHm7fTmz7gsO/AjUvEwPj", - "KyRDt9Q5wNKWahhnlQW/VXuRsBlOLuBT5ienUeKC6W20WN/WxXBlOJ0QcbHEF0nhtts2N4hY9znjIICv", - "/NepUxPC0DXeaoGNBlHXsRfwCaJ8aBulLi5Nzi4T8121vDnCazQhLmJ7f9rmScWoaU3qziyAivHeNgFq", - "tnVPW9psAvzipb9sNHtbr+F1ieqQipBoVUHeEIkGfMNg9SAohIga9x1PPRzsBHZD8ur2QK2R0qAo9FJf", - "O8AhaLeGQOjSVrsN9nf9i0L7rTlnfwIdqgZrWiyGOc4TOXk+x4mApqejK4qIQDqwhcxNPLTZI6OlDrOX", - "aAZAkZ0LFOc6qAaf0yVgLmeAJYrZFVUkoYitgEOMZtcIoxQra5oqVqEMOGHx/jnVAThyCZ6vCGgspsZb", - "3BAglixPYjQDlFPrvjI9p5jGqCD9iiSJKiBAKrL0OPd1mgKPBsdCXgiJ+WClWolN7Depig84GVAh42xF", - "lDCZiVvju1oU3aWeLYlp6/KcUsWLYXutCCfg3x1uv9/RMmaFpyoq7VmuTF85Ly21U+V/XQm5sbsBbbQT", - "sbzdjQL69V9nknH42WYn6GtAV6pd++ar9r2l1WbXNjRzjXvPVMeUrTVPTeCZadRnnFpifoVt/CzrjQQ3", - "Do2+tj/vbPfbHoCfS9MNDP/yVKOHs1PV99EG/6nK/UaxDee9iDOBX8fYtwPSEVzqf6oChel1a1imoHcE", - "pv3Nd+xVAn3AqbS/6Z7dtrHNlr1CxoAZqtAenppthK9KVZh5uxK5Chvbbpk2fDe+wP6li4iLoox/p2Nj", - "FHcksp179bKzBmHT+kC8bGgwWayiyXSyYnqtnOtFDNQvuVDWMBXmt0j98zFwF2F/pDgldLH/q5mIDZcx", - "00iZmafLk8UW2NCP5S3IK8Y9Poo6THvgMfycQ8CQCV4U0LL/3uq6w9kwF9DHy7TuIe1oUNXxQjNODcS2", - "1qH5LfNen3hEP1vrxnnSGH/nPavryc1XlzgFb0P4+u3/qecSTadHsSLnsg0ZzyRLTCdvhqnbkqUefBUf", - "t1C3Dbo8Crfey/YHpK256+vM2C0dm8QC9JmwTaarY7J2MFVrJmpX02TFaZOLd1V38KW79p0YeuGuKnVd", - "tqvv9/CivcKg9koTuOCunHxcLDiO4MKcf9S3wmVqSo8jPI6vh1f6NyN0sw5FlhAZvgtuxjnqm8bgKBv0", - "+ylr9Llmm610+NaXfXohcHn0/FG0OjuiL3OW/l0foy2hsFV0Ehrzab+wXXuIwRtVxad5aDCbWZHJzJJQ", - "zbygybhaAjdZNC2t+hBN55rDXCdAI3Shc7Tu+wCQ+XPXmQZ8w5YMCck4XgDS5COBqemvNyvOXrzVqQR9", - "uTeqcLOTUruRNvT2QU0x2TvCzcZbTRck2VoMXKt3FUvrCBiwvjmSfatnAfCgtdDOjllATFXU4PaitDgx", - "qLegf6430Uxs1W1khA8Y9Gi2MAQK1gYmfocmwKBrZt/BUbDh0PXx0BviTS7dbv5S9nYvVB/pfeZdXk72", - "P83XC8bWd4m19SJ4h7iwKU5a04Iz4v+9yEmy8UVQK62JzxBX9bD0o3eDG8uFcS8Jkes571jjyFHaaC3S", - "U0Iv9MXRRQppwD+0KCKucNbjxMVMlJmW+iQUrKpfTy20T16dlFa/9Zt8O6Q+6Nz2nqkGTuGs4P6LmarQ", - "XPUDVpfYkdllsiusC0pvg0EjcjPN5rz/dWHfumho8t9rDHQPuFF3aANM/5G4vXHs7eT2G86285Aefme+", - "AyfoooliCe3Vwpl0bt1hL+ou9+jNr/tv1ul5M+fliwIsF+a5hB6X/v3u9/v4K1sQVyHb9Ekur/19zsgN", - "DNTck+t+Ac5BueaW3Bp9t01RqIfN94oV9eLZOFRa33TPaJrYZtdYEtF/O1Qh3INi83WL3VaVpCDbdrTj", - "qjCwRezAW8Fw88WLAv1Vwbu6di78wyaU6RgBw4ol1scpxl7m0guj2j7nnznkvhNZ3+ZpyLlsazPVZFGz", - "fR+zTnB0iReeM3DMo2V47UsSiNuWLPZbCI0rMFf/RdNIU5X336sWui4PRS3f1NqrkelkBVz0Opt1Jxq2", - "/NTwoLhHqQ7ckNHB0M31l5sRjxRW276r4M0KDf21S5Vwj+DZz1uorxpVYc7tyJ/iBEsjHA1Kw5JR2M6b", - "SIJ+gCeQ9cesvz2wbRqpVKkDOjjMbYCsuOSfDLm8YxDbkQ1BmGOGF8AyWm7oedise92nA48PYnkD5fiM", - "41g7hGO6MNmTUrYCL8+39V+cuv9bb7u6EDnVUHBqttIExdR6oeda34EWaOyYGiaDPmfzvzTkKqLCzHb0", - "6de1jF0xnSQMxwivXGZGgRg3Zx62caH2+vo2kAPWFviSzP0GSWNvFnwDqaDMWftlHlxJUu0lTBndq/x1", - "oAQtpzHM/R3bLWDjgt0l8GzO7FobbRsPpR5bvKVi5KC0jUP2j+scmHq0sWJJnkJ4J9npCbI0MKlxv9Fk", - "bzcoNbEDNaiCgk+3MZZsI/AFIT55d21vv2tRTf1Ls6o7GLA/Lom4YDxbYhqKHwvFt4eOVXpjsZXrUvtA", - "WjevSnR0SeEaJBjGDMeDZWgAFebrltiokhZASKWfXeBESJdCciFeMiq5TwUmsIKkvmYQc+brKIthli+0", - "maZ/vsJcPzCqM75PJ3MssZk0qvOS6zVhLfWm126yz/LZi8ifwrVtY3Bwq1X5L8u8S4HIZx43CZNHs/LK", - "XEL0Q1hlJvbi1Yvl7G9H+/xTrwdIvDaHpiA0eHdUe8LZwp+yiIiLDHNJcNLnfnJDF6vwfWXY+crVCQ1N", - "mctlclr3AkZ7dovkvRsMocz8a0axWcLbOgkdR2hqWOagx2D11OKwNai5e4PLkwN6batnV8S704tBSEKL", - "PMdhlZ8SapXi0RqQVpsMDVg/2/gbCOE9sam8Crnmerz2FqSrFrRwUrEYmgvGnzXXTGmtP9N6pS3v0O17", - "FJ5pkPa6qPlq0jJPMd1TZjGeJYDgU5Zgw1wkMojInERIMhPQyaIo5/rZN+sVdk4z02MtVrLuNpAHnvX5", - "+/v3Jy5CM2IxoK9/P3318r+ePD36OEVn9p2f775BC6DAdczo7Nr0yThZEIqEedhjzniAOuQjrmplEpmA", - "jydiybicNlkj8jTF/LrROFLt7iP0WqKzv7/78Ob4nL599x6ZvaR5rrxCmGRhMqcIPkWQyXOqhpTlPGNC", - "bWXmSDsykD/NrHwN+4v9KcoFoQtVVensFSD7nsk5pbBgkuiy/xcJAORh69P9Z994p6wlatJcoAh3IWx4", - "FsCeAtx1IFZi4F7BPErqtQ3drIUdt9SXo6pIqx+eqL2mO9tRPzztyHPvbAf3JKt7I9V03uXL5diwxWmQ", - "Y2TFBrsTl73qUAZYklUG+MxV+30bY7VGmM9Urfaxg/OL+jVr811Z/dDPFLn7O8Q4Kp7irVz8NU8KbDB9", - "Sj7p/aM+H5A8B59FaJOQD0qVvnA5eDdOot4jVnR97vPuPOblCxomobkh2jcJ3Us6jmO/irlHa/0uAtbV", - "MKdDzIS1t8hVvjqZ7JkYvzIhHkGvfhe+t6TKL3270fS1n2iyDflHVy4lm4SutPPH9Qxf8aSd6RfC0syj", - "8LljVCF3JSIu7OvAcTBPkR1HRwmlzuLZtf87L7cR3rR96uNFrF887xcf0nppphxCg94acdPKxrrebd+k", - "Cg1m7ia5gmt0czOgQK5nfau1vunVUIHQLUyEKiFDntSokO/VHOb7FiZCnbAOFu7oLKtoji0G0/iGLYLX", - "Wa0y4eMxDwiKxbLPWVdZoWuAu8ozt3G4bTP9BCfdMxIMLKhosAGK3B2fNMlyrQ3VOrtNKhUg1hOCJuQg", - "B2sOKSa0fnEZMvHKstOio64ZKszrkBv7ICfbyrFgb3/I5oGhNdTDnrmNRbqt4M3S1d4tLAmVJoSo2CKQ", - "BWUcBMJJYrYISHJMhXZkR+ZEVngTUQGNjHN5vQtCYxJhCaobLBt9CbTENE6K0xSkGxF5ok9YtBu6sMmx", - "DF0xsm0srzO10xGMI60vAtmxiHX2rtN0Cdd7JoAqw4QLsy2K9Tv5VALXJ3rq/80E2xeeI5YkEMlzxQvY", - "uyIxIDxjuTTHPW5MVTrKCUpccJgnlGcxQDE3LL76qCQkiZlMezZP5ohIl29McrJYAEcY2QbsZCKXvOyc", - "VueFMonyLMDVauqwxmyXnHCnaXix4LDQE0qoZOidcVvVG1TAMWJz9GKFSVLuWE3F/XOqX8kViFDkeixb", - "jxn9SiIhWYZwCKgB8gf4KYeUwjqTs2KsthKBWO6YacHJFb4WOhtcNkWwAorwXOp50mMbNrKhL/SK4p33", - "BpSaD53rcnWk6xwmQpAFhRjph9jbh2N4MfDKv1+eBKfPnNIp7tqMnBmpKiWlliytlROtvAazFnxxuli8", - "c6/HEXr+ob6iOu5sHbTCC4NbKXhmtHfpIWR8xmcJji4TIqT7YaFviLTHgEljOJlO/s30pwTwyjwJyPR4", - "/8ixlMC9BrsLU/a4yhFJcI8Np23hdVFew8EFbfSo+d4Ubpm+RYNFe74VsdW9Z12yn1wQ7ZIJiYRS6y6s", - "GwGNM0ao3De46R3Wi9EV40ms14ickj/0QlNpD5EYqCRzAnxfv1HpbkrJH3T/yeHhs72jQ4WK/XyWU5k/", - "Pzx6Dt/N4mf46ezbb5+F71FbYnydFTHCRd/6hqDeq4gE6Rs3HHwDpcnyzfeaPuw0N0ze3u7KLdFHTP/N", - "oXcoHt3YLLfFftRPcA827+gI2zW7CZ86WLMDjqxhxG7H/75QiA251b87yW3knLgXGuqHvaMjraHsurUv", - "+Op5DKsn9Gjf0rtvRrF/NFxf4VvSWNES4jyBLn+Z3o60emvJ82GBwkWlOQncIprEr3kUgRDhUhQ+De/c", - "surCbmwYDx2tmmINq9ljfFbY2dOZuKji8nZWudhkj48Z9aH7xuQfQBcctli43HBu6pB0F29hVIc5QD9W", - "mePTwPb7Niq4RphPB1f72P6QtDwtcR3kaisRsytauu1V/aTV1iCeXSNdzPyvLuy1oPXeIXQjkmG1BYak", - "xxv3RdHeWZmrPe/mHK/+HFDv+awS4oHM+0pYbulMOcckYeYpSq+reyVK1U1bpco8gU/e+fggfA8T3+oL", - "sIqE13pZ9bml4Dx4cY6Ny3goncTQ3XWFpHAuQ5yCyHDA6YXjq4uCrF5rcFnDDajaR5BbG2tiPd0eFVK0", - "eldbBUdAf7VYkOyZUPVtC41bEhNg1U7MXe1hG+WcyGulwVObeh0LEr2woNcEaS2ofi2NlaWUOv/CDDAH", - "7kqbv145I+cf//PemhKmCf212cbnymGw9dmaWKVnzpmRSXZSBJ1Onu4fPdl/Yo47geoMNZOn+4f7h5NK", - "6rgDJbYHrmFrzKt5MB618eT55BeQinCbGMQlAta1nxwe2rt/aTPj4CxLiHGkPfi3MCaoma21eW5cH3qo", - "ddX57lf16+epJVeySxPykDFfruKXHLAEnUyPg8w5RRj94+zdW/Q/MEPvVV1tnkcJUWyLMEW5AISV2a6I", - "YNz6BurHNGLgiFBEpEBzliTsitAF4saTWeyf03P6Xt8I6B8gRpwlYNL3QTqDOIbYtPyV1hpfoSjBJEVk", - "jlIso6VqTNGSC35OXRGbaNp4FNbn4oQJPRl6FOb5FZyCBC4mz3/387cscnCqaJt8njYZluJPSPMUOX+C", - "KUrxJ5LmqUnKhp48W+pDysnzyR856OTOdnmpeCCU81xudI4OU8825+MN48iwJwCk6eSZ6c7XSkHWgSqk", - "yx71KXtkyj7tU/apKvttHxq+NTR826ddVaiqqjQgKkrq949q4quK6PePaiLMGffvZv3+qIXMukAdmG3O", - "AZ45s8srbi/UZ3MvZh7lQLa+vWMytzS1pABabk7BHMnbZJjuVsckJEMmz5iFH6P6Ok8HI4bEwvqb2XS0", - "muQbRJkv0cK9xtuzw+/7lP3elP2hT9kfhuF4C2xaQPnhOecAxnvSj89X+rsGkFH7unYBpnN6wmGlF9Ak", - "Qdb91KFRoBgivecWU+0abzWbKyeQxJegbHfdks6gVXkgySTBQTOYM64WpOvaA0sFhhW+9UHZtZCQTs9p", - "hc4rtZRon3xAKaZ4oRaUErb9xMGwYJSHRyEPOV0nER9siQ6ZOAUhFWaD8qCAr3W9i+m93kRAFK1VEUkA", - "r5xNZELP3WVySHCMsFjJQQMEZ4oEQznFUgJVVprbhCMizilQ7SqJ8AIT2kvEHE9HIXuYQmZcUpyM6Svp", - "sEUUxwgjCldF2u2qkFmXhHLLgDOiC7Z2ExyluZBKTvTOAGJd8SvOmPxKQfsrRcZXZstRVM44i0DocCbb", - "kyrl2jROD9c0WnJGWV5W0/FjjnmqlFBLYvEwYK0Ns1wusTCPEGb5LCFiCWrH8n5JhP1OhMnrDLEe3Y/n", - "+eHh0whn5EL9qf+yQ2Z2a+XCtzron+q9mvq13I2Z7uYkUZuc6TndQ/9ghJ6ZY/ZpsO8pVrsz+6n8GX2t", - "lY+bvGKUurT2o6oqy29cd6+Ne1dHd2oYe5XPwS6v1IYx0fn4Ea51V/SmHYs27AtTpOOxTOic2vIpJpqw", - "hlpvOiL6m4DyMyHb/zCuGY1taDs60ckBjtssDGwsrWNqeU4jeQ7+TSaFqwtbPCX0DdCFkuYnvfedX/4e", - "cQs1pz0GKU68es743AQV3SksiDDrsy5ZaAjJkMn60wAwSiGdaVtgkJ57oxpfr+jqNGyo6eqN3LKqq3Xe", - "T9dp3qxXdmY6fOquruZsOb+i032t13R6FCH1o7uzDpoe7aa7WKfeOjvYpX57Y33O1io4Z7hW29+BYmMx", - "7F1JtmdTaN2Jftu5bknY4iCqpCexqiU4B5VsJoZtIORPLL7emV3t78tjWQuQzjc5YQvkAj3qU/nZPwnd", - "nH7iVpJHsuoYLtZxUTr/hu4hbJ4Y55k67PT7rbuue+ecaRWJayqdgXHDKOvc5Nl1bXwdO7uHNvH57KD0", - "WFqnD8o0QTetDcqePHPhzrWp0wgin5XZhMSoFrZHBxUHcZ5mFY1Qn4LjPM1qW+vjt2foT0aL9B2+kxul", - "Rt6eqao3eVRz/PbsfxmFhyrEVNg5KtxsOrT260oG+GEq+wTL5RBt/ZbFcDua2o3JhtC3Jlm7v7p4IBMb", - "Py0Dl2hsY4Qe2U7TYqUOnYMMy+XBX4U3zeeDvy4JjT+bnz4fZNW8aMG1oZVFbSjWCFVoK4yEPnAzVfTj", - "y71Lqw4sNG9m6WoxwoPOlyadklKdBUgdOG3AltrAzxPtsGZ2qroxtU+1MZTVQL2YxHo/p2ORIN7vu/iN", - "By/l5qivOJRW8nph2NBSfgii0GCBRwgU+5B1yixC5kbYDoRt5dHo0PpvX04W685RqkGb5cnQDEeXQGNU", - "vpvuPVSxmcUKbNymy1H1aeiHafAVT3lX5/yAZD2m/fXJQ593+3b7o5h5m5gkOOf2Qmfgycytme3F860B", - "k10fC4/muijfly2m/SBKAPMOr1v1WZijd4G+rviCTLVvBcTfIELb/n7a3tz3nsHrd9x1r+PZyfD5WufU", - "XX3u8UYFTnS5dT8wpqv16OAvly7wc9CFtg32E2g6r25ktLMYKnb16Ij0AByRemIs5pjQvhg71oVHjI0Y", - "G4Sxnr7WbpH3L+slCgu/5O1g2OfA4Z9q33DqHE7OSHzzhqbV5lEEmbzv4L1PIMtysTzAwiZ2CnkezTmI", - "pbHN1TbROVm6uHn9l24ExUREbAX8Omxlmqk6ycXyhTApkx45Ih8JymIiLrcFmWpjGMaOVa8jxB4HxLLi", - "zc4tMJaZZ0SHwcy8gzni7JHg7HJxNyi7XIwYe/gYExGmB83nM7vBVhz1VauhCEdL2D+nL4vAMaTapsBN", - "2LzJZVtm1I106KTJcED1r6CgWUmkyokJL9MtYtuNaioXxpHZRHMhxpFNvInmgGXOQaAZVmVs/KV5SUi6", - "+DK6sHFl9owy4ClcIuUswuWwCIhRLh6BXFwLDllngPxLo2RL5Wtiw4qa67TsWdHFreHpFePRuLF+aFgd", - "EBnc9wSnEvY6nuGMUPvcMhG8jrunJi2PXnOrtoGNgrLBsA/CQrAXbTs1C24S9CXT1zk1jIA3gC+SCnZd", - "tBbpDG9aS/68wkmOZa+yr9MMuGBUF7/RuxztZecyKY6Q6gWpA4snrwb9BaTSVmBnG2GXNazmY2Eashm6", - "0JwknvW8BtBfbvPs+1dDsRhQZQi6bZVbA7kdzqg2h2DchOWGDdJjSEACEualXTFFORXgtlLSgV4MRn3h", - "XqTLfjBU3BryzaiGAP+DGvaQCme6+I1aCixNiRyt4j5or2dVqLy8EzpB0wWqIRbaANVh85VEMRruOgMo", - "jdGKlU8QCRQzHZFhH8525qnNnAAuql8bwZTpRPn6JSOW81qqJRMNIvRp8TW6IjoLnzynkl/rM2Sb3KlM", - "92TD7e0zWWoU+50R9qfFAzY3YhOPLoLB8MoeQBXLXOoM4UGkni1zqZOIF7nEwpjU6bmoeRaqRLZJwddC", - "ZA2V9fRfGXDC4mkdlZJfn1MvIrFAgjGq/pVLILzysrRNu2dHaQn6SpxTl6RC/dyN3zPHoqEAPnaJU/vH", - "y9zKBtAM64SMan0DeZEs65AVD/A30uJb63AFcOkRlZxKktiMeUX9iwXHEVwYqVNCAZ8ywiFeIxeKFff5", - "oGPE+QY41+mHgptStfUBavBszRZdQQTC8HWRn1c2XcJN295DFO4bkhLZ77gFqHyl0zHdVDoRCZ+kYfye", - "kBxw2h/jmrpxQ9oX43wWH+AkYUafdJ69aDXOZ7E9W0YpoYwjmqczfUpNY5QxLit5/Eyz5UmyteNDxzHH", - "pz8dvyhJudeKtE7qTpB2PzZtCg+t492GwzPIaInmnKUIG8WHDS7ahxBozvEiDWclcdN+a0fFZWe3A5Lx", - "/LcOvGnITrQOWr0BpQprmzFJupxU7h5cN5Pxoj42ez/sg1n5nu4Y5r+9clTrnli7SBpj0BYOaT39+X4v", - "cprE0ZTqhY2eqUz65Iy6lTP52852csM5qcxjW2NOqiE5qdABEqtIJ5kofljphKOVH6L5ov6DgEaVXPAd", - "CIY7Tpox1nFJ8BNjRsXaHDbFA4peA8CBwzg0/WTeUH9QorWhB1n/aoNKm8cjB1R4jxdDSrPb0SWj/9tA", - "hbE76Y/1JfHaq/ENNYCpPeqAG/ciHSVpF0tva6VtrcW7XXoHBLpvIHy3GPc+Ct8ofHe6jBWv1oeF6cQV", - "2VSeigYerUgdG7f1U5YkMxxd3mCozxv9Ju5obo966qHpqTVOeWeFS15DQ6ErIpcIIw4C+Mo8cNtHaZ2e", - "7cTzbVRZowYaNdAD0UC9/Md2p3924KM1qp9R/Yzq5wGon37O3tpResNt2sa+0g9F54yaY9QcD1FzbLhx", - "6qUzxj3SaKSMqmZUNRVVo2rEs+tNjmoIRbY2SoPpbTwa6Mx2OSqiURGNimhURAebHdX00zeP+FRm1Baj", - "tniA2mJg5r0NtMatJuIbnUpGebpjeerhVvKhLLS5VGWP3rVkdBAZ1/BHrXP6PHyIMDVPH6KvzycmeZN5", - "9PB8gipPIRZPIPqf3Q5Fb7rZd48hPoaQqBHV9y4sSWP0YE6Smh271PpeT92nPQ4JlmQFe6op33u80zXP", - "lcMr1fxDhHif4GcWSfCn+ZgznmI5eT6ZEYr1C8hNzgZWgoZsPeuTFe3Z/XlT71mfss/u5Zt61t50Ilz8", - "uWZZSlg43cQZ8BXo1wMSthDhPBJv2OI2Uuq8YYv+uW9UYZYk7Kpn4TeE9kuRqagWN5xJR9PzcB/5XhPQ", - "baC7bnnYFrhuGbgl8N6z3dZtyNJdi8hovd2A9dZPONUsxXkCfd5kcGWRNK8izCvHJ4HkRSbD+5nrZMza", - "0FtsHM9G2dl2P58pOzgEbywuTS4ayZAqqPPVRkkupMuj2ZGW60S1vPv0NF+WLX5PsvRtr8jWpN7bmRIb", - "tca9MWDNKz3Fc79e3PyLwFXxoG8QHLqhnTzge6Ppqoi4HKExBBoLzvJsPTZMsU5w/GKL3F90aApHeAyB", - "xxLz+ApzWI8QV1J0o+TvrsH7DBRH5IiVIVghGY5jDkLsRJ28PnlhW7vPSCmoHKEyBCr2xe31QHEFO6Fy", - "UhS6v0CxNI4wGQYTGS37gMQ8+N8JEVPkPgNERssRHoPgwdWMy+seCHElu0FSlrrHOLFEjlAZAhWB6QGh", - "RBIsGV+Pl7JoJ2DOXrx9XSl5j49NXrxVnRXEjuAZCh7n19GNG4n5AqRYixo1GV8CYEacDMFJLqCHblGl", - "1iDkg7jnaeEVgSM2mtgwN4Wdz0Tp6xdTrni81d7GdN4wDt8FKzAMeW5p21euRzhUnJ9qgGiuHYEpNu48", - "m0zzbUyvdTZ6kG5AoTmruRWIVeR8CmITFNyRotoU0NJ9tWSJeeaQcSRYqq/jiBSIg2A5j0D4AzLOVpFt", - "ZtOVYLiPwNDQo1uIRBhvj4d6XPaGMdBuFP9MdwFi08qI4RHDO8Nw79d0zNJ1e9i7bz5WZvyhd3EezMrt", - "dzWv/Gl8DcvCUCtc+hn2cym0sZh4xurZn9rqz/Dfxq7p4o8XijxagpCGQf/MIb/vUXzPDr/vU/b7Ly4E", - "46blov1+S7dgbPciyygZo2R8KZLRzoXRLRnbPZcySsYoGbsN2xsE9gVZgU6Z1hvuv7gaI+BHwN/jHWfn", - "Cz/dEN/6wZ4R4yPG71CpZzlfDDBgTnTxEeoj1L88qHNopffuBvuprXD/4d4nI8dAp7QALzTaVYeEQzx5", - "LnkOn0eBG+0nf/r8bvk6+0Kka8T2iO1WmuR10N489fGI7BHZt4rsK2KDMnpi25QfraKCFaNRNIrXoIzZ", - "3QK2bQbscQEZEX6H++xASut1mM/Gk9QR9l8i7CNG52Sx3mvtpSl3r9G9vvTPK5zkWPYq+zrNgAtGdfGb", - "d4uzDB5DFu7EJcKIQZFUuY8sbJMh+T5q+10nPZ7WU1InWMi9lMVkTiDe4/Po6dOnP1BMWa2HGEvYkyQ1", - "nJUSuGrt/52fx389+7yn/nni/nlv/nle++fr8/N99X9H0x8+f/Pf//vf/+EndhSm4cI0XWsMfVFyMeYB", - "H/OAbyEKuU8S8lEQRkF4TIIw2MCyhpU3IPkXkIhxBNZGRxhdwvUV47GLTLadm6ZyI3hImWz74UhlI4q/", - "gPzSty42gutXwxIxoMqQTY+tcmt7HzucMVr7ziUzz5Tt3RFErOMiVGfqBzFFORUgEaYxUv9aURUbyGrT", - "gPxgKHkY8mrYNkRcPyi+DqlwpouPUZ33VcAuV0Iy3uNc4dd/nemCD2alEje8eBh+/UwlJ6CzQTxKLPfc", - "sbjkhQ3lq37+guB3U3fjig1tPK2/GP/S9i0P4crjRtTzAVDJr43d4yJG66JilvKarPys6zwYfT1aETeh", - "eXst+o8ASTd2+TBib+Mj+weNvls42XxY5sG9RHDnSfuI3xG/9xm/w83QS7Vt7ntUoPfYj9bjrGSCOz8e", - "cdiBw4yxpAtZJ4wlHjTVj4H1y23YZNzDaIajS6AxUtDFC0C6i+mEqJJ/KGU2mU5U6clz88+0MrNNbXSj", - "WeoZSx7ww8Ga7eUkH6xYkqewbq7/pUs94Bk3A3wk857PEhIdsAwozkjX1J9d4cVCJ/Teivl2Mm2y2vvN", - "34JfmkmWYxwSfH2QghD1p39aDDtVBX+z5YYutrryW5uavc/iqSu8NDm4Xx/f7AJaHdlDzaasp3nNXrgx", - "wzcVfFXrxsNtRSDC5n3cGEssQKI5ZynCSI8CLQFzOQMsJz0jttZZ7oePynJ3UDDSbzIFdwu+zSa8Zbb7", - "9UKvFMSQ8qckvp1k+o4FoUV0AbJIumwv+6coZZRIxo1vgMQyF48MZhZaBmlXS4bTzhXZlrjhBzJex0Cl", - "Gs4O9Pxg7qi92f8PAAD//4TWSrRJzQEA", + "OrrvNLIandRbPQMZnEMBctgksgyMP8aZPxCrnf55fnj4NLq80v/C7+ZPQmP4ZH75aH5hmfnT/KVVl/nB", + "qHvEMpSQS0A/ov/zI9r7sQ0UwPLHOc+JFEOgcpbP1EBDPDBf163vuqX3eBFqRuJFzzZYsAnWr4UPVHTM", + "aU57zmp1CdYWdbkIG0Hd9SL8WalPkTEqjAXx5PBQ/RMxKoHq+cFZlpBIA+zg38K4DPtZdyeczRJITS/1", + "cb77VdHy5PBZmwVvGXppe/88nTy7HXoqK5Lp9eg2ev1AcS6XjJM/ITbdPr2Nbl8xPiNxDNT0+ew2+nzL", + "JHrFcmrH+f1t9OlMjPckBZbbif3hNnp+yeg8IZHu8tvbQfBrKoFTnKAz4Cvg6GdlV5n+bwVUqlsSAfpA", + "8QqTRFnqWj/aqqrlF3xGJMeSceP71kcmXC1fkhjtI4rfu6iwtT9PJzlP/Fq5NAl/14WmrumPhQY0zh3V", + "yotcLl/TOWvTk4JcMmtuOYUNNE9VsywDqi2AGRYkUuv9t4c/qI6MGVHpKWzq2TZa/UYJASovzKdWK1eQ", + "JBeXlF3Ri5yT9QxolJ9Wmv/YLOtGHOLTe3YJtE0wfMpUCxdY1syvGEvYkyRg97qmuqmvNO3q+Ih7iTM8", + "IwmR123qnC+ruyNdqrvp1xLSdvMxlmslp0Le56lxmlSw1OjBB50U1nfylsXwmyrXHJp10ug2pobe9QMV", + "vb1UDfI9QC9LvCFCtlm4QTeim5G6n4/TNXNuGWO697LEOEc9IqpPdteSbKqbY2DVnj4f6FdJzaZWEoaY", + "fpXeFZT306W2mlOpDfbYQU7dsYYlpVOb1odcSl2rxFvLitD3d8W4QyXKZaRdgqUpkcoGbmNNREtMFxAH", + "dqBVBpRlfUM9fnt2ChHjXg2OhfBqb6ctWh8CWmo6kTLxHZM5gnrpNVt4agkzjXaoguO3Z//LKPSWzZIV", + "Huk/Pv3p+EWSsKiIOdh+9TBr45CNvjngSAll3M/OjHEZOI+s8lMXcw1N6+sTCQClCAUJrx7FUGbX0u+Y", + "qhIRnjgMKaNvWGTOBBoLYeVgp8VRznLpDn3XsKDqxna1wsSc+EycrGbcBHkdYqhuOGRGRqXS7qH7jFCq", + "9taCvMLblqq0jRVteckm4tKDAFhl9vQ3APMesGYxJP5phQVhtP8qe6rL+8RYkD+hLnShEI2gappOVkDj", + "mgAGAK7F2XHG9l3UduMttJobZIjpmxtueso8lkbR6k0Za5o421TXsPrPbEGyT0UTcbmFaVYSE2DVjsyx", + "Y05WPmtsSwvfNLsFSAxZvrHrL+JukVKMbsCElhzxoUV/3QYvFZLCXNsRaHRgoCO2FUKgfZ9FEUQEwsid", + "6QuhlEu5MhOKtVu3NY2/cJZnHl741jif+u6HX60TgyDWNGyOYTMEz2SU7d4VgAsK+gOsJNoDX/1xC/RW", + "6Anxa0fQ/Tvm8RXmMGiDUUW473uhQ1ufgmZIv52GXaurBJQbDtutbatrsJtjuGCXZ1pqrd8VkqtE9Mdb", + "jXQPnt33LSBdJ6yDfTsC9uuTF3HMQXisd1x+aM3RPMGLGDIOEZbeDXxdub5K8OK4LK6P6+Tc23KKo8Dv", + "xmTfUCRUs9NiSK0BWIJsNx2yUfBrc+EoWe6Z3nr7dyUeNSr6g7dOvEdAigJbSEiDNh8Pj6u97EBGqJCY", + "RrCp89HVL72PKaNEMt634m+2eG9voqtYcScGR/VCn2y/iCLIvG46e45yMdzRUw/kqLK80mYXw0OuGpxl", + "XlUQLSG6FHka+EiSmJuTjgJoa0+GYp753JPTCdBVQDPCp4sUf/K7tsxXQju+SswXIP0FLG4ucBS0Kjpd", + "S4xHSxCS2yi1Lgi9qxTVFgl3t6v6My9oxmQJjiAFKi8ylpDoeu15pyt/YoprJyHz+1kyDhc9+JRxwrg9", + "pWoz2oUnu4WQmOjekxoMu703poFS5lso1wGMwxja8vyEHT8mdGf9Ga4pViGTZSxhi7VT8t6V+zyd5Oby", + "1AC/cUMfKIGuiG9FWI0EGnGrCFdFkupi05IRLyCmVc9pVSimznZ2ePdgtYKdKlDchJasrzCzxqOWyrNq", + "s5hEo/32X7pTn+LrHkmdc9zI1mRB5DKf7UcsPWAZULGKDlj69CBiHA5cQ+aumv1jC7ulaM6z5FZb39Rq", + "KZa7LY5Oq4QMsCmq5PvsFvt9G7OlRlgHC/sZLaZP20oXI37D2aY6rDrh4fbtxLYPOPwrUPMwMTC+QjJ0", + "S50DLG2phnFWWfBbtRcJm+HkAj5lfnIaJS6Y3kaL9W1dDFeG0wkRF0t8kRRhu21zg4h1nzMOAvjKf5w6", + "NVcYusZbLbDRIOo69gI+QZQPbaPUxaXJ2WVivquWNy68RhPiIrbnp22eVIya1qTuzAKoGO9tE6BmW/e0", + "pc0mwC9e+stGs7f1Gl6XqA6pCIlWFeQNkWjANwxWD4JCiKhx3/HUw8FOYDckr24P1BopDYpCL/W1AxyC", + "dmsIhA5tddhg/9C/KLTfmnP2J9CharCmxWKY4zyRk+dznAhoRjq6oogIpC+2kLm5D232yGipr9lLNAOg", + "yM4FinN9qQaf0yVgLmeAJYrZFVUkoYitgEOMZtcIoxQra5oqVqEMOGHx/jnVF3DkEjxfEdBYTE20uCFA", + "LFmexGgGKKc2fGV6TjGNUUH6FUkSVUCAVGTpce7rNAUeDY6FvBAS88FKtXI3sd+kKj7gZECFjLMVUcJk", + "Jm5N7GpRdJd6tiSmrctzShUvhu21IpyAf3e4/X5Hy5gVnqqotGe5Mn3lvLTUTpX/dSXkxu4GtNFOxPJ2", + "Nwro13+dScbhZ5udoK8BXal27Zuv2veWVptd26uZa8J7pvpO2Vrz1Fw8M436jFNLzK+wTZxlvZHgxqHR", + "1/b+zna/7QH4uTTdwPAvvRo9gp2qsY/28p+q3G8U23Deizhz8esY+3ZA+gaX+p+qQGF63RqWKegdgWl/", + "8x17lUAfcCrtb7pnt21ss2WvkDFghiq0h6dmG+GrUhVm3q5ErsLGdlimvb4bX2D/0kXERVHGv9OxdxR3", + "JLKde/WyswZh0/pAvGxoMFmsosl0smJ6rZzrRQzUL7lQ1jAV5rdI/fMxcBZhf6Q4JXSx/6uZiA2XMdNI", + "mZmnK5LFFtgwjuUtyCvGPTGK+pr2QDf8nEPAkAkeFNCy/97quiPYMBfQJ8q0HiHtaFDV8UIzTg3Ettah", + "+S3zXp94RD9bG8Z50hh/5zmr68nNV5c4BU9D+Prt/6nnEE2nR7Ei57INmcgkS0wnb4ap25KlHnwVH7dQ", + "tw26PAq33sv2DtLW3PUNZuyWjk3uAvSZsE2mq2OydjBVayZqV9NkxWmTg3dVd/Chu46dGHrgrip1Hbar", + "7/fwoL3CoPZKEzjgrng+LhYcR3Bh/B/1rXCZmtITCI/j6+GV/s0I3axDkSVEhs+Cm/cc9UljcJQN+v2U", + "Nfpcs81WOnzrwz69ELg8ev5btDo7oi9zlv5du9GWUNgqOgmN+bRf2K49xOCNquLTPDSYzazIZGZJqGZe", + "0GRcLYGbLJqWVu1E07nmMNcJ0Ahd6Byt+z4AZP7cdaYB37AlQ0IyjheANPlIYGr6682KsxdvdSpBX+6N", + "KtzspNROpA29fVBTTPaOcLPxVtNdkmwtBq7Vu7pL6wgYsL45kn2rZwHwoLXQzo5ZQExV1OD2orTwGNRb", + "0D/Xm2gmtuo2MsIOBj2aLQyBgrWBid+hCTDomNnnOAo2HDo+HnpCvMmh280fyt7ugeojPc+8y8PJ/t58", + "vWBsfZZYWy+CZ4gLm+KkNS04I/7fi5wkGx8EtdKa+AxxVQ9LP3o3OLFcmPCSELkef8eaQI7SRmuRnhJ6", + "oQ+OLlJIA/GhRRFxhbMeHhczUWZa6pNQsKp+PLXQMXl1Ulr91k/y7ZD6oHPbc6YaOIWzgvsvZqpCc9UP", + "WF1iR2aXya6w7lJ6GwwakZtpNhf9rwv71kVDk/9cY2B4wI2GQxtg+l3i9sSxd5DbbzjbLkJ6+Jn5DoKg", + "iyaKJbRXC2fShXWHo6i7wqM3P+6/2aDnzYKXLwqwXJjnEnoc+vc73+8Tr2xBXIVsMya5PPb3BSM3MFAL", + "T67HBbgA5VpYcmv03TZFoR423ytW1Itn41BpfdM9o2lim11jSUT/7VCFcA+KzdctdltVkoJs29GOq8LA", + "FrEDTwXDzRcvCvRXBe/q2rmID5tQpu8IGFYssXanGHuZSy+Mavucf+aQ+zyyvs3TEL9sazPVZFGzfR+z", + "TnB0iRceHzjm0TK89iUJxG1LFvsthMYRmKv/ommkqcr771ULXYeHopZvau3RyHSyAi56+WadR8OWnxoe", + "FOco1YEbMjoYurn+cjPikcJq23d1ebNCQ3/tUiXcI3j28xbqq0ZVmHM7iqc4wdIIR4PSsGQUtvMmkqAf", + "4Alk/THrbw9sm0YqVeqADg5zGyArLvknQy7vGMR2ZEMQ5pjhBbCMlhtGHjbrXvfpwBODWJ5AOT7jONYB", + "4ZguTPaklK3Ay/Nt4xen7v/W267uipxqKDg1W2mCYmq90HOt70ALNHZMDZNB+9n8Lw25iqgwsx19+nUt", + "Y1dMJwnDMcIrl5lRIMaNz8M2LtReX58GcsDaAl+Sud8gaezNgm8gFZQ5a7/MgytJqqOEKaN7lb8OlKDl", + "NIa5v2O7BWwcsLsEns2ZXWujbROh1GOLt1SMHJS2ccj+cV0AU482VizJUwjvJDsjQZYGJjXuN5rsHQal", + "JnagBlVQ8Ok2xpJtBL4gxCfvru3tdy2qqX9pVnVfBuyPSyIuGM+WmIbuj4Xut4fcKr2x2Mp1qWMgbZhX", + "5XZ0SeEaJBjGDMeDZWgAFebrltiokhZASKWfXeBESJdCciFeMiq5TwUmsIKkvmYQ4/N1lMUwyxfaTNM/", + "X2GuHxjVGd+nkzmW2Ewa1XnJ9ZqwlnrTazfZZ/nsReRP4dq2MTi41ar8l2XepUDkM0+YhMmjWXllLiH6", + "IawyE3vx6sVy9rejff6p1wMkXptDUxAavHPVnnC28KcsIuIiw1wSnPQ5n9wwxCp8XhkOvnJ1QkNT5nKZ", + "nNa9gNGe3SJ57wZDKDP/mlFslvC2TkKHC00Nyzh6DFZPLQ5bg5q7N7g8OaDXtnp2Rbw7vRiEJLTIcxxW", + "+SmhVikerQFptcnQgPWzjb+BEF6PTeVVyDXH47W3IF21oIWTisXQXDD+rLlmSmv9mdYrbXmHbt+j8EyD", + "tMdFzVeTlnmK6Z4yi/EsAQSfsgQb5iKRQUTmJEKSmQudLIpyrp99s1Fh5zQzPdbuStbDBvLAsz5/f//+", + "xN3QjFgM6OvfT1+9/K8nT48+TtGZfefnu2/QAihwfWd0dm36ZJwsCEXCPOwxZzxAHfIRV7UyiUzAxxOx", + "ZFxOm6wReZpift1oHKl29xF6LdHZ3999eHN8Tt++e4/MXtI8V14hTLIwmVMEnyLI5DlVQ8pynjGhtjJz", + "pAMZyJ9mVr6G/cX+FOWC0IWqqnT2CpB9z+ScUlgwSXTZ/4sEAPKw9en+s2+8U9YSNWkOUIQ7EDY8C2BP", + "Ae46cFdi4F7BPErqtQ3drIUDt9SXo6pIqx+eqL2m8+2oH5525Ll3toN7ktW9kWo674rlcmzYwhvkGFmx", + "we4kZK86lAGWZJUBPnPVft/GWK0R5jNVq33swH9RP2ZtviurH/qZInd+hxhHxVO8lYO/pqfAXqZPySe9", + "f9T+Aclz8FmENgn5oFTpC5eDd+Mk6j3uiq7Pfd6dx7x8QcMkNDdE+yahe0nHcexXMfdord/FhXU1zOkQ", + "M2HtKXKVr04meybGr0yIR9Cr34XvLanyS99uNH3tJ5psQ/7RlUvJJldX2vnjel5f8aSd6XeFpZlH4XPH", + "qELhSkRc2NeB42CeIjuOjhJKncWza/93Xm4jvGn71MeLWL943u9+SOulmXIIDXprxE0rG+t6t32TKjSY", + "uZvkCq7Rzc2AArme9a3W+qZHQwVCtzARqoQMeVKjQr5Xc5jvW5gIdcI6WLgjX1bRHFsMpvENWwSPs1pl", + "wu4xDwiKxbKPr6us0DXAXeWZ2/i6bTP9BCfdMxK8WFDRYAMUuXOfNMlyrQ3VOrtNKhUg1nMFTchBAdYc", + "Ukxo/eAyZOKVZadFR10zVJjXoTD2QUG2Fbdg73jIpsPQGurhyNzGIt1W8Gbpau8WloRKc4Wo2CKQBWUc", + "BMJJYrYISHJMhQ5kR8YjK7yJqIBGJri83gWhMYmwBNUNlo2+BFpiGieFNwXpRkSeaA+LDkMXNjmWoStG", + "to3ldaZ2OoJxpPVFIDsWscHedZou4XrPXKDKMOHCbIti/U4+lcC1R0/9v5lg+8JzxJIEInmueAF7VyQG", + "hGcsl8bd48ZUpaOcoMRdDvNc5VkMUMwNi68+KglJYibT+ubJHBHp8o1JThYL4Agj24CdTOSSl53T6rxQ", + "JlGeBbhaTR3WmO2SE86bhhcLDgs9oYRKht6ZsFW9QQUcIzZHL1aYJOWO1VTcP6f6lVyBCEWux7L1mNGv", + "JBKSZQiHgBogf0CcckgprDM5K8ZqKxGI5Y6ZFpxc4Wuhs8FlUwQroAjPpZ4nPbZhIxv6Qq8o3nlvQKn5", + "0LkuV0e6zmEiBFlQiJF+iL3tHMOLgUf+/fIkOH3mlE5x1mbkzEhVKSm1ZGmtnGjlMZi14AvvYvHOvR5H", + "6PmH+orquLP1pRVeGNxKwTOjvcsIIRMzPktwdJkQId0PC31CpCMGTBrDyXTyb6Y/JYBX5klApsf7R46l", + "BO412N01ZU+oHJEE99hw2hZeF+U1HNyljR4135vCLdO3aLBoz7citrr3rEv2k7tEu2RCIqHUurvWjYDG", + "GSNU7hvc9L7Wi9EV40ms14ickj/0QlNpD5EYqCRzAnxfv1HpTkrJH3T/yeHhs72jQ4WK/XyWU5k/Pzx6", + "Dt/N4mf46ezbb5+Fz1FbYnydFXeEi771CUG9VxEJ0vfecPANlCbLN99r+rDT3DB5e7ursEQfMf03h96h", + "eHRjs9wW+1E/wT3YvCMXtmt2Ez51sGYHHFnDiN2O/32hEBtyq393ktvIOXEvNNQPe0dHWkPZdWtf8NXz", + "GFZP6NG+pXffjGL/aLi+wreksaIlxHkCXfEyvQNp9daS58MuCheV5iRwimgSv+ZRBEKES1H4NLxzy6oL", + "u7FhPORaNcUaVrPH+Kyws2cwcVHF5e2scrHJHh8z6kP3jck/gC44bLFwueHclJN0F29hVIc5QD9WmePT", + "wPb7Niq4RphPB1f72N5JWnpLXAe52krE7IqWYXvVOGm1NYhn10gXM/+rC3staL13CJ2IZFhtgSHp8cZ9", + "UbR3VuZqz7vx49WfA+o9n1VCPJB5X7mWWwZTzjFJmHmK0hvqXrml6qatUmWewCfvfHwQvoeJb/UFWEXC", + "a72s+sJScB48OMcmZDyUTmLo7rpCUjiXIU5BZDgQ9MLx1UVBVq81uKzhBlTtI8itjTWxnm6PCilavaut", + "giOgv1osSPZMqPq2hcYtiQmwaifmro6wjXJO5LXS4KlNvY4FiV5Y0GuCtBZUv5bGylJKnX9hBpgDd6XN", + "X6+ckfOP/3lvTQnThP7abONzxRlsY7YmVukZPzMyyU6KS6eTp/tHT/afGHcnUJ2hZvJ0/3D/cFJJHXeg", + "xPbANWyNeTUPJqI2njyf/AJSEW4Tg7hEwLr2k8NDe/YvbWYcnGUJMYG0B/8WxgQ1s7U2z43rQw+1rjrf", + "/ap+/Ty15Ep2aa48ZMyXq/glByxBJ9PjIHNOEUb/OHv3Fv0PzNB7VVeb51FCFNsiTFEuAGFltisiGLex", + "gfoxjRg4IhQRKdCcJQm7InSBuIlkFvvn9Jy+1ycC+geIEWcJmPR9kM4gjiE2LX+ltcZXKEowSRGZoxTL", + "aKkaU7Tkgp9TV8QmmjYRhfW5OGFCT4YehXl+BacggYvJ89/9/C2LHJwq2iafp02GpfgT0jxFLp5gilL8", + "iaR5apKyoSfPltpJOXk++SMHndzZLi+VCIRynsuNztFh6tnmfLxhHBn2BIA0nTwz3flaKcg6UIV02aM+", + "ZY9M2ad9yj5VZb/tQ8O3hoZv+7SrClVVlQZERUn9/lFNfFUR/f5RTYTxcf9u1u+PWshsCNSB2eYc4Jkz", + "u7zi9kJ9Nudi5lEOZOvbMyZzSlNLCqDl5hSMS94mw3SnOiYhGTJ5xiz8GNXHefoyYkgsbLyZTUerSb5B", + "lPkSLdxrvD07/L5P2e9N2R/6lP1hGI63wKYFlB+ecw5goif9+Hylv2sAGbWvaxdgOqcnHFZ6AU0SZMNP", + "HRoFiiHSe24x1aHxVrO5cgJJfAnKdtct6QxalQeSTBIcNIM542pBuq49sFRgWOFbO8quhYR0ek4rdF6p", + "pUTH5ANKMcULtaCUsO0nDoYFozw8CnnI6TqJ+GBLdMjEKQipMBuUBwV8revdnd7rTQRE0VoVkQTwytlE", + "5uq5O0wOCY4RFis5aIDgTJFgKKdYSqDKSnObcETEOQWqQyURXmBCe4mY4+koZA9TyExIipMxfSQdtoji", + "GGFE4apIu10VMhuSUG4ZcEZ0wdZugqM0F1LJid4ZQKwrfsUZk18paH+lyPjKbDmKyhlnEQh9ncn2pEq5", + "Nk3QwzWNlpxRlpfV9P0xxzxVSqglsXgYsNaGWS6XWJhHCLN8lhCxBLVjeb8kwn4nwuR1hliP7sfz/PDw", + "aYQzcqH+1H/ZITO7tXLXtzron+q9mvq13I2Z7uYkUZuc6TndQ/9ghJ4ZN/s02PcUq92Z/VT+jL7WysdN", + "XjFKXVrHUVWV5Teuu9cmvKujOzWMvcrnYJdXasOY6Hz8CNe6K3rTgUUb9oUp0vexzNU5teVTTDTXGmq9", + "6RvR3wSUn7my/Q8TmtHYhrZvJzo5wHGbhYGNpQ1MLf00kufg32RSuLqwxVNC3wBdKGl+0nvf+eXvEbdQ", + "czpikOLEq+dMzE1Q0Z3CggizPuuShYaQDJmsPw0AoxTSmbYFBum5N6rx9YquTsOGmq7eyC2rulrn/XSd", + "5s16ZWemw6fu6mrOlvMrOt3Xek2nRxFSP7o7G6Dp0W66i3XqrbODXeq3NzbmbK2Cc4Zrtf0dKDYWw96V", + "ZHs2hdad6Led65aELQ6iSnoSq1qCc1DJZmLYBkL+xOLrndnV/r48lrUA6WKTE7ZA7qJHfSo/+yehm9NP", + "3ErySFYdw8U6Lsrg39A5hM0T4yJTh3m/37rjuncumFaRuKbSGZgwjLLOTfqua+Pr2Nk9tInPZwdlxNI6", + "fVCmCbppbVD25JkL59emTiOIfFZmExKjWtgeHVQcxHmaVTRCfQqO8zSrba2P356hPxkt0nf4PDdKjbw9", + "U1Vv0lVz/PbsfxmFhyrEVNg5KsJsOrT260oG+GEq+wTL5RBt/ZbFcDua2o3JXqFvTbIOf3X3gczd+Gl5", + "cYnG9o7QI9tpWqzUoXOQYbk8+KuIpvl88NclofFn89Png6yaFy24NrSyqA3FGqEKbYWR0Adupop+fLl3", + "adWBhebNLF0tRnjQ+dKkU1KqswCpA6e9sKU28PNEB6yZnapuTO1T7R3K6kW9mMR6P6fvIkG833fxGx0v", + "5eaorziUVvJ6YdjQUn4IotBggUcIFPuQDcosrsyNsB0I28qj0aH1376cLNb5UaqXNkvP0AxHl0BjVL6b", + "7nWq2MxiBTZuM+So+jT0wzT4iqe8q3N+QLIe0/765KHPu327/VHMvE1MEpxze6Az0DNza2Z78XxrwGTX", + "buHRXBfl+7LFtB9ECWDeEXWrPgvjehfo60osyFTHVkD8DSK0He+n7c19rw9ev+Ouex19J8Pna11Qd/W5", + "xxsVONEV1v3AmK7Wo4O/XLrAz8EQ2jbYT6AZvLqR0c5iqNjVYyDSAwhE6omxmGNC+2LsWBceMTZibBDG", + "esZau0Xev6yXKCzikreDYR+Hwz/VvuHUBZyckfjmDU2rzaMIMnnfwXufQJblYnmAhU3sFIo8mnMQS2Ob", + "q22iC7J09+b1X7oRFBMRsRXw67CVaabqJBfLF8KkTHrkiHwkKIuJuNwWZKqNYRg7Vr2OEHscEMuKNzu3", + "wFhmnhEdBjPzDuaIs0eCs8vF3aDscjFi7OFjTESYHjSfz+wGW+Hqq1ZDEY6WsH9OXxYXx5BqmwI31+ZN", + "Ltsyo26kr06aDAdU/woKmpVEqpyY62W6RWy7UU3lwgQym9tciHFkE2+iOWCZcxBohlUZe//SvCQk3f0y", + "urD3yqyPMhApXCLlLMLlsAiIUS4egVxcCw5Z5wX5l0bJlsrX3A0raq7TsmdFF7eGp1eMR+PG+qFhdcDN", + "4L4enMq119GHM0Ltc8tE8Abunpq0PHrNrdoG9haUvQz7ICwEe9C2U7PgJkFfMn1dUMMIeAP4Iqlg10Fr", + "kc7wprXkzyuc5Fj2Kvs6zYALRnXxGz3L0VF2LpPiCKlekDqwePJq0F9AKm0FdrYRdlnDajEWpiGboQvN", + "SeJZz2sA/eU2fd+/GorFgCpD0G2r3BrI7XBGtTkE4+ZabtggPYYEJCBhXtoVU5RTAW4rJR3oxWDUF+FF", + "uuwHQ8WtId+MagjwP6hhD6lwpovfqKXA0pTI0Srug/Z6VoXKyzshD5ouUL1ioQ1QfW2+kihGw11nAKUx", + "WrHyCSKBYqZvZNiHs515ajMngLvVr41gynSifP2SEct5LdWSuQ0itLf4Gl0RnYVPnlPJr7UP2SZ3KtM9", + "2ev29pksNYr9zhv2p8UDNjdiE48hgsHrlT2AKpa51BnCg0g9W+ZSJxEvcomFManTc1HzLFSJbJOCr4XI", + "Girr6b8y4ITF0zoqJb8+p15EYoEEY1T9K5dAeOVlaZt2z47SEvSVOKcuSYX6uRu/Z45FQwF87BKn9r8v", + "cysbQDOsEzKq9Q3kRbKsQ1Y8wN9Ii2+twxXApUdUcipJYjPmFfUvFhxHcGGkTgkFfMoIh3iNXChW3GdH", + "x4jzDXCu0w8FN6Vq6wPU4NmaLbqCCFzD10V+Xtl0CTdtew9RuG9ISmQ/dwtQ+UqnY7qpdCISPknD+D0h", + "OeC0P8Y1deOGtC/G+Sw+wEnCjD7p9L1oNc5nsfUto5RQxhHN05n2UtMYZYzLSh4/02zpSbZ2fMgdc3z6", + "0/GLkpR7rUjrpO4Eafdj06bw0HLvNgKeQUZLNOcsRdgoPmxw0XZCoDnHizSclcRN+625isvObgcko/+3", + "DrxpyE60AVq9AaUKa5sxSbqCVO4eXDeT8aI+Nns+7INZ+Z7ueM1/e+Wo1j2xdpE0xqAtHNJ6+vP9XuQ0", + "iaMp1QsbPVOZ9MkZdSs++dvOdnLDOanMY1tjTqohOanQARKrSCeZKH5Y6YSjlR+i+aL+g4BGlVzwHQiG", + "cyfNGOs4JPiJMaNibQ6b4gFFrwHgwGECmn4yb6g/KNHaMIKsf7VBpc3jkQMqvMeLIaXZ7eiSMf5toMLY", + "nfTH+pB47dH4hhrA1B51wI1HkY6StIult7XSttbi3S69Ay66byB8t3jvfRS+UfjudBkrXq0PC9OJK7Kp", + "PBUNPFqROjZh66csSWY4urzBqz5v9Ju4o7k96qmHpqfWBOWdFSF5DQ2FrohcIow4COAr88BtH6V1eraT", + "yLdRZY0aaNRAD0QD9Yof253+2UGM1qh+RvUzqp8HoH76BXvrQOkNt2kbx0o/FJ0zao5RczxEzbHhxqmX", + "zhj3SKORMqqaUdVUVI2qEc+uN3HVEIpsbZQG09t4NNCZ7XJURKMiGhXRqIgONnPV9NM3j9grM2qLUVs8", + "QG0xMPPeBlrjVhPxjUElozzdsTz1CCv5UBbaXKqyRx9aMgaIjGv4o9Y5fR4+RJiapw/R1+cTk7zJPHp4", + "PkGVpxCLJxD9z26Hbm+62XePIT6GK1Ejqu/dtSSN0YM5SWp27FLrez11n/Y4JFiSFeyppnzv8U7XPFcO", + "r1TzDxHifS4/s0iCP83HnPEUy8nzyYxQrF9AbnI2sBI0ZOtZn6xoz+7Pm3rP+pR9di/f1LP2phPh4s81", + "y1LCwukmzoCvQL8ekLCFCOeReMMWt5FS5w1b9M99owqzJGFXPQu/IbRfikxFtbjhTDqanof7yPeaC90G", + "uuuWh22B65aBWwLvPdtt3YYs3bWIjNbbDVhv/YRTzVKcJ9DnTQZXFknzKsK84j4JJC8yGd7PXCdj1obe", + "YuN4NsrOtvv5TNnBIXhjcWly0UiGVEGdrzZKciFdHs2OtFwnquXdp6f5smzxe5Klb3tFtib13s6U2Kg1", + "7o0Ba17pKZ779eLmXwSuigd9g+DQDe3kAd8bTVdFxOUIjSHQWHCWZ+uxYYp1guMXW+T+okNTOMJjCDyW", + "mMdXmMN6hLiSohslf3cN3megOCJHrAzBCslwHHMQYifq5PXJC9vafUZKQeUIlSFQsS9urweKK9gJlZOi", + "0P0FiqVxhMkwmMho2Qck5sH/ToiYIvcZIDJajvAYBA+uZlxe90CIK9kNkrLUPcaJJXKEyhCoCEwPCCWS", + "YMn4eryURTsBc/bi7etKyXvsNnnxVnVWEDuCZyh4XFxHN24k5guQYi1q1GR8CYAZcTIEJ7mAHrpFlVqD", + "kA/inqeFVwSO2Ghiw5wUdj4TpY9fTLni8VZ7GtN5wjh8F6zAMOS5pW1fuR7hUAl+qgGiuXYEptiE82wy", + "zbcxvTbY6EGGAYXmrBZWIFaRiymIzaXgjhTVpoCW7qslS8wzh4wjwVJ9HEekQBwEy3kEwn8h42wV2WY2", + "XQmGxwgMvXp0CzcRxtPjoRGXvWEMtBvFP9NdgNi0MmJ4xPDOMNz7NR2zdN0e9u5bjJUZf+hdnAezcvtD", + "zSt/mljDsjDUCpdxhv1CCu1dTDxj9exPbfVn+G/vrunijxeKPFqCkIZB/8whv++3+J4dft+n7Pdf3BWM", + "m5aL9vst3YKx3Ysso2SMkvGlSEY7F0a3ZGz3XMooGaNk7Pba3iCwL8gKdMq03nD/xdUYAT8C/h7vODtf", + "+OmG+NYP9owYHzF+h0o9y/ligAFzoouPUB+h/uVBnUMrvXc32E9thfsP9z4ZOQYGpQV4odGuOiQc4slz", + "yXP4PArcaD/50+d3y9fZFyJdI7ZHbLfSJK+D9uapj0dkj8i+VWRfEXspoye2TfnRKipYMRpFo3gNypjd", + "LWDbZsAeF5AR4Xe4zw6ktF6H+Wz0pI6w/xJhHzE6J4v1UWsvTbl7je71pX9e4STHslfZ12kGXDCqi998", + "WJxl8Hhl4U5CIowYFEmV+8jCNhmS76O233XS42k9JXWChdxLWUzmBOI9Po+ePn36A8WU1XqIsYQ9SVLD", + "WSmBq9b+3/l5/Nezz3vqnyfun/fmn+e1f74+P99X/3c0/eHzN//9v//9H35iR2EaLkzTtcbQFyUXYx7w", + "MQ/4FqKQ+yQhHwVhFITHJAiDDSxrWHkvJP8CEjGOwNroCKNLuL5iPHY3k23npqncCB5SJtt++KayEcVf", + "QH7pWxd7g+tXwxIxoMqQTY+tcmt7Hzuc8bb2nUtmninbu+MSsb4XoTpTP4gpyqkAiTCNkfrXiqrYQFab", + "BuQHQ8nDkFfDtiHi+kHxdUiFM118vNV5XwXsciUk4z38Cr/+60wXfDArlbjhxcPw62cqOQGdDeJRYrnn", + "jsUlL2woX/XzFwS/mzobV2xo42n9wfiXtm95CEceN6KeD4BKfm3sHndjtC4qZimvycrPus6D0dejFXET", + "mrfXov8IkHRjhw9fllPo/loIa9z7Dxqpt+AFfVimxL1EcKdXfsTviN/7jN/hJuul2mL3dSvo/fijjU4r", + "meB8zSMOO3CYMZZ0IeuEscSDprrLWL/yhk12PoxmOLoEGiMFXbwApLuYTogq+YdSZpPpRJWePDf/TCsz", + "29RGN5rRnrHkAT8yrNleTvLBiiV5Cuvm+l+61AOecTPARzLv+Swh0QHLgOKMdE392RVeLHTy762YbyfT", + "Jra93/wt+KWZZDnGIcHXBykIUX8mqMWwU1XwN1tu6GKrK7+1adz7LJ66wkuTr/v18c0uoNWRPdTMy3qa", + "1+yFGzN8Uxe1at14uK0IRNi8pRtjiQVINOcsRRjpUaAlYC5ngOWk5+2udZb74aOy3B0UjPSbrMLdgm8z", + "D2+ZGX+90CsFMaT8KYlvJ/G+Y0FoEV2ALBI028CAKUoZJZJxE0cgsczFI4OZhZZB2tWS4bRzRbYlbvgx", + "jdcxUKmGswM9P5g7am/2/wMAAP//wmFDhXXNAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/daemon/daemonapi/get_object_kvstore_entry.go b/daemon/daemonapi/get_object_kvstore_entry.go index 758b0e403..5f3d8eb90 100644 --- a/daemon/daemonapi/get_object_kvstore_entry.go +++ b/daemon/daemonapi/get_object_kvstore_entry.go @@ -39,9 +39,9 @@ func (a *DaemonAPI) GetObjectKVStoreEntry(ctx echo.Context, namespace string, ki return JSONProblemf(ctx, http.StatusInternalServerError, "NewKeystore", "%s", err) } - if b, err := ks.DecodeKey(params.Key); err != nil { - return JSONProblemf(ctx, http.StatusInternalServerError, "DecodeKey", "%s: %s", params.Key, err) - } else { + b, err := ks.DecodeKey(params.Key) + switch { + case err == nil: var contentType string if utf8.Valid(b) { contentType = "text/plain" @@ -49,6 +49,12 @@ func (a *DaemonAPI) GetObjectKVStoreEntry(ctx echo.Context, namespace string, ki contentType = "application/octet-stream" } return ctx.Blob(http.StatusOK, contentType, b) + case errors.Is(err, object.KeystoreErrKeyEmpty): + return JSONProblemf(ctx, http.StatusBadRequest, "DecodeKey", "%s", err) + case errors.Is(err, object.KeystoreErrNotExist): + return JSONProblemf(ctx, http.StatusNotFound, "DecodeKey", "%s", err) + default: + return JSONProblemf(ctx, http.StatusInternalServerError, "DecodeKey", "%s: %s", params.Key, err) } }