diff --git a/examples/upload-install-firmware/main.go b/examples/upload-install-firmware/main.go new file mode 100644 index 00000000..d68e13b4 --- /dev/null +++ b/examples/upload-install-firmware/main.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "crypto/x509" + "errors" + "flag" + "io/ioutil" + "log" + "os" + "strings" + "time" + + "github.com/bombsimon/logrusr/v2" + bmclib "github.com/metal-toolbox/bmclib" + "github.com/metal-toolbox/bmclib/constants" + bmclibErrs "github.com/metal-toolbox/bmclib/errors" + "github.com/sirupsen/logrus" +) + +func main() { + user := flag.String("user", "", "Username to login with") + pass := flag.String("password", "", "Username to login with") + host := flag.String("host", "", "BMC hostname to connect to") + component := flag.String("component", "", "Component to be updated (bmc, bios.. etc)") + withSecureTLS := flag.Bool("secure-tls", false, "Enable secure TLS") + certPoolPath := flag.String("cert-pool", "", "Path to an file containing x509 CAs. An empty string uses the system CAs. Only takes effect when --secure-tls=true") + firmwarePath := flag.String("firmware", "", "The local path of the firmware to install") + firmwareVersion := flag.String("version", "", "The firmware version being installed") + + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + l := logrus.New() + l.Level = logrus.TraceLevel + logger := logrusr.New(l) + + if *host == "" || *user == "" || *pass == "" { + l.Fatal("required host/user/pass parameters not defined") + } + + if *component == "" { + l.Fatal("component parameter required (must be a component slug - bmc, bios etc)") + } + + clientOpts := []bmclib.Option{ + bmclib.WithLogger(logger), + bmclib.WithPerProviderTimeout(time.Minute * 30), + } + + if *withSecureTLS { + var pool *x509.CertPool + if *certPoolPath != "" { + pool = x509.NewCertPool() + data, err := ioutil.ReadFile(*certPoolPath) + if err != nil { + l.Fatal(err) + } + pool.AppendCertsFromPEM(data) + } + // a nil pool uses the system certs + clientOpts = append(clientOpts, bmclib.WithSecureTLS(pool)) + } + + cl := bmclib.NewClient(*host, *user, *pass, clientOpts...) + err := cl.Open(ctx) + if err != nil { + l.Fatal(err, "bmc login failed") + } + + defer cl.Close(ctx) + + // open file handle + fh, err := os.Open(*firmwarePath) + if err != nil { + l.Fatal(err) + } + defer fh.Close() + + steps, err := cl.FirmwareInstallSteps(ctx, *component) + if err != nil { + l.Fatal(err) + } + + l.Infof("Steps: %+v", steps) + + taskID := "" + var lastStep constants.FirmwareInstallStep = "" + for _, step := range steps { + l.Infof("Step: %s", step) + + switch step { + case constants.FirmwareInstallStepUploadInitiateInstall: + taskID, err = cl.FirmwareInstallUploadAndInitiate(ctx, *component, fh) + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepInstallStatus: + fallthrough + case constants.FirmwareInstallStepUploadStatus: + if taskID == "" { + l.Fatal("taskID wasnt set") + } + if lastStep == "" { + l.Fatal("lastStep wasnt set") + } + firmwareInstallStatusWait(ctx, cl, l, lastStep, *component, *firmwareVersion, taskID) + case constants.FirmwareInstallStepUpload: + taskID, err = cl.FirmwareUpload(ctx, *component, fh) + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepInstallUploaded: + if taskID == "" { + l.Fatal("taskID wasnt set") + } + taskID, err = cl.FirmwareInstallUploaded(ctx, *component, taskID) + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepPowerOffHost: + _, err = cl.SetPowerState(ctx, "off") + if err != nil { + l.Fatal(err) + } + case constants.FirmwareInstallStepResetBMCPostInstall: + fallthrough + case constants.FirmwareInstallStepResetBMCOnInstallFailure: + _, err = cl.ResetBMC(ctx, "GracefulRestart") + if err != nil { + l.Fatal(err) + } + default: + l.Fatal("unknown firmware install step") + } + + lastStep = step + } +} + +func firmwareInstallStatusWait(ctx context.Context, cl *bmclib.Client, l *logrus.Logger, step constants.FirmwareInstallStep, component, firmwareVersion, taskID string) { + for range 300 { + if ctx.Err() != nil { + l.Fatal(ctx.Err()) + } + + state, status, err := cl.FirmwareTaskStatus(ctx, step, component, taskID, firmwareVersion) + if err != nil { + // when its under update a connection refused is returned + if strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "operation timed out") { + l.Info("BMC refused connection, BMC most likely resetting...") + time.Sleep(2 * time.Second) + + continue + } + + if errors.Is(err, bmclibErrs.ErrSessionExpired) || strings.Contains(err.Error(), "session expired") { + err := cl.Open(ctx) + if err != nil { + l.Fatal(err, "bmc re-login failed") + } + + l.WithFields(logrus.Fields{"state": state, "component": component}).Info("BMC session expired, logging in...") + + continue + } + + log.Fatal(err) + } + + switch state { + case constants.FirmwareInstallRunning, constants.FirmwareInstallInitializing: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s running", step) + case constants.FirmwareInstallFailed: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s failed", step) + os.Exit(1) + case constants.FirmwareInstallComplete, constants.FirmwareInstallQueued: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s completed", step) + return + case constants.FirmwareInstallPowerCycleHost: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("host powercycle required") + + if _, err := cl.SetPowerState(ctx, "cycle"); err != nil { + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("error power cycling host for %s", step) + os.Exit(1) + } + + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("host power cycled, all done!") + return + default: + l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("unknown state returned") + } + + time.Sleep(2 * time.Second) + } +} diff --git a/go.mod b/go.mod index 573dcd99..7a9254e3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/metal-toolbox/bmclib -go 1.21 +go 1.22 require ( dario.cat/mergo v1.0.0 @@ -13,6 +13,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef github.com/jacobweinstock/registrar v0.4.7 + github.com/metal-toolbox/bmc-common v1.0.2 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 github.com/sirupsen/logrus v1.9.3 @@ -34,7 +35,6 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/metal-toolbox/bmc-common v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/satori/go.uuid v1.2.0 // indirect golang.org/x/sys v0.20.0 // indirect diff --git a/go.sum b/go.sum index 87ec3331..13ccddb6 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 h1:t95Grn2 github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230/go.mod h1:t2EzW1qybnPDQ3LR/GgeF0GOzHUXT5IVMLP2gkW1cmc= github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 h1:a0MBqYm44o0NcthLKCljZHe1mxlN6oahCQHHThnSwB4= github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22/go.mod h1:/B7V22rcz4860iDqstGvia/2+IYWXf3/JdQCVd/1D2A= -github.com/bmc-toolbox/common v0.0.0-20240806132831-ba8adc6a35e3 h1:/BjZSX/sphptIdxpYo4wxAQkgMLyMMgfdl48J9DKNeE= -github.com/bmc-toolbox/common v0.0.0-20240806132831-ba8adc6a35e3/go.mod h1:Cdnkm+edb6C0pVkyCrwh3JTXAe0iUF9diDG/DztPI9I= github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM= github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= diff --git a/internal/redfishwrapper/firmware.go b/internal/redfishwrapper/firmware.go index 287cda17..44523e13 100644 --- a/internal/redfishwrapper/firmware.go +++ b/internal/redfishwrapper/firmware.go @@ -169,6 +169,21 @@ func (c *Client) StartUpdateForUploadedFirmware(ctx context.Context) (taskID str return taskIDFromResponseBody(response) } +// StartUpdateForUploadedFirmware starts an update for a firmware file previously uploaded +func (c *Client) StartUpdateForUploadedFirmwareNoTaskID(ctx context.Context) error { + updateService, err := c.client.Service.UpdateService() + if err != nil { + return errors.Wrap(err, "error querying redfish update service") + } + + err = updateService.StartUpdate() + if err != nil { + return errors.Wrap(err, "error querying redfish start update endpoint") + } + + return nil +} + type TaskAccepted struct { Accepted struct { Code string `json:"code"` diff --git a/providers/supermicro/supermicro.go b/providers/supermicro/supermicro.go index bff35d82..84390782 100644 --- a/providers/supermicro/supermicro.go +++ b/providers/supermicro/supermicro.go @@ -287,18 +287,31 @@ func (c *Client) ResetBiosConfiguration(ctx context.Context) (err error) { } func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) { - x11 := newX11Client(c.serviceClient, c.log) - x12 := newX12Client(c.serviceClient, c.log) - x13 := newX13Client(c.serviceClient, c.log) - - var queryor bmcQueryor - - for _, bmc := range []bmcQueryor{x11, x12, x13} { + bmcModels := []struct { + bmc bmcQueryor + modelFamily string + }{ + { + newX11Client(c.serviceClient, c.log), + "x11", + }, + { + newX12Client(c.serviceClient, c.log), + "x12", + }, + { + newX13Client(c.serviceClient, c.log), + "x13", + }, + } + + var model string + for _, bmcModel := range bmcModels { var err error - // Note to maintainers: x12 lacks support for the ipmi.cgi endpoint, + // Note to maintainers: x12 and x13 lacks support for the ipmi.cgi endpoint, // which will lead to our graceful handling of ErrXMLAPIUnsupported below. - _, err = bmc.queryDeviceModel(ctx) + tempModel, err := bmcModel.bmc.queryDeviceModel(ctx) if err != nil { if errors.Is(err, ErrXMLAPIUnsupported) { continue @@ -307,24 +320,17 @@ func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) { return nil, errors.Wrap(ErrModelUnknown, err.Error()) } - queryor = bmc - break - } - - if queryor == nil { - return nil, errors.Wrap(ErrModelUnknown, "failed to setup query client") - } - - model := strings.ToLower(queryor.deviceModel()) - acceptedModels := []string{"x11", "x12", "x13"} + if strings.HasPrefix(tempModel, bmcModel.modelFamily) { + return bmcModel.bmc, nil + } - for _, acceptedModel := range acceptedModels { - if strings.HasPrefix(model, acceptedModel) { - return queryor, nil + // For returning more informative error bellow + if tempModel != "" { + model = tempModel } } - return nil, errors.Wrapf(ErrModelUnsupported, "Got: %s, expected one of: %s", model, strings.Join(acceptedModels, ", ")) + return nil, errors.Wrapf(ErrModelUnknown, "failed to setup query client, had unsupported model: %s", model) } func parseToken(body []byte) string { diff --git a/providers/supermicro/x13.go b/providers/supermicro/x13.go index 5d6388cf..0df7d08e 100644 --- a/providers/supermicro/x13.go +++ b/providers/supermicro/x13.go @@ -139,24 +139,15 @@ func (c *x13) firmwareTaskActive(ctx context.Context, component string) error { // redfish OEM fw install parameters func (c *x13) biosFwInstallParams() (map[string]bool, error) { switch c.model { - case "x12spo-ntf": + case "x13dem": return map[string]bool{ - "PreserveME": false, - "PreserveNVRAM": false, - "PreserveSMBIOS": true, - "BackupBIOS": false, - "PreserveBOOTCONF": true, - }, nil - case "x12sth-sys": - return map[string]bool{ - "PreserveME": false, - "PreserveNVRAM": false, "PreserveSMBIOS": true, "PreserveOA": true, "PreserveSETUPCONF": true, "PreserveSETUPPWD": true, "PreserveSECBOOTKEY": true, "PreserveBOOTCONF": true, + "BackupBIOS": false, }, nil default: // ideally we never get in this position, since theres model number validation in parent callers. @@ -170,6 +161,7 @@ func (c *x13) bmcFwInstallParams() map[string]bool { "PreserveCfg": true, "PreserveSdr": true, "PreserveSsl": true, + "BackupBMC": false, } } @@ -234,7 +226,7 @@ func (c *x13) firmwareInstallUploaded(ctx context.Context, component, uploadTask taskInfo := fmt.Sprintf("id: %s, state: %s, status: %s", task.ID, task.TaskState, task.TaskStatus) - if task.TaskState != redfish.CompletedTaskState { + if task.TaskState != redfish.CompletedTaskState && task.TaskState != redfish.PendingTaskState { return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo) } @@ -242,7 +234,13 @@ func (c *x13) firmwareInstallUploaded(ctx context.Context, component, uploadTask return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo) } - return c.redfish.StartUpdateForUploadedFirmware(ctx) + err = c.redfish.StartUpdateForUploadedFirmwareNoTaskID(ctx) + if err != nil { + return "", err + } + + // X13s dont create a new task id when going from upload to install, so we pass through the same one + return uploadTaskID, nil } func (c *x13) firmwareTaskStatus(ctx context.Context, component, taskID string) (state constants.TaskState, status string, err error) { @@ -267,7 +265,7 @@ func (c *x13) bootComplete() (bool, error) { if err != nil { return false, err } - // TODO? + // we determined this by experiment on X12STH-SYS with redfish 1.14.0 return bp.LastState == redfish.SystemHardwareInitializationCompleteBootProgressTypes, nil }