Skip to content

Commit

Permalink
X13 Firmware update support
Browse files Browse the repository at this point in the history
  • Loading branch information
jakeschuurmans committed Nov 19, 2024
1 parent e9c80b4 commit 9ab35be
Show file tree
Hide file tree
Showing 6 changed files with 256 additions and 41 deletions.
198 changes: 198 additions & 0 deletions examples/upload-install-firmware/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
package main

import (
"context"
"crypto/x509"
"errors"
"flag"
"io/ioutil"
"log"
"os"
"strings"
"time"

"github.com/bombsimon/logrusr/v2"
bmclib "github.com/metal-toolbox/bmclib"
"github.com/metal-toolbox/bmclib/constants"
bmclibErrs "github.com/metal-toolbox/bmclib/errors"
"github.com/sirupsen/logrus"
)

func main() {
user := flag.String("user", "", "Username to login with")
pass := flag.String("password", "", "Username to login with")
host := flag.String("host", "", "BMC hostname to connect to")
component := flag.String("component", "", "Component to be updated (bmc, bios.. etc)")
withSecureTLS := flag.Bool("secure-tls", false, "Enable secure TLS")
certPoolPath := flag.String("cert-pool", "", "Path to an file containing x509 CAs. An empty string uses the system CAs. Only takes effect when --secure-tls=true")
firmwarePath := flag.String("firmware", "", "The local path of the firmware to install")
firmwareVersion := flag.String("version", "", "The firmware version being installed")

flag.Parse()

ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
defer cancel()

l := logrus.New()
l.Level = logrus.TraceLevel
logger := logrusr.New(l)

if *host == "" || *user == "" || *pass == "" {
l.Fatal("required host/user/pass parameters not defined")
}

if *component == "" {
l.Fatal("component parameter required (must be a component slug - bmc, bios etc)")
}

clientOpts := []bmclib.Option{
bmclib.WithLogger(logger),
bmclib.WithPerProviderTimeout(time.Minute * 30),
}

if *withSecureTLS {
var pool *x509.CertPool
if *certPoolPath != "" {
pool = x509.NewCertPool()
data, err := ioutil.ReadFile(*certPoolPath)
if err != nil {
l.Fatal(err)
}
pool.AppendCertsFromPEM(data)
}
// a nil pool uses the system certs
clientOpts = append(clientOpts, bmclib.WithSecureTLS(pool))
}

cl := bmclib.NewClient(*host, *user, *pass, clientOpts...)
err := cl.Open(ctx)
if err != nil {
l.Fatal(err, "bmc login failed")
}

defer cl.Close(ctx)

// open file handle
fh, err := os.Open(*firmwarePath)
if err != nil {
l.Fatal(err)
}
defer fh.Close()

steps, err := cl.FirmwareInstallSteps(ctx, *component)
if err != nil {
l.Fatal(err)
}

l.Infof("Steps: %+v", steps)

taskID := ""
var lastStep constants.FirmwareInstallStep = ""
for _, step := range steps {
l.Infof("Step: %s", step)

switch step {
case constants.FirmwareInstallStepUploadInitiateInstall:
taskID, err = cl.FirmwareInstallUploadAndInitiate(ctx, *component, fh)
if err != nil {
l.Fatal(err)
}
case constants.FirmwareInstallStepInstallStatus:
fallthrough
case constants.FirmwareInstallStepUploadStatus:
if taskID == "" {
l.Fatal("taskID wasnt set")
}
if lastStep == "" {
l.Fatal("lastStep wasnt set")
}
firmwareInstallStatusWait(ctx, cl, l, lastStep, *component, *firmwareVersion, taskID)
case constants.FirmwareInstallStepUpload:
taskID, err = cl.FirmwareUpload(ctx, *component, fh)
if err != nil {
l.Fatal(err)
}
case constants.FirmwareInstallStepInstallUploaded:
if taskID == "" {
l.Fatal("taskID wasnt set")
}
taskID, err = cl.FirmwareInstallUploaded(ctx, *component, taskID)
if err != nil {
l.Fatal(err)
}
case constants.FirmwareInstallStepPowerOffHost:
_, err = cl.SetPowerState(ctx, "off")
if err != nil {
l.Fatal(err)
}
case constants.FirmwareInstallStepResetBMCPostInstall:
fallthrough
case constants.FirmwareInstallStepResetBMCOnInstallFailure:
_, err = cl.ResetBMC(ctx, "GracefulRestart")
if err != nil {
l.Fatal(err)
}
default:
l.Fatal("unknown firmware install step")
}

lastStep = step
}
}

func firmwareInstallStatusWait(ctx context.Context, cl *bmclib.Client, l *logrus.Logger, step constants.FirmwareInstallStep, component, firmwareVersion, taskID string) {
for range 300 {
if ctx.Err() != nil {
l.Fatal(ctx.Err())
}

state, status, err := cl.FirmwareTaskStatus(ctx, step, component, taskID, firmwareVersion)
if err != nil {
// when its under update a connection refused is returned
if strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "operation timed out") {
l.Info("BMC refused connection, BMC most likely resetting...")
time.Sleep(2 * time.Second)

continue
}

if errors.Is(err, bmclibErrs.ErrSessionExpired) || strings.Contains(err.Error(), "session expired") {
err := cl.Open(ctx)
if err != nil {
l.Fatal(err, "bmc re-login failed")
}

l.WithFields(logrus.Fields{"state": state, "component": component}).Info("BMC session expired, logging in...")

continue
}

log.Fatal(err)
}

switch state {
case constants.FirmwareInstallRunning, constants.FirmwareInstallInitializing:
l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s running", step)
case constants.FirmwareInstallFailed:
l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s failed", step)
os.Exit(1)
case constants.FirmwareInstallComplete, constants.FirmwareInstallQueued:
l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("%s completed", step)
return
case constants.FirmwareInstallPowerCycleHost:
l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("host powercycle required")

if _, err := cl.SetPowerState(ctx, "cycle"); err != nil {
l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Infof("error power cycling host for %s", step)
os.Exit(1)
}

l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("host power cycled, all done!")
return
default:
l.WithFields(logrus.Fields{"state": state, "status": status, "component": component}).Info("unknown state returned")
}

time.Sleep(2 * time.Second)
}
}
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/metal-toolbox/bmclib

go 1.21
go 1.22

require (
dario.cat/mergo v1.0.0
Expand All @@ -13,6 +13,7 @@ require (
github.com/hashicorp/go-multierror v1.1.1
github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef
github.com/jacobweinstock/registrar v0.4.7
github.com/metal-toolbox/bmc-common v1.0.2
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.31.0
github.com/sirupsen/logrus v1.9.3
Expand All @@ -34,7 +35,6 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/metal-toolbox/bmc-common v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
golang.org/x/sys v0.20.0 // indirect
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 h1:t95Grn2
github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230/go.mod h1:t2EzW1qybnPDQ3LR/GgeF0GOzHUXT5IVMLP2gkW1cmc=
github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 h1:a0MBqYm44o0NcthLKCljZHe1mxlN6oahCQHHThnSwB4=
github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22/go.mod h1:/B7V22rcz4860iDqstGvia/2+IYWXf3/JdQCVd/1D2A=
github.com/bmc-toolbox/common v0.0.0-20240806132831-ba8adc6a35e3 h1:/BjZSX/sphptIdxpYo4wxAQkgMLyMMgfdl48J9DKNeE=
github.com/bmc-toolbox/common v0.0.0-20240806132831-ba8adc6a35e3/go.mod h1:Cdnkm+edb6C0pVkyCrwh3JTXAe0iUF9diDG/DztPI9I=
github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM=
github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
Expand Down
15 changes: 15 additions & 0 deletions internal/redfishwrapper/firmware.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,21 @@ func (c *Client) StartUpdateForUploadedFirmware(ctx context.Context) (taskID str
return taskIDFromResponseBody(response)
}

// StartUpdateForUploadedFirmware starts an update for a firmware file previously uploaded
func (c *Client) StartUpdateForUploadedFirmwareNoTaskID(ctx context.Context) error {
updateService, err := c.client.Service.UpdateService()
if err != nil {
return errors.Wrap(err, "error querying redfish update service")
}

err = updateService.StartUpdate()
if err != nil {
return errors.Wrap(err, "error querying redfish start update endpoint")
}

return nil
}

type TaskAccepted struct {
Accepted struct {
Code string `json:"code"`
Expand Down
52 changes: 29 additions & 23 deletions providers/supermicro/supermicro.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,18 +287,31 @@ func (c *Client) ResetBiosConfiguration(ctx context.Context) (err error) {
}

func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) {
x11 := newX11Client(c.serviceClient, c.log)
x12 := newX12Client(c.serviceClient, c.log)
x13 := newX13Client(c.serviceClient, c.log)

var queryor bmcQueryor

for _, bmc := range []bmcQueryor{x11, x12, x13} {
bmcModels := []struct {
bmc bmcQueryor
modelFamily string
}{
{
newX11Client(c.serviceClient, c.log),
"x11",
},
{
newX12Client(c.serviceClient, c.log),
"x12",
},
{
newX13Client(c.serviceClient, c.log),
"x13",
},
}

var model string
for _, bmcModel := range bmcModels {
var err error

// Note to maintainers: x12 lacks support for the ipmi.cgi endpoint,
// Note to maintainers: x12 and x13 lacks support for the ipmi.cgi endpoint,
// which will lead to our graceful handling of ErrXMLAPIUnsupported below.
_, err = bmc.queryDeviceModel(ctx)
tempModel, err := bmcModel.bmc.queryDeviceModel(ctx)
if err != nil {
if errors.Is(err, ErrXMLAPIUnsupported) {
continue
Expand All @@ -307,24 +320,17 @@ func (c *Client) bmcQueryor(ctx context.Context) (bmcQueryor, error) {
return nil, errors.Wrap(ErrModelUnknown, err.Error())
}

queryor = bmc
break
}

if queryor == nil {
return nil, errors.Wrap(ErrModelUnknown, "failed to setup query client")
}

model := strings.ToLower(queryor.deviceModel())
acceptedModels := []string{"x11", "x12", "x13"}
if strings.HasPrefix(strings.ToLower(tempModel), bmcModel.modelFamily) {
return bmcModel.bmc, nil
}

for _, acceptedModel := range acceptedModels {
if strings.HasPrefix(model, acceptedModel) {
return queryor, nil
// For returning more informative error bellow
if tempModel != "" {
model = tempModel
}
}

return nil, errors.Wrapf(ErrModelUnsupported, "Got: %s, expected one of: %s", model, strings.Join(acceptedModels, ", "))
return nil, errors.Wrapf(ErrModelUnknown, "failed to setup query client, had unsupported model: %s", model)
}

func parseToken(body []byte) string {
Expand Down
26 changes: 12 additions & 14 deletions providers/supermicro/x13.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,24 +139,15 @@ func (c *x13) firmwareTaskActive(ctx context.Context, component string) error {
// redfish OEM fw install parameters
func (c *x13) biosFwInstallParams() (map[string]bool, error) {
switch c.model {
case "x12spo-ntf":
case "x13dem":
return map[string]bool{
"PreserveME": false,
"PreserveNVRAM": false,
"PreserveSMBIOS": true,
"BackupBIOS": false,
"PreserveBOOTCONF": true,
}, nil
case "x12sth-sys":
return map[string]bool{
"PreserveME": false,
"PreserveNVRAM": false,
"PreserveSMBIOS": true,
"PreserveOA": true,
"PreserveSETUPCONF": true,
"PreserveSETUPPWD": true,
"PreserveSECBOOTKEY": true,
"PreserveBOOTCONF": true,
"BackupBIOS": false,
}, nil
default:
// ideally we never get in this position, since theres model number validation in parent callers.
Expand All @@ -170,6 +161,7 @@ func (c *x13) bmcFwInstallParams() map[string]bool {
"PreserveCfg": true,
"PreserveSdr": true,
"PreserveSsl": true,
"BackupBMC": false,
}
}

Expand Down Expand Up @@ -234,15 +226,21 @@ func (c *x13) firmwareInstallUploaded(ctx context.Context, component, uploadTask

taskInfo := fmt.Sprintf("id: %s, state: %s, status: %s", task.ID, task.TaskState, task.TaskStatus)

if task.TaskState != redfish.CompletedTaskState {
if task.TaskState != redfish.CompletedTaskState && task.TaskState != redfish.PendingTaskState {
return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo)
}

if task.TaskStatus != "OK" {
return "", errors.Wrap(brrs.ErrFirmwareVerifyTask, taskInfo)
}

return c.redfish.StartUpdateForUploadedFirmware(ctx)
err = c.redfish.StartUpdateForUploadedFirmwareNoTaskID(ctx)
if err != nil {
return "", err
}

// X13s dont create a new task id when going from upload to install, so we pass through the same one
return uploadTaskID, nil
}

func (c *x13) firmwareTaskStatus(ctx context.Context, component, taskID string) (state constants.TaskState, status string, err error) {
Expand All @@ -267,7 +265,7 @@ func (c *x13) bootComplete() (bool, error) {
if err != nil {
return false, err
}
// TODO?

// we determined this by experiment on X12STH-SYS with redfish 1.14.0
return bp.LastState == redfish.SystemHardwareInitializationCompleteBootProgressTypes, nil
}

0 comments on commit 9ab35be

Please sign in to comment.