From 6e1f57eeec21412c37c26e682653614617ea055a Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Mon, 19 Dec 2022 09:02:34 -0600 Subject: [PATCH 01/71] Debug build for NVM Format command Signed-off-by: Nate Thornton --- go.mod | 2 +- go.sum | 4 +- .../internal/switchtec/pkg/nvme/nvme.go | 30 ------ .../pkg/manager-nnf/allocation_policy.go | 29 ++++-- .../nnf-ec/pkg/manager-nnf/manager.go | 14 ++- .../nnf-ec/pkg/manager-nnf/storage_group.go | 26 +++-- .../nnf-ec/pkg/manager-nnf/storage_pool.go | 33 ++++--- .../nnf-ec/pkg/manager-nvme/manager.go | 98 +++++++++++-------- .../nnf-ec/pkg/manager-nvme/nvme_api.go | 1 - .../nnf-ec/pkg/manager-nvme/nvme_cli.go | 17 ---- .../nnf-ec/pkg/manager-nvme/nvme_device.go | 5 - .../nnf-ec/pkg/manager-nvme/nvme_mock.go | 5 - vendor/modules.txt | 2 +- 13 files changed, 137 insertions(+), 129 deletions(-) diff --git a/go.mod b/go.mod index 48ad71a5e..38e3c074c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20221216152313-0643bca404f9 + github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index bd7e3a51f..bd0a87f9d 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h1:upOQcpaEDmgLxoK7SZeqzGOQs8raxd9/paJYmBS/3/M= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221216152313-0643bca404f9 h1:R57C6RJiTMgAoq+sLDADqYpo34mpeblv46JNkxaT5LU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221216152313-0643bca404f9/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936 h1:q/Kx0q0G2I9N//snKIQxGXvxgBZKxKb2JAyI2+RW+Uw= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go index 24c5fe055..35e836cc8 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go @@ -725,36 +725,6 @@ func (dev *Device) FormatNamespace(namespaceID uint32) error { return dev.ops.submitAdminPassthru(dev, &cmd, nil) } -// WaitFormatComplete polls the namespace waiting for the Utilization to reach 0. -func (dev *Device) WaitFormatComplete(namespaceID uint32) error { - idns, err := dev.IdentifyNamespace(namespaceID, true /* namespace present */) - if err != nil { - return err - } - - lastUtilization := idns.Utilization - - // As the format command runs, the Utilization should decrease to 0 - for lastUtilization > 0 { - // Pause briefly to for format to make progress - delay := 100 * time.Millisecond - time.Sleep(delay) - idns, err = dev.IdentifyNamespace(namespaceID, true /* namespace present */) - if err != nil { - return err - } - - // Fail if format is stuck - if lastUtilization == idns.Utilization { - return fmt.Errorf("Format is not progressing after %s", delay.String()) - } - - lastUtilization = idns.Utilization - } - - return nil -} - func (dev *Device) manageNamespace(namespaceID uint32, controllers []uint16, attach bool) error { list := CtrlList{ diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/allocation_policy.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/allocation_policy.go index bc65cc112..7f47635e6 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/allocation_policy.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/allocation_policy.go @@ -114,6 +114,9 @@ func NewAllocationPolicy(config AllocationConfig, oem map[string]interface{}) Al /* ------------------------------ Spare Allocation Policy --------------------- */ +// Required number of drives for the Spare Allocation Policy +const SpareAllocationPolicyRequiredDriveCount = 16 + type SpareAllocationPolicy struct { compliance AllocationComplianceType storage []*nvme.Storage @@ -125,7 +128,7 @@ func (p *SpareAllocationPolicy) Initialize(capacityBytes uint64) error { storage := []*nvme.Storage{} for _, s := range nvme.GetStorage() { - if s.IsEnabled() { + if s.IsEnabled() && s.UnallocatedBytes() > 0 { storage = append(storage, s) } } @@ -135,7 +138,7 @@ func (p *SpareAllocationPolicy) Initialize(capacityBytes uint64) error { return !!!(storage[i].UnallocatedBytes() < storage[j].UnallocatedBytes()) }) - count := 16 + count := SpareAllocationPolicyRequiredDriveCount if len(storage) < count { count = len(storage) } @@ -148,7 +151,7 @@ func (p *SpareAllocationPolicy) Initialize(capacityBytes uint64) error { func (p *SpareAllocationPolicy) CheckCapacity() error { if p.capacityBytes == 0 { - return fmt.Errorf("Requested capacity %#x is invalid", p.capacityBytes) + return fmt.Errorf("Requested capacity must be non-zero") } var availableBytes = uint64(0) @@ -157,17 +160,27 @@ func (p *SpareAllocationPolicy) CheckCapacity() error { } if availableBytes < p.capacityBytes { - return fmt.Errorf("Insufficient capacity available. Requested: %#x Available: %#x", p.capacityBytes, availableBytes) + return fmt.Errorf("Insufficient capacity available. Requested: %d Available: %d", p.capacityBytes, availableBytes) } if p.compliance != RelaxedAllocationComplianceType { - if len(p.storage) != 16 { - return fmt.Errorf("Insufficient drive count. Available %d", len(p.storage)) + if len(p.storage) != SpareAllocationPolicyRequiredDriveCount { + return fmt.Errorf("Insufficient drive count. Required: %d Available: %d", SpareAllocationPolicyRequiredDriveCount, len(p.storage)) + } + + roundUpToMultiple := func(n, m uint64) uint64 { // Round 'n' up to a multiple of 'm' + return ((n + m - 1) / m) * m } - if p.capacityBytes%uint64(len(p.storage)) != 0 { - return fmt.Errorf("Requested capacity is a non-multiple of available storage") + // Validate each drive can contribute sufficient capacity towards the entire pool. + poolCapacityBytes := roundUpToMultiple(p.capacityBytes, SpareAllocationPolicyRequiredDriveCount) + driveCapacityBytes := roundUpToMultiple(poolCapacityBytes/SpareAllocationPolicyRequiredDriveCount, 4096) + + for _, s := range p.storage { + if driveCapacityBytes > s.UnallocatedBytes() { + return fmt.Errorf("Insufficient drive capacity available. Requested: %d Available: %d", driveCapacityBytes, s.UnallocatedBytes()) + } } } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go index 20118845e..e7222769e 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go @@ -985,7 +985,12 @@ func (*StorageService) StorageServiceIdStorageGroupPost(storageServiceId string, updateFunc := func() error { for _, pv := range sp.providingVolumes { - if err := nvme.AttachController(pv.Storage.FindVolume(pv.VolumeId), sg.endpoint.controllerId); err != nil { + volume := pv.Storage.FindVolume(pv.VolumeId) + if volume == nil { + return ec.NewErrInternalServerError().WithResourceType(StorageGroupOdataType).WithCause(fmt.Sprintf("Storage group '%s' attach volume '%s' not found", sg.id, pv.VolumeId)) + } + + if err := volume.AttachController(sg.endpoint.controllerId); err != nil { return err } } @@ -1079,7 +1084,12 @@ func (*StorageService) StorageServiceIdStorageGroupIdDelete(storageServiceId, st // Detach the endpoint from the NVMe namespaces for _, pv := range sp.providingVolumes { - if err := nvme.DetachController(pv.Storage.FindVolume(pv.VolumeId), sg.endpoint.controllerId); err != nil { + volume := pv.Storage.FindVolume(pv.VolumeId) + if volume == nil { + return ec.NewErrInternalServerError().WithResourceType(StorageGroupOdataType).WithCause(fmt.Sprintf("Storage group '%s' detach volume '%s' not found", storageGroupId, pv.VolumeId)) + } + + if err := volume.DetachController(sg.endpoint.controllerId); err != nil { return ec.NewErrInternalServerError().WithResourceType(StorageGroupOdataType).WithError(err).WithCause(fmt.Sprintf("Storage group '%s' failed to detach controller '%d'", storageGroupId, sg.endpoint.controllerId)) } } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_group.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_group.go index f813d724a..c89204f9d 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_group.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_group.go @@ -23,7 +23,6 @@ import ( "encoding/json" "fmt" - nvme "github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme" server "github.com/NearNodeFlash/nnf-ec/pkg/manager-server" "github.com/NearNodeFlash/nnf-ec/pkg/persistent" sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models" @@ -122,11 +121,16 @@ func (sg *StorageGroup) Rollback(state uint32) error { sp := sg.storageService.findStoragePool(sg.storagePoolId) if sp == nil { - return fmt.Errorf("Storage Pool %s not found", sg.storagePoolId) + return fmt.Errorf("Rollback Storage Group %s Create Start: Storage Pool %s not found", sg.id, sg.storagePoolId) } for _, pv := range sp.providingVolumes { - if err := nvme.DetachController(pv.Storage.FindVolume(pv.VolumeId), sg.endpoint.controllerId); err != nil { + volume := pv.Storage.FindVolume(pv.VolumeId) + if volume == nil { + return fmt.Errorf("Rollback Storage Group %s Create Start: Volume %s not found", sg.id, pv.VolumeId) + } + + if err := volume.DetachController(sg.endpoint.controllerId); err != nil { return err } } @@ -136,11 +140,16 @@ func (sg *StorageGroup) Rollback(state uint32) error { sp := sg.storageService.findStoragePool(sg.storagePoolId) if sp == nil { - return fmt.Errorf("Storage Pool %s not found", sg.storagePoolId) + return fmt.Errorf("Rollback Storage Group %s Delete Start: Storage Pool %s not found", sg.id, sg.storagePoolId) } for _, pv := range sp.providingVolumes { - if err := nvme.AttachController(pv.Storage.FindVolume(pv.VolumeId), sg.endpoint.controllerId); err != nil { + volume := pv.Storage.FindVolume(pv.VolumeId) + if volume == nil { + return fmt.Errorf("Rollback Storage Group %s Delete Start: Volume %s not found", sg.id, pv.VolumeId) + } + + if err := volume.AttachController(sg.endpoint.controllerId); err != nil { return err } } @@ -206,7 +215,12 @@ func (rh *storageGroupRecoveryReplyHandler) Done() (bool, error) { sp := rh.storageService.findStoragePool(sg.storagePoolId) for _, pv := range sp.providingVolumes { - if err := nvme.DetachController(pv.Storage.FindVolume(pv.VolumeId), sg.endpoint.controllerId); err != nil { + volume := pv.Storage.FindVolume(pv.VolumeId) + if volume == nil { + return false, fmt.Errorf("Storage Group %s Recover: Volume %s not found", sg.id, pv.VolumeId) + } + + if err := volume.DetachController(sg.endpoint.controllerId); err != nil { return false, err } } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go index 0adf16eae..7cbfdea71 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go @@ -145,23 +145,34 @@ func (p *StoragePool) recoverVolumes(volumes []storagePoolPersistentVolumeInfo, } func (p *StoragePool) deallocateVolumes() error { - // In order to speed up deleting volumes, we format them first. - // Format runs asynchronously on each namespace. Launch format for each namespace - // and wait all format operations to complete. + // In order to speed up deleting volumes, we format them first. Format runs asynchronously, so after + // each format call, wait for completion before deleting the volume. - /* - TEMP: Can't do this just yet - the format must be attached to a controller + runOnProvidingVolumes := func(volFn func(*nvme.Volume) error) error { for _, pv := range p.providingVolumes { - if err := nvme.FormatVolumeAndWaitForComplete(pv.Storage.FindVolume(pv.VolumeId)); err != nil { + volume := pv.Storage.FindVolume(pv.VolumeId) + if volume == nil { + return fmt.Errorf("Volume %s not found", pv.VolumeId) + } + + if err := volFn(volume); err != nil { return err } } - */ - for _, pv := range p.providingVolumes { - if err := nvme.DeleteVolume(pv.Storage.FindVolume(pv.VolumeId)); err != nil { - return err - } + return nil + } + + if err := runOnProvidingVolumes(func(v *nvme.Volume) error { return v.Format() }); err != nil { + return fmt.Errorf("Failed to format volumes: %v", err) + } + + if err := runOnProvidingVolumes(func(v *nvme.Volume) error { return v.WaitFormatComplete() }); err != nil { + return fmt.Errorf("Failed to wait on format completions: %v", err) + } + + if err := runOnProvidingVolumes(func(v *nvme.Volume) error { return v.Delete() }); err != nil { + return fmt.Errorf("Failed to delete volumes: %v", err) } return nil diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go index 2524edbb1..7e4beb916 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go @@ -26,6 +26,7 @@ import ( "math" "strconv" "strings" + "time" . "github.com/NearNodeFlash/nnf-ec/pkg/api" event "github.com/NearNodeFlash/nnf-ec/pkg/manager-event" @@ -281,22 +282,6 @@ func CreateVolume(s *Storage, capacityBytes uint64) (*Volume, error) { return s.createVolume(capacityBytes) } -func DeleteVolume(v *Volume) error { - return v.storage.deleteVolume(v.id) -} - -func FormatVolumeAndWaitForComplete(v *Volume) error { - return v.storage.formatVolumeAndWaitForComplete(v.id) -} - -func AttachController(v *Volume, controllerId uint16) error { - return v.attach(controllerId) -} - -func DetachController(v *Volume, controllerId uint16) error { - return v.detach(controllerId) -} - func (s *Storage) UnallocatedBytes() uint64 { return s.unallocatedBytes } func (s *Storage) IsEnabled() bool { return s.state == sf.ENABLED_RST } func (s *Storage) SerialNumber() string { return s.serialNumber } @@ -539,26 +524,6 @@ func (s *Storage) formatVolume(volumeID string) error { return ec.NewErrNotFound() } -func (s *Storage) formatVolumeAndWaitForComplete(volumeID string) error { - // Launch formats for all namespaces - if err := s.formatVolume(volumeID); err != nil { - return err - } - - // Wait all formats to finish - for _, volume := range s.volumes { - if volume.id == volumeID { - if err := s.device.WaitFormatComplete(volume.namespaceId); err != nil { - return err - } - - return nil - } - } - - return ec.NewErrNotFound() -} - func (s *Storage) recoverVolumes() error { namespaces, err := s.device.ListNamespaces(0) if err != nil { @@ -604,22 +569,75 @@ func (v *Volume) GetGloballyUniqueIdentifier() nvme.NamespaceGloballyUniqueIdent return v.guid } +func (v *Volume) Delete() error { return v.storage.deleteVolume(v.id) } + +func (v *Volume) AttachController(controllerId uint16) error { return v.attach(controllerId) } +func (v *Volume) DetachController(controllerId uint16) error { return v.detach(controllerId) } + +func (v *Volume) Format() error { + + // According to the NVM Express Base Specification 2.0b section 5.14 Format NVM Command + // The scope of the format operation and the scope of the format with secure erase depend + // on the attributes that the controller supports for the Format NVM command and the + // Namespace Identifier specified in the command. + // + // So the namespace must be attached to a controller. We attach to the controller associated with + // the physical function to avoid any noise generated by attaching the the volume to a working host. + + return v.runInAttachDetachBlock(func() error { return v.storage.formatVolume(v.id)}) +} + func (v *Volume) SetFeature(data []byte) error { - // Set feature requires the volume is attached to receive the feature data. We attach - // to the physical function to avoid any noise generated by attaching the volume to a - // working host. + // Set feature requires the volume is attached to a controller to receive the feature data. We + // attach to the controller associated with the the physical function to avoid any noise generated by + // attaching the volume to a working host. + + return v.runInAttachDetachBlock(func() error { return v.storage.device.SetNamespaceFeature(v.namespaceId, data) }) +} + +func (v *Volume) runInAttachDetachBlock(fn func() error) error { if err := v.attach(v.storage.physicalFunctionControllerId); err != nil { return err } - if err := v.storage.device.SetNamespaceFeature(v.namespaceId, data); err != nil { + if err := fn(); err != nil { return err } return v.detach(v.storage.physicalFunctionControllerId) } +// Wait for Format Completion by polling on the namespace Utilization value to reach zero. +func (v *Volume) WaitFormatComplete() error { + + ns, err := v.storage.device.IdentifyNamespace(v.namespaceId) + if err != nil { + return err + } + + for ns.Utilization != 0 { + + const delay = 100 * time.Millisecond + + // Pause briefly for format to make progress + time.Sleep(delay) + + lastUtilization := ns.Utilization + + ns, err = v.storage.device.IdentifyNamespace(v.namespaceId) + if err != nil { + return err + } + + if lastUtilization == ns.Utilization { + return fmt.Errorf("Device %s Format Stalled: Namespace: %d Delay: %s", v.storage.id, v.namespaceId, delay.String()) + } + } + + return nil +} + func (v *Volume) attach(controllerId uint16) error { // These are really controller indicies that are passed into the nvme-manager; we've always // just assumed that they map 1-1 to the secondary devices because that is how the Samsung diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_api.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_api.go index c0f44a470..4a60efb73 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_api.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_api.go @@ -57,7 +57,6 @@ type NvmeDeviceApi interface { DeleteNamespace(namespaceId nvme.NamespaceIdentifier) error FormatNamespace(namespaceID nvme.NamespaceIdentifier) error - WaitFormatComplete(namespaceID nvme.NamespaceIdentifier) error AttachNamespace(namespaceId nvme.NamespaceIdentifier, controllers []uint16) error DetachNamespace(namespaceId nvme.NamespaceIdentifier, controllers []uint16) error diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_cli.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_cli.go index c20187230..0b0a9a690 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_cli.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_cli.go @@ -305,23 +305,6 @@ func (d *cliDevice) FormatNamespace(namespaceID nvme.NamespaceIdentifier) error return nil } -func (d *cliDevice) WaitFormatComplete(namespaceID nvme.NamespaceIdentifier) error { - idns := &nvme.IdNs{} - idns.Utilization = 1 // something other than 0 to get the loop going below - - for idns.Utilization > 0 { - idns, err := d.IdentifyNamespace(namespaceID) - if err != nil { - return err - } - - fmt.Printf("Formatting, Utilization(bytes): %d \r", idns.Utilization) // wipe out straggling digits - } - - fmt.Printf("Formatting, Utilization(bytes): %d \r", idns.Utilization) - return nil -} - func (d *cliDevice) AttachNamespace(namespaceId nvme.NamespaceIdentifier, controllers []uint16) error { // Example Command // # nvme attach-ns /dev/nvme2 --namespace-id=1 --controllers=0x41 diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_device.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_device.go index 21c701db2..86c684ad4 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_device.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_device.go @@ -192,11 +192,6 @@ func (d *nvmeDevice) FormatNamespace(namespaceID nvme.NamespaceIdentifier) error return d.dev.FormatNamespace(uint32(namespaceID)) } -// WaitFormatComplete - -func (d *nvmeDevice) WaitFormatComplete(namespaceID nvme.NamespaceIdentifier) error { - return d.dev.WaitFormatComplete(uint32(namespaceID)) -} - // AttachNamespace - func (d *nvmeDevice) AttachNamespace(namespaceId nvme.NamespaceIdentifier, controllers []uint16) error { return d.dev.AttachNamespace(uint32(namespaceId), controllers) diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go index d6d989c54..0aa4c4ce3 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go @@ -373,11 +373,6 @@ func (d *mockDevice) FormatNamespace(namespaceID nvme.NamespaceIdentifier) error return nil } -// FormatNamespace - -func (d *mockDevice) WaitFormatComplete(namespaceID nvme.NamespaceIdentifier) error { - return nil -} - // AttachNamespace - func (d *mockDevice) AttachNamespace(namespaceId nvme.NamespaceIdentifier, controllers []uint16) error { ns := d.findNamespace(namespaceId) diff --git a/vendor/modules.txt b/vendor/modules.txt index 121ca19ed..18d311e4a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20221216152313-0643bca404f9 +# github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936 ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From 6197c5211bbf9a7041e09f23cd7bfbb2af8a77c2 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Mon, 19 Dec 2022 10:55:12 -0600 Subject: [PATCH 02/71] Upgrade nnf-ec to latest (e4ba0b) Signed-off-by: Nate Thornton --- go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 38e3c074c..c574e0a68 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936 + github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index bd0a87f9d..ddf050707 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h1:upOQcpaEDmgLxoK7SZeqzGOQs8raxd9/paJYmBS/3/M= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936 h1:q/Kx0q0G2I9N//snKIQxGXvxgBZKxKb2JAyI2+RW+Uw= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 h1:yMRoXhdvJSd+QMfZBv98K82+u9NAvXI5N/dZdxEfo7Q= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/modules.txt b/vendor/modules.txt index 18d311e4a..60a857ecf 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20221216224339-ce198019e936 +# github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From e062a5ef0aa2ce4fef9bbbe2badce696ac5c4137 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Tue, 20 Dec 2022 12:34:23 -0600 Subject: [PATCH 03/71] Upgrade nnf-ec to latest (96d6a3) Signed-off-by: Nate Thornton --- go.mod | 2 +- go.sum | 2 + .../nnf-ec/pkg/manager-nvme/nvme_mock.go | 48 ++++++++++--------- .../pkg/manager-nvme/nvme_mock_persistence.go | 18 +++---- vendor/modules.txt | 2 +- 5 files changed, 39 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index c574e0a68..9e6ab9032 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 + github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index ddf050707..5887cd508 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,8 @@ github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 h1:yMRoXhdvJSd+QMfZBv98K82+u9NAvXI5N/dZdxEfo7Q= github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 h1:f09NvmieiaSYE8Gxd6+vhmrtx1PB4W4Sldc+rJAAxZQ= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go index 0aa4c4ce3..54dae95c3 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go @@ -67,8 +67,8 @@ func (ctrl *MockNvmeDeviceController) Close() error { func (ctrl MockNvmeDeviceController) NewNvmeDevice(fabricId, switchId, portId string) (NvmeDeviceApi, error) { dev := &mockDevice{ virtualizationManagement: true, - capacity: mockCapacityInBytes, - allocatedCapacity: 0, + capacityInBytes: mockCapacityInBytes, + allocatedCapacityInBytes: 0, // identification data fabricId: fabricId, @@ -117,8 +117,9 @@ type mockDevice struct { virtualizationManagement bool controllers [1 + mockSecondaryControllerCount]mockController // +1 for PF namespaces [1 + mockMaximumNamespaceCount]mockNamespace // +1 for CommonNamespaceId = 0xFFFFFFFF - capacity uint64 - allocatedCapacity uint64 + + capacityInBytes uint64 + allocatedCapacityInBytes uint64 fabricId string switchId string @@ -128,19 +129,18 @@ type mockDevice struct { } type mockController struct { - id uint16 - online bool - capacity uint64 - allocatedCapacity uint64 - vqresources uint32 - viresources uint32 + id uint16 + online bool + capacity uint64 + //allocatedCapacity uint64 + vqresources uint32 + viresources uint32 } type mockNamespace struct { id nvme.NamespaceIdentifier idx int - size uint64 - capacity uint64 + capacityInBytes uint64 guid [16]byte attachedControllers [1 + mockSecondaryControllerCount]*mockController // +1 for PF metadata []byte @@ -199,8 +199,8 @@ func (d *mockDevice) IdentifyController(controllerId uint16) (*nvme.IdCtrl, erro return nil, err } - binary.LittleEndian.PutUint64(ctrl.TotalNVMCapacity[:], d.capacity) - binary.LittleEndian.PutUint64(ctrl.UnallocatedNVMCapacity[:], d.capacity-d.allocatedCapacity) + binary.LittleEndian.PutUint64(ctrl.TotalNVMCapacity[:], d.capacityInBytes) + binary.LittleEndian.PutUint64(ctrl.UnallocatedNVMCapacity[:], d.capacityInBytes-d.allocatedCapacityInBytes) ctrl.OptionalAdminCommandSupport = nvme.VirtualiztionManagementSupport @@ -216,8 +216,9 @@ func (d *mockDevice) IdentifyNamespace(namespaceId nvme.NamespaceIdentifier) (*n idns := new(nvme.IdNs) - idns.Size = ns.capacity - idns.Capacity = ns.capacity + idns.Size = ns.capacityInBytes / mockSectorSizeInBytes // NOTE: Namespace Size is in Sectors + idns.Capacity = ns.capacityInBytes / mockSectorSizeInBytes // NOTE: Namespace Capacity is in Sectors + idns.MultiPathIOSharingCapabilities.Sharing = 1 idns.FormattedLBASize = nvme.FormattedLBASize{Format: 0} @@ -302,9 +303,10 @@ func (d *mockDevice) ListAttachedControllers(namespaceId nvme.NamespaceIdentifie // CreateNamespace - func (d *mockDevice) CreateNamespace(sizeInSectors uint64, sectorSizeIndex uint8) (nvme.NamespaceIdentifier, nvme.NamespaceGloballyUniqueIdentifier, error) { - capacityBytes := sizeInSectors * mockSectorSizeInBytes - if capacityBytes > (d.capacity - d.allocatedCapacity) { - return 0, nvme.NamespaceGloballyUniqueIdentifier{}, fmt.Errorf("Insufficient capacity: Requested: %d Available: %d", capacityBytes, (d.capacity - d.allocatedCapacity)) + capacityInBytes := sizeInSectors * mockSectorSizeInBytes + unallocatedCapacityInBytes := d.capacityInBytes - d.allocatedCapacityInBytes + if capacityInBytes > unallocatedCapacityInBytes { + return 0, nvme.NamespaceGloballyUniqueIdentifier{}, fmt.Errorf("Insufficient capacity: Requested: %d Unallocated: %d", capacityInBytes, unallocatedCapacityInBytes) } ns := d.findNamespace(invalidNamespaceId) // find a free namespace @@ -313,7 +315,7 @@ func (d *mockDevice) CreateNamespace(sizeInSectors uint64, sectorSizeIndex uint8 } ns.id = nvme.NamespaceIdentifier(ns.idx) - ns.capacity = capacityBytes + ns.capacityInBytes = capacityInBytes ns.guid = [16]byte{ 0, 0, 0, 0, 0, 0, 0, 0, @@ -328,7 +330,7 @@ func (d *mockDevice) CreateNamespace(sizeInSectors uint64, sectorSizeIndex uint8 ns.attachedControllers[idx] = nil } - d.allocatedCapacity += (ns.capacity * mockSectorSizeInBytes) + d.allocatedCapacityInBytes += capacityInBytes if d.persistenceMgr != nil { d.persistenceMgr.recordCreateNamespace(d, ns) @@ -358,13 +360,15 @@ func (d *mockDevice) DeleteNamespace(namespaceId nvme.NamespaceIdentifier) error } } - d.allocatedCapacity -= (ns.capacity * mockSectorSizeInBytes) + d.allocatedCapacityInBytes -= ns.capacityInBytes if d.persistenceMgr != nil { d.persistenceMgr.recordDeleteNamespace(d, ns) } ns.id = invalidNamespaceId + ns.capacityInBytes = 0 + return nil } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock_persistence.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock_persistence.go index 696a0649c..dbdf2c864 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock_persistence.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock_persistence.go @@ -24,8 +24,8 @@ import ( "errors" "fmt" - "github.com/NearNodeFlash/nnf-ec/pkg/persistent" "github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme" + "github.com/NearNodeFlash/nnf-ec/pkg/persistent" ) // NVME Mock Persistence - Handles the various persistent aspects of an NVMe device, specifically namespaces @@ -78,7 +78,7 @@ func (mgr *MockNvmePersistenceManager) close() error { // all the NVMe Namespaces with the attached Controllers. func (mgr *MockNvmePersistenceManager) load(dev *mockDevice) error { - dev.allocatedCapacity = 0 + dev.allocatedCapacityInBytes = 0 for replayIdx := range mgr.replays { replay := &mgr.replays[replayIdx] if dev.id() == replay.id { @@ -89,14 +89,14 @@ func (mgr *MockNvmePersistenceManager) load(dev *mockDevice) error { } ns.id = nvme.NamespaceIdentifier(namespace.NamespaceId) - ns.capacity = namespace.Capacity + ns.capacityInBytes = namespace.Capacity ns.guid = namespace.GUID for _, ctrlId := range namespace.controllerIds { ns.attachedControllers[ctrlId] = &dev.controllers[ctrlId] } - dev.allocatedCapacity += ns.capacity + dev.allocatedCapacityInBytes += ns.capacityInBytes } return nil @@ -124,7 +124,7 @@ func (mgr *MockNvmePersistenceManager) new(dev *mockDevice) error { } func (mgr *MockNvmePersistenceManager) recordCreateNamespace(dev *mockDevice, ns *mockNamespace) { - ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix+dev.id()) + ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix + dev.id()) if err != nil { panic(err) } @@ -132,7 +132,7 @@ func (mgr *MockNvmePersistenceManager) recordCreateNamespace(dev *mockDevice, ns data, _ := json.Marshal(&mockNvmeDevicePersistentNamespaceData{ NamespaceId: uint32(ns.id), GUID: ns.guid, - Capacity: ns.capacity, + Capacity: ns.capacityInBytes, }) if err := ledger.Log(mockNvmePersistenceNamespaceCreate, data); err != nil { @@ -143,7 +143,7 @@ func (mgr *MockNvmePersistenceManager) recordCreateNamespace(dev *mockDevice, ns } func (mgr *MockNvmePersistenceManager) recordDeleteNamespace(dev *mockDevice, ns *mockNamespace) { - ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix+dev.id()) + ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix + dev.id()) if err != nil { panic(err) } @@ -162,7 +162,7 @@ func (mgr *MockNvmePersistenceManager) recordDeleteNamespace(dev *mockDevice, ns } func (mgr *MockNvmePersistenceManager) recordAttachController(dev *mockDevice, ns *mockNamespace, ctrlId uint16) { - ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix+dev.id()) + ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix + dev.id()) if err != nil { panic(err) } @@ -180,7 +180,7 @@ func (mgr *MockNvmePersistenceManager) recordAttachController(dev *mockDevice, n } func (mgr *MockNvmePersistenceManager) recordDetachController(dev *mockDevice, ns *mockNamespace, ctrlId uint16) { - ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix+dev.id()) + ledger, err := mgr.store.OpenKey(mockNvmePersistenceRegistryPrefix + dev.id()) if err != nil { panic(err) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 60a857ecf..e20626c9d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 +# github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From b709dca55ffc1cc370fe4b0b6a391318e61a50ef Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Wed, 21 Dec 2022 10:14:54 -0600 Subject: [PATCH 04/71] Upgrade nnf-ec to latest (83d47b) Signed-off-by: Nate Thornton --- go.mod | 2 +- go.sum | 2 ++ .../NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go | 7 +++---- vendor/modules.txt | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9e6ab9032..1d6b51925 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 + github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 5887cd508..8c68f6f04 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 h1:yMRoXhdvJS github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 h1:f09NvmieiaSYE8Gxd6+vhmrtx1PB4W4Sldc+rJAAxZQ= github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 h1:Eg/6n0p/qGbFhL1w+zxiPj2jjK+5rAEM3q7+vO0CWy4= +github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go index 54dae95c3..60b454880 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/nvme_mock.go @@ -129,10 +129,9 @@ type mockDevice struct { } type mockController struct { - id uint16 - online bool - capacity uint64 - //allocatedCapacity uint64 + id uint16 + online bool + capacity uint64 vqresources uint32 viresources uint32 } diff --git a/vendor/modules.txt b/vendor/modules.txt index e20626c9d..f58221982 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 +# github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From 227b5032816f6061b7e5d9bad3c25d5f2370108c Mon Sep 17 00:00:00 2001 From: Dean Roehrich Date: Thu, 5 Jan 2023 09:09:39 -0600 Subject: [PATCH 05/71] Use DWS variable for storage label (#114) Signed-off-by: Dean Roehrich --- controllers/directivebreakdown_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/directivebreakdown_controller.go b/controllers/directivebreakdown_controller.go index f7f23daee..179cb888f 100644 --- a/controllers/directivebreakdown_controller.go +++ b/controllers/directivebreakdown_controller.go @@ -496,7 +496,7 @@ func populateStorageAllocationSet(a *dwsv1alpha1.StorageAllocationSet, strategy a.AllocationStrategy = strategy a.Label = labelStr a.MinimumCapacity = cap - a.Constraints.Labels = []string{"dws.cray.hpe.com/storage=Rabbit"} + a.Constraints.Labels = []string{dwsv1alpha1.StorageTypeLabel + "=Rabbit"} if constraint != nil { a.Constraints.Colocation = []dwsv1alpha1.AllocationSetColocationConstraint{*constraint} } From 0cb357237f5420c986b1344182b8bc5fb8b1ff98 Mon Sep 17 00:00:00 2001 From: Dean Roehrich Date: Thu, 5 Jan 2023 09:41:19 -0600 Subject: [PATCH 06/71] Use the DWS workflowname vars for label names (#115) Signed-off-by: Dean Roehrich --- controllers/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/integration_test.go b/controllers/integration_test.go index 1a1e24258..865076fa7 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -735,8 +735,8 @@ var _ = Describe("Integration Test", func() { } Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(clientMount), clientMount)).To(Succeed()) Expect(clientMount.Status.Mounts).To(HaveLen(1)) - Expect(clientMount.Labels["dws.cray.hpe.com/workflow.name"]).To(Equal(workflow.Name)) - Expect(clientMount.Labels["dws.cray.hpe.com/workflow.namespace"]).To(Equal(workflow.Namespace)) + Expect(clientMount.Labels[dwsv1alpha1.WorkflowNameLabel]).To(Equal(workflow.Name)) + Expect(clientMount.Labels[dwsv1alpha1.WorkflowNamespaceLabel]).To(Equal(workflow.Namespace)) Expect(clientMount.Status.Mounts[0].Ready).To(BeTrue()) } From cd40c17943d12b61d4f56bee982e3b1ee30a7308 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Fri, 6 Jan 2023 16:47:57 -0600 Subject: [PATCH 07/71] RABSW-1069: Support for refactored DWS Storage resource and NNF Fencing functionality (#113) Support for NNF Node Fencing with DWS Storage interaction Signed-off-by: Nate Thornton --- .vscode/launch.json | 4 +- api/v1alpha1/nnf_node_types.go | 3 + api/v1alpha1/nnf_resource_status_type.go | 21 ++ .../bases/nnf.cray.hpe.com_nnfaccesses.yaml | 19 +- .../crd/bases/nnf.cray.hpe.com_nnfnodes.yaml | 4 + config/rbac/role.yaml | 8 + controllers/integration_test.go | 4 +- controllers/nnf_access_controller.go | 4 +- controllers/nnf_node_slc.go | 258 ++++++++++-------- controllers/nnf_node_slc_test.go | 9 +- controllers/nnf_node_storage_controller.go | 4 +- controllers/nnf_workflow_controller_test.go | 30 +- go.mod | 2 +- go.sum | 14 +- .../dws/api/v1alpha1/resource.go | 71 +++++ .../dws/api/v1alpha1/storage_types.go | 73 +++-- .../dws/api/v1alpha1/storagepool_types.go | 63 ----- .../api/v1alpha1/systemconfiguration_types.go | 1 - .../dws/api/v1alpha1/workflow_types.go | 10 +- .../dws/api/v1alpha1/workflow_webhook.go | 2 +- .../dws/api/v1alpha1/zz_generated.deepcopy.go | 103 +------ .../bases/dws.cray.hpe.com_storagepools.yaml | 67 ----- .../crd/bases/dws.cray.hpe.com_storages.yaml | 63 ++++- ...dws.cray.hpe.com_systemconfigurations.yaml | 2 - .../crd/bases/dws.cray.hpe.com_workflows.yaml | 21 +- .../utils/dwdparse/zz_generated.deepcopy.go | 2 +- vendor/modules.txt | 2 +- 27 files changed, 460 insertions(+), 404 deletions(-) create mode 100644 vendor/github.com/HewlettPackard/dws/api/v1alpha1/resource.go delete mode 100644 vendor/github.com/HewlettPackard/dws/api/v1alpha1/storagepool_types.go delete mode 100644 vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storagepools.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index 596b75d94..e9754e1ac 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -28,7 +28,9 @@ "-ginkgo.progress" ], "env": { - "KUBEBUILDER_ASSETS": "${workspaceFolder}/testbin/bin" + "KUBEBUILDER_ASSETS": "${workspaceFolder}/bin/k8s/1.25.0-darwin-amd64", + "GOMEGA_DEFAULT_EVENTUALLY_TIMEOUT": "10m", + "GOMEGA_DEFAULT_EVENTUALLY_POLLING_INTERVAL": "100ms" }, "showLog": true }, diff --git a/api/v1alpha1/nnf_node_types.go b/api/v1alpha1/nnf_node_types.go index 015ac8c76..75ebafabe 100644 --- a/api/v1alpha1/nnf_node_types.go +++ b/api/v1alpha1/nnf_node_types.go @@ -52,6 +52,9 @@ type NnfNodeStatus struct { Health NnfResourceHealthType `json:"health,omitempty"` + // Fenced is true when the NNF Node is fenced by the STONITH agent, and false otherwise. + Fenced bool `json:"fenced,omitempty"` + Capacity int64 `json:"capacity,omitempty"` CapacityAllocated int64 `json:"capacityAllocated,omitempty"` diff --git a/api/v1alpha1/nnf_resource_status_type.go b/api/v1alpha1/nnf_resource_status_type.go index 310f6e97c..34335763a 100644 --- a/api/v1alpha1/nnf_resource_status_type.go +++ b/api/v1alpha1/nnf_resource_status_type.go @@ -20,6 +20,8 @@ package v1alpha1 import ( + dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" + sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models" ) @@ -93,6 +95,25 @@ func (rst NnfResourceStatusType) UpdateIfWorseThan(status *NnfResourceStatusType } } +func (rst NnfResourceStatusType) ConvertToDWSResourceStatus() dwsv1alpha1.ResourceStatus { + switch rst { + case ResourceStarting: + return dwsv1alpha1.StartingStatus + case ResourceReady: + return dwsv1alpha1.ReadyStatus + case ResourceDisabled: + return dwsv1alpha1.DisabledStatus + case ResourceNotPresent: + return dwsv1alpha1.NotPresentStatus + case ResourceOffline: + return dwsv1alpha1.OfflineStatus + case ResourceFailed: + return dwsv1alpha1.FailedStatus + default: + return dwsv1alpha1.UnknownStatus + } +} + // StaticResourceStatus will convert a Swordfish ResourceStatus to the NNF Resource Status. func StaticResourceStatus(s sf.ResourceStatus) NnfResourceStatusType { switch s.State { diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml index 36fee7f85..88631932c 100644 --- a/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml +++ b/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml @@ -149,13 +149,22 @@ spec: - all type: string teardownState: + allOf: + - enum: + - Proposal + - Setup + - DataIn + - PreRun + - PostRun + - DataOut + - Teardown + - enum: + - DataIn + - PreRun + - PostRun + - DataOut description: TeardownState is the desired state of the workflow for this NNF Access resource to be torn down and deleted. - enum: - - DataIn - - PreRun - - PostRun - - DataOut type: string required: - desiredState diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfnodes.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfnodes.yaml index 74eb12cb9..4b399ea74 100644 --- a/config/crd/bases/nnf.cray.hpe.com_nnfnodes.yaml +++ b/config/crd/bases/nnf.cray.hpe.com_nnfnodes.yaml @@ -126,6 +126,10 @@ spec: type: integer type: object type: array + fenced: + description: Fenced is true when the NNF Node is fenced by the STONITH + agent, and false otherwise. + type: boolean health: description: NnfResourceHealthType defines the health of an NNF resource. type: string diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4d52812f1..0eeb1dfaf 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -200,6 +200,14 @@ rules: - patch - update - watch +- apiGroups: + - dws.cray.hpe.com + resources: + - storages/status + verbs: + - get + - patch + - update - apiGroups: - dws.cray.hpe.com resources: diff --git a/controllers/integration_test.go b/controllers/integration_test.go index 865076fa7..9369a52ee 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -360,7 +360,7 @@ var _ = Describe("Integration Test", func() { }, Spec: nnfv1alpha1.NnfNodeSpec{ Name: nodeName, - State: "Enable", + State: nnfv1alpha1.ResourceEnable, }, Status: nnfv1alpha1.NnfNodeStatus{}, } @@ -380,7 +380,7 @@ var _ = Describe("Integration Test", func() { Eventually(func() bool { Expect(k8sClient.Get(context.TODO(), namespacedName, storage)).To(Succeed()) - return len(storage.Data.Access.Computes) == 16 + return len(storage.Status.Access.Computes) == 16 }).Should(BeTrue()) // Check that a namespace was created for each compute node diff --git a/controllers/nnf_access_controller.go b/controllers/nnf_access_controller.go index 67a17d113..27e7e5a8b 100644 --- a/controllers/nnf_access_controller.go +++ b/controllers/nnf_access_controller.go @@ -608,11 +608,11 @@ func (r *NnfAccessReconciler) mapClientLocalStorage(ctx context.Context, access // Build a list of all nodes with access to the storage clients := []string{} - for _, compute := range storage.Data.Access.Computes { + for _, compute := range storage.Status.Access.Computes { clients = append(clients, compute.Name) } - for _, server := range storage.Data.Access.Servers { + for _, server := range storage.Status.Access.Servers { clients = append(clients, server.Name) } diff --git a/controllers/nnf_node_slc.go b/controllers/nnf_node_slc.go index 4880e41df..004bf341a 100644 --- a/controllers/nnf_node_slc.go +++ b/controllers/nnf_node_slc.go @@ -39,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" + "github.com/HewlettPackard/dws/utils/updater" nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" ) @@ -59,6 +60,7 @@ const ( //+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodes/status,verbs=get;update;patch //+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodes/finalizers,verbs=update //+kubebuilder:rbac:groups=dws.cray.hpe.com,resources=storages,verbs=get;create;list;watch;update;patch;delete +//+kubebuilder:rbac:groups=dws.cray.hpe.com,resources=storages/status,verbs=get;update;patch //+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch //+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch @@ -115,22 +117,6 @@ func (r *NnfNodeSLCReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - // Get the kubernetes node resource corresponding to the same node as the nnfNode resource. - // The kubelet has a heartbeat mechanism, so we can determine node failures from this resource. - node := &corev1.Node{} - if err := r.Get(ctx, types.NamespacedName{Name: nnfNode.Namespace}, node); err != nil { - return ctrl.Result{}, err - } - - // Look through the node conditions to determine if the node is up - nodeStatus := "Offline" - for _, condition := range node.Status.Conditions { - if condition.Type == "Ready" && condition.Status == "True" { - nodeStatus = "Ready" - break - } - } - // Create a DWS Storage resource based on the information from the NNFNode and Node resources storage := &dwsv1alpha1.Storage{ ObjectMeta: metav1.ObjectMeta{ @@ -139,91 +125,132 @@ func (r *NnfNodeSLCReconciler) Reconcile(ctx context.Context, req ctrl.Request) }, } - result, err := ctrl.CreateOrUpdate(ctx, r.Client, storage, - func() error { - storage.Data.Type = "NVMe" - storage.Data.Capacity = nnfNode.Status.Capacity - if nodeStatus == "Ready" { - storage.Data.Status = string(nnfNode.Status.Status) - } else { - storage.Data.Status = nodeStatus - } + if err := r.Get(ctx, client.ObjectKeyFromObject(storage), storage); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } - // Add a label for a storage type of Rabbit. Don't overwrite the - // existing labels map since there may be user applied labels. - labels := storage.GetLabels() - if labels == nil { - labels = make(map[string]string) - } - labels[dwsv1alpha1.StorageTypeLabel] = "Rabbit" - storage.SetLabels(labels) + storage.Spec = dwsv1alpha1.StorageSpec{ + State: dwsv1alpha1.EnabledState, + } - // Access information for how to get to the storage - storage.Data.Access.Protocol = "PCIe" + if err := r.Create(ctx, storage); err != nil { + return ctrl.Result{}, err + } - // Wait until the servers array has been filled in with the Rabbit info - if len(nnfNode.Status.Servers) == 0 { - return nil - } + log.Info("Created DWS Storage resource", "name", storage.Name) - storage.Data.Access.Servers = []dwsv1alpha1.Node{{ - // The Rabbit node is the name of the nnfNode namespace - Name: nnfNode.Namespace, - Status: string(nnfNode.Status.Servers[0].Status), - }} + return ctrl.Result{Requeue: true}, nil + } - // Wait until the servers array has been filled in with the compute node info - if len(nnfNode.Status.Servers) == 1 { - return nil - } + // Create a pair of updaters for updating the NNF Node status and Storage status + nnfNodeStatusUpdater := updater.NewStatusUpdater[*nnfv1alpha1.NnfNodeStatus](nnfNode) + defer func() { err = nnfNodeStatusUpdater.CloseWithStatusUpdate(ctx, r, err) }() + + storageStatusUpdater := updater.NewStatusUpdater[*dwsv1alpha1.StorageStatus](storage) + defer func() { err = storageStatusUpdater.CloseWithStatusUpdate(ctx, r, err) }() + + // Populate the status of the storage resource. + storage.Status.Type = dwsv1alpha1.NVMe + storage.Status.Capacity = nnfNode.Status.Capacity + + switch storage.Spec.State { + case dwsv1alpha1.EnabledState: + // Clear the fenced status if the node is enabled from a disabled status + if storage.Status.Status == dwsv1alpha1.DisabledStatus { + nnfNode.Status.Fenced = false + + storage.Status.RebootRequired = false + storage.Status.Message = "" - computes := []dwsv1alpha1.Node{} - for _, c := range nnfNode.Status.Servers[1:] { - if c.Hostname == "" { - continue - } - - compute := dwsv1alpha1.Node{ - Name: c.Hostname, - Status: string(c.Status), - } - computes = append(computes, compute) + // TODO: Fencing Agent Phase #2: Resume Rabbit NLC pods, wait for the pods to + // resume, then change Node Status to Enabled + } + + if nnfNode.Status.Fenced { + storage.Status.Status = dwsv1alpha1.DegradedStatus + storage.Status.RebootRequired = true + storage.Status.Message = "Storage node requires reboot to recover from STONITH event" + } else { + + // Check if the kubernetes node itself is offline + ready, err := r.isKubernetesNodeReady(ctx, nnfNode) + if err != nil { + return ctrl.Result{}, err } - storage.Data.Access.Computes = computes - - devices := []dwsv1alpha1.StorageDevice{} - for _, d := range nnfNode.Status.Drives { - if d.Status == nnfv1alpha1.ResourceOffline { - continue - } - - wearLevel := d.WearLevel - device := dwsv1alpha1.StorageDevice{ - Model: d.Model, - SerialNumber: d.SerialNumber, - FirmwareVersion: d.FirmwareVersion, - Capacity: d.Capacity, - Status: string(d.Status), - WearLevel: &wearLevel, - Slot: d.Slot, - } - devices = append(devices, device) + + if !ready { + storage.Status.Status = dwsv1alpha1.OfflineStatus + storage.Status.Message = "Kubernetes node is not ready" + } else { + storage.Status.Status = nnfNode.Status.Status.ConvertToDWSResourceStatus() } - storage.Data.Devices = devices + } + case dwsv1alpha1.DisabledState: + // TODO: Fencing Agent Phase #2: Pause Rabbit NLC pods, wait for pods to be + // removed, then change Node Status to Disabled + + storage.Status.Status = dwsv1alpha1.DisabledStatus + storage.Status.Message = "Storage node was manually disabled" + default: + log.Info("Unhandled storage state", "state", storage.Spec.State) + } - return nil - }) - if err != nil { - return ctrl.Result{}, err + // Add a label for a storage type of Rabbit. Don't overwrite the + // existing labels map since there may be user applied labels. + labels := storage.GetLabels() + if labels == nil { + labels = make(map[string]string) } + labels[dwsv1alpha1.StorageTypeLabel] = "Rabbit" + storage.SetLabels(labels) - if result == controllerutil.OperationResultCreated { - log.Info("Created DWS Storage resource", "name", storage.Name) - } else if result == controllerutil.OperationResultNone { - // no change - } else { - log.Info("Updated DWS Storage", "name", storage.Name) + // Access information for how to get to the storage + storage.Status.Access.Protocol = dwsv1alpha1.PCIe + + // Wait until the servers array has been filled in with the Rabbit info + if len(nnfNode.Status.Servers) == 0 { + return ctrl.Result{}, nil + } + + storage.Status.Access.Servers = []dwsv1alpha1.Node{{ + Name: nnfNode.Namespace, // The Rabbit node is the name of the nnfNode namespace + Status: nnfNode.Status.Servers[0].Status.ConvertToDWSResourceStatus(), + }} + + computes := []dwsv1alpha1.Node{} + for _, c := range nnfNode.Status.Servers[1:] { + if c.Hostname == "" { + continue + } + + compute := dwsv1alpha1.Node{ + Name: c.Hostname, + Status: c.Status.ConvertToDWSResourceStatus(), + } + computes = append(computes, compute) } + storage.Status.Access.Computes = computes + + devices := []dwsv1alpha1.StorageDevice{} + for _, d := range nnfNode.Status.Drives { + if d.Status == nnfv1alpha1.ResourceOffline { + continue + } + + wearLevel := d.WearLevel + device := dwsv1alpha1.StorageDevice{ + Model: d.Model, + SerialNumber: d.SerialNumber, + FirmwareVersion: d.FirmwareVersion, + Capacity: d.Capacity, + Status: d.Status.ConvertToDWSResourceStatus(), + WearLevel: &wearLevel, + Slot: d.Slot, + } + devices = append(devices, device) + } + storage.Status.Devices = devices return ctrl.Result{}, nil } @@ -238,46 +265,49 @@ func (r *NnfNodeSLCReconciler) deleteStorage(ctx context.Context, nnfNode *nnfv1 }, } - err := r.Delete(ctx, storage) - if err != nil { - if !apierrors.IsNotFound(err) { - return err - } - } else { - log.Info("Deleted DWS Storage resource") + if err := r.Delete(ctx, storage); err != nil { + return client.IgnoreNotFound(err) } + log.Info("Deleted DWS Storage resource", "name", storage.Name) + return nil } -// Map a DWS storage resource to a NnfNode resource. This is a cross-namespace -// mapping, so we can't use owner references -func nnfStorageMapFunc(o client.Object) []reconcile.Request { - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: "nnf-nlc", - Namespace: o.GetName(), - }}, +func (r *NnfNodeSLCReconciler) isKubernetesNodeReady(ctx context.Context, nnfNode *nnfv1alpha1.NnfNode) (bool, error) { + // Get the kubernetes node resource corresponding to the same node as the nnfNode resource. + // The kubelet has a heartbeat mechanism, so we can determine node failures from this resource. + node := &corev1.Node{} + if err := r.Get(ctx, types.NamespacedName{Name: nnfNode.Namespace}, node); err != nil { + return false, err } -} -// Map a kubernetes Node resource to a NnfNode resource -func nnfNodeMapFunc(o client.Object) []reconcile.Request { - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: "nnf-nlc", - Namespace: o.GetName(), - }}, + // Look through the node conditions to determine if the node is up. + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + return true, nil + } } + + return false, nil } // SetupWithManager sets up the controller with the Manager. func (r *NnfNodeSLCReconciler) SetupWithManager(mgr ctrl.Manager) error { + maxReconciles := runtime.GOMAXPROCS(0) + + mapFunc := func(o client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: "nnf-nlc", + Namespace: o.GetName(), + }}} + } + return ctrl.NewControllerManagedBy(mgr). WithOptions(controller.Options{MaxConcurrentReconciles: maxReconciles}). For(&nnfv1alpha1.NnfNode{}). - Watches(&source.Kind{Type: &dwsv1alpha1.Storage{}}, handler.EnqueueRequestsFromMapFunc(nnfStorageMapFunc)). - Watches(&source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(nnfNodeMapFunc)). + Watches(&source.Kind{Type: &dwsv1alpha1.Storage{}}, handler.EnqueueRequestsFromMapFunc(mapFunc)). + Watches(&source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(mapFunc)). Complete(r) } diff --git a/controllers/nnf_node_slc_test.go b/controllers/nnf_node_slc_test.go index a327c022b..1168c98e1 100644 --- a/controllers/nnf_node_slc_test.go +++ b/controllers/nnf_node_slc_test.go @@ -58,11 +58,10 @@ var _ = Describe("System Level Controller Test", func() { Expect(k8sClient.Create(context.TODO(), nnfNode)).To(Succeed()) storage := &dwsv1alpha1.Storage{} - Eventually(func() error { - return k8sClient.Get(context.TODO(), types.NamespacedName{Name: nnfNode.Namespace, Namespace: corev1.NamespaceDefault}, storage) - }).Should(Succeed(), "Create the DWS Storage object") - - Expect(storage.Data.Status).To(Equal("Offline")) + Eventually(func(g Gomega) dwsv1alpha1.ResourceStatus { + g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: nnfNode.Namespace, Namespace: corev1.NamespaceDefault}, storage)).To(Succeed()) + return storage.Status.Status + }).Should(Equal(dwsv1alpha1.OfflineStatus), "Create the DWS Storage object") By("Deleting the NNF Node") Expect(k8sClient.Delete(context.TODO(), nnfNode)).To(Succeed()) diff --git a/controllers/nnf_node_storage_controller.go b/controllers/nnf_node_storage_controller.go index 474a6a3a8..754969d37 100644 --- a/controllers/nnf_node_storage_controller.go +++ b/controllers/nnf_node_storage_controller.go @@ -315,11 +315,11 @@ func (r *NnfNodeStorageReconciler) createBlockDevice(ctx context.Context, nodeSt // Build a list of all nodes with access to the storage clients := []string{} - for _, server := range storage.Data.Access.Servers { + for _, server := range storage.Status.Access.Servers { clients = append(clients, server.Name) } - for _, compute := range storage.Data.Access.Computes { + for _, compute := range storage.Status.Access.Computes { clients = append(clients, compute.Name) } diff --git a/controllers/nnf_workflow_controller_test.go b/controllers/nnf_workflow_controller_test.go index 4b0f99634..3983211c0 100644 --- a/controllers/nnf_workflow_controller_test.go +++ b/controllers/nnf_workflow_controller_test.go @@ -538,6 +538,8 @@ var _ = Describe("NNF Workflow Unit Tests", func() { }) // When("Using copy_in directives", func() When("Using server allocation input", func() { + const nodeName = "rabbit-node" + var ( storage *dwsv1alpha1.Storage directiveBreakdown *dwsv1alpha1.DirectiveBreakdown @@ -574,7 +576,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { AllocationSize: directiveAllocationSet.MinimumCapacity, Storage: []dwsv1alpha1.ServersSpecStorage{ { - Name: "rabbit-node", + Name: nodeName, AllocationCount: 1, }, }, @@ -591,7 +593,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { BeforeEach(func() { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: "rabbit-node", + Name: nodeName, }, } @@ -601,23 +603,31 @@ var _ = Describe("NNF Workflow Unit Tests", func() { BeforeEach(func() { storage = &dwsv1alpha1.Storage{ ObjectMeta: metav1.ObjectMeta{ - Name: "rabbit-node", + Name: nodeName, Namespace: corev1.NamespaceDefault, }, - Data: dwsv1alpha1.StorageData{ + } + + Expect(k8sClient.Create(context.TODO(), storage)).To(Succeed()) + + Eventually(func(g Gomega) error { + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(storage), storage)).To(Succeed()) + + storage.Status = dwsv1alpha1.StorageStatus{ Capacity: 100000000000, Access: dwsv1alpha1.StorageAccess{ - Protocol: "PCIe", + Protocol: dwsv1alpha1.PCIe, Servers: []dwsv1alpha1.Node{ { - Name: "rabbit-node", - Status: "Ready", + Name: nodeName, + Status: dwsv1alpha1.ReadyStatus, }, }, }, - }, - } - Expect(k8sClient.Create(context.TODO(), storage)).To(Succeed()) + } + + return k8sClient.Status().Update(context.TODO(), storage) + }).Should(Succeed()) }) AfterEach(func() { diff --git a/go.mod b/go.mod index 1d6b51925..1c25d6d59 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/NearNodeFlash/nnf-sos go 1.19 require ( - github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47 + github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index 8c68f6f04..12df7bc60 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,18 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47 h1:OE72Yx1nLW1ZJBbUBTCzgOGg7JcSEY7qAXLymfdLSz4= -github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230103152723-c01b98e1d754 h1:ggvsAahh1WyRdSzP2eblyi0ysZAfwsKfS3k55492Osc= +github.com/HewlettPackard/dws v0.0.0-20230103152723-c01b98e1d754/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230103221621-3ddba3d818ed h1:fkTNMUhY9AVYpu6vc7B36pxdhZMEwtXBZuXbs5RtVcM= +github.com/HewlettPackard/dws v0.0.0-20230103221621-3ddba3d818ed/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230104151937-fc882a72469a h1:emYNSAkP0Nb9CRfrMGGMy7PRTqNISTYkbbqEHx/LY5c= +github.com/HewlettPackard/dws v0.0.0-20230104151937-fc882a72469a/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230104161449-b57ee6b0744b h1:3nbllp+Fw6K8oiYg8x3Ctoz3oYEcJLtCwyKxyVPCl9Y= +github.com/HewlettPackard/dws v0.0.0-20230104161449-b57ee6b0744b/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230104182823-40864bb5d259 h1:MysF14Ceu52QJPvAlHp6RQCEfSs2EfiDJl1jgFizgKE= +github.com/HewlettPackard/dws v0.0.0-20230104182823-40864bb5d259/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5 h1:J+ok11bmTTSAc4ZURVSGstjFYpQg0nN8lFIWoEhUzXA= +github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= github.com/HewlettPackard/structex v1.0.2 h1:p2EH/p6zvUd5fSa0onudAUvLWdKsvQSRP0jGKeblA5E= github.com/HewlettPackard/structex v1.0.2/go.mod h1:3frC4RY/cPsP/4+N8rkxsNAGlQwHV+zDC7qvrN+N+rE= github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/resource.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/resource.go new file mode 100644 index 000000000..d9266241e --- /dev/null +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/resource.go @@ -0,0 +1,71 @@ +/* + * Copyright 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1alpha1 + +// ResourceState is the enumeration of the state of a DWS resource +// +kubebuilder:validation:Enum:=Enabled;Disabled +type ResourceState string + +const ( + // NOTE: Any new enumeration values must update the related kubebuilder validation + + // Enabled means the resource shall be enabled. + EnabledState ResourceState = "Enabled" + + // Disabled means the resource shall be disabled. + DisabledState ResourceState = "Disabled" +) + +// ResourceStatus is the enumeration of the status of a DWS resource +// +kubebuilder:validation:Enum:=Starting;Ready;Disabled;NotPresent;Offline;Failed;Degraded;Unknown +type ResourceStatus string + +const ( + // NOTE: Any new enumeration values must update the related kubebuilder validation + + // Starting means the resource is currently starting prior to becoming ready. + StartingStatus ResourceStatus = "Starting" + + // Ready means the resource is fully operational and ready for use. + ReadyStatus ResourceStatus = "Ready" + + // Disabled means the resource is present but disabled by an administrator or external + // user. + DisabledStatus ResourceStatus = "Disabled" + + // NotPresent means the resource is not present within the system, likely because + // it is missing or powered down. This differs from the Offline state in that the + // resource is not known to exist. + NotPresentStatus ResourceStatus = "NotPresent" + + // Offline means the resource is offline and cannot be communicated with. This differs + // fro the NotPresent state in that the resource is known to exist. + OfflineStatus ResourceStatus = "Offline" + + // Failed means the resource has failed during startup or execution. + FailedStatus ResourceStatus = "Failed" + + // Degraded means the resource is ready but operating in a degraded state. Certain + // recovery actions may alleviate a degraded status. + DegradedStatus ResourceStatus = "Degraded" + + // Unknown means the resource status is unknown. + UnknownStatus ResourceStatus = "Unknown" +) diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/storage_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/storage_types.go index e9ec9f53d..b0c36d7c6 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/storage_types.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/storage_types.go @@ -20,6 +20,7 @@ package v1alpha1 import ( + "github.com/HewlettPackard/dws/utils/updater" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -27,13 +28,15 @@ const ( // StorageTypeLabel is the label key used for tagging Storage resources // with a driver specific label. For example: dws.cray.hpe.com/storage=Rabbit StorageTypeLabel = "dws.cray.hpe.com/storage" - - // StoragePoolLabelPrefix is the prefix for the label key used for tagging - // Storage resources with a storage pool label. - // For example: dws.cray.hpe.com/storage-pool-default=true - StoragePoolLabelPrefix = "dws.cray.hpe.com/storage-pool-" ) +// StorageSpec defines the desired specifications of Storage resource +type StorageSpec struct { + // State describes the desired state of the Storage resource. + // +kubebuilder:default:=Enabled + State ResourceState `json:"state,omitempty"` +} + // StorageDevice contains the details of the storage hardware type StorageDevice struct { // Model is the manufacturer information about the device @@ -57,8 +60,7 @@ type StorageDevice struct { WearLevel *int64 `json:"wearLevel,omitempty"` // Status of the individual device - // +kubebuilder:validation:Enum=Starting;Ready;Disabled;NotPresent;Offline;Failed - Status string `json:"status,omitempty"` + Status ResourceStatus `json:"status,omitempty"` } // Node provides the status of either a compute or a server @@ -67,30 +69,41 @@ type Node struct { Name string `json:"name,omitempty"` // Status of the node - // +kubebuilder:validation:Enum=Starting;Ready;Disabled;NotPresent;Offline;Failed - Status string `json:"status,omitempty"` + Status ResourceStatus `json:"status,omitempty"` } +// StorageAccessProtocol is the enumeration of supported protocols. +// +kubebuilder:validation:Enum:=PCIe +type StorageAccessProtocol string + +const ( + PCIe StorageAccessProtocol = "PCIe" +) + // StorageAccess contains nodes and the protocol that may access the storage type StorageAccess struct { // Protocol is the method that this storage can be accessed - // +kubebuilder:validation:Enum=PCIe - Protocol string `json:"protocol,omitempty"` + Protocol StorageAccessProtocol `json:"protocol,omitempty"` - // Servers is the list of non-compute nodes that have access to - // the storage + // Servers is the list of non-compute nodes that have access to the storage Servers []Node `json:"servers,omitempty"` - // Computes is the list of compute nodes that have access to - // the storage + // Computes is the list of compute nodes that have access to the storage Computes []Node `json:"computes,omitempty"` } +// StorageType is the enumeration of storage types. +// +kubebuilder:validation:Enum:=NVMe +type StorageType string + +const ( + NVMe StorageType = "NVMe" +) + // StorageData contains the data about the storage -type StorageData struct { +type StorageStatus struct { // Type describes what type of storage this is - // +kubebuilder:validation:Enum=NVMe - Type string `json:"type,omitempty"` + Type StorageType `json:"type,omitempty"` // Devices is the list of physical devices that make up this storage Devices []StorageDevice `json:"devices,omitempty"` @@ -105,17 +118,33 @@ type StorageData struct { Capacity int64 `json:"capacity"` // Status is the overall status of the storage - // +kubebuilder:validation:Enum=Starting;Ready;Disabled;NotPresent;Offline;Failed - Status string `json:"status,omitempty"` + Status ResourceStatus `json:"status,omitempty"` + + // Reboot Required is true if the node requires a reboot and false otherwise. A reboot my be + // necessary to recover from certain hardware failures or high-availability clustering events. + RebootRequired bool `json:"rebootRequired,omitempty"` + + // Message provides additional details on the current status of the resource + Message string `json:"message,omitempty"` } +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="State",type="string",JSONPath=".spec.state",description="State of the storage resource" +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status",description="Status of the storage resource" +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + // Storage is the Schema for the storages API -// +kubebuilder:object:root=true type Storage struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Data StorageData `json:"data,omitempty"` + Spec StorageSpec `json:"spec"` + Status StorageStatus `json:"status,omitempty"` +} + +func (s *Storage) GetStatus() updater.Status[*StorageStatus] { + return &s.Status } //+kubebuilder:object:root=true diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/storagepool_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/storagepool_types.go deleted file mode 100644 index f04d3f193..000000000 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/storagepool_types.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP - * Other additional copyright holders may be indicated within. - * - * The entirety of this work is licensed under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// StoragePoolSpec defines the desired state of StoragePool -type StoragePoolSpec struct { - PoolID string `json:"poolID"` - Units string `json:"units"` - Granularity string `json:"granularity"` - Quantity int `json:"quantity"` - Free int `json:"free"` -} - -// StoragePoolStatus defines the observed state of StoragePool -type StoragePoolStatus struct { - State string `json:"state"` -} - -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status - -// StoragePool is the Schema for the storagepools API -type StoragePool struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec StoragePoolSpec `json:"spec,omitempty"` - Status StoragePoolStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// StoragePoolList contains a list of StoragePool -type StoragePoolList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []StoragePool `json:"items"` -} - -func init() { - SchemeBuilder.Register(&StoragePool{}, &StoragePoolList{}) -} diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/systemconfiguration_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/systemconfiguration_types.go index b82b3a37d..09e7c53f0 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/systemconfiguration_types.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/systemconfiguration_types.go @@ -42,7 +42,6 @@ type SystemConfigurationComputeNodeReference struct { // SystemConfigurationStorageNode describes a storage node in the system type SystemConfigurationStorageNode struct { // Type is the type of server - // +kubebuilder:validation:Enum=Rabbit Type string `json:"type"` // Name of the server node diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go index 33d4604a4..86c5be191 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go @@ -34,6 +34,7 @@ const ( ) // WorkflowState is the enumeration of the state of the workflow +// +kubebuilder:validation:Enum:=Proposal;Setup;DataIn;PreRun;PostRun;DataOut;Teardown type WorkflowState string // WorkflowState values @@ -100,8 +101,6 @@ const ( type WorkflowSpec struct { // Desired state for the workflow to be in. Unless progressing to the teardown state, // this can only be set to the next state when the current desired state has been achieved. - // +kubebuilder:validation:Enum:=Proposal;Setup;DataIn;PreRun;PostRun;DataOut;Teardown - // +kubebuilder:validation:Type:=string DesiredState WorkflowState `json:"desiredState"` WLMID string `json:"wlmID"` @@ -144,6 +143,7 @@ type WorkflowDriverStatus struct { // +kubebuilder:validation:Enum=Pending;Queued;Running;Completed;Error;DriverWait Status string `json:"status,omitempty"` + // Message provides additional details on the current status of the resource Message string `json:"message,omitempty"` // Driver error string. This is not rolled up into the workflow's @@ -158,7 +158,7 @@ type WorkflowDriverStatus struct { type WorkflowStatus struct { // The state the resource is currently transitioning to. // Updated by the controller once started. - State WorkflowState `json:"state"` + State WorkflowState `json:"state,omitempty"` // Ready can be 'True', 'False' // Indicates whether State has been reached. @@ -166,7 +166,9 @@ type WorkflowStatus struct { // User readable reason and status message // +kubebuilder:validation:Enum=Completed;DriverWait;Error - Status string `json:"status,omitempty"` + Status string `json:"status,omitempty"` + + // Message provides additional details on the current status of the resource Message string `json:"message,omitempty"` // Set of DW environment variable settings for WLM to apply to the job. diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go index 116cdb5fe..5298697b2 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go @@ -87,7 +87,7 @@ func (w *Workflow) ValidateCreate() error { return field.Forbidden(specPath.Child("Hurry"), "the hurry flag may not be set on creation") } if w.Status.State != "" { - return field.Forbidden(field.NewPath("Status").Child("State"), "the status state may not be set") + return field.Forbidden(field.NewPath("Status").Child("State"), "the status state may not be set on creation") } return checkDirectives(w, &ValidatingRuleParser{}) diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go index 3913c095e..a91fa9351 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* - * Copyright 2022 Hewlett Packard Enterprise Development LP + * Copyright 2023 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -922,7 +922,8 @@ func (in *Storage) DeepCopyInto(out *Storage) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Data.DeepCopyInto(&out.Data) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Storage. @@ -1007,29 +1008,6 @@ func (in *StorageBreakdown) DeepCopy() *StorageBreakdown { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StorageData) DeepCopyInto(out *StorageData) { - *out = *in - if in.Devices != nil { - in, out := &in.Devices, &out.Devices - *out = make([]StorageDevice, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.Access.DeepCopyInto(&out.Access) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageData. -func (in *StorageData) DeepCopy() *StorageData { - if in == nil { - return nil - } - out := new(StorageData) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageDevice) DeepCopyInto(out *StorageDevice) { *out = *in @@ -1083,90 +1061,39 @@ func (in *StorageList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoragePool) DeepCopyInto(out *StoragePool) { +func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoragePool. -func (in *StoragePool) DeepCopy() *StoragePool { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageSpec. +func (in *StorageSpec) DeepCopy() *StorageSpec { if in == nil { return nil } - out := new(StoragePool) + out := new(StorageSpec) in.DeepCopyInto(out) return out } -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *StoragePool) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoragePoolList) DeepCopyInto(out *StoragePoolList) { +func (in *StorageStatus) DeepCopyInto(out *StorageStatus) { *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]StoragePool, len(*in)) + if in.Devices != nil { + in, out := &in.Devices, &out.Devices + *out = make([]StorageDevice, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.Access.DeepCopyInto(&out.Access) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoragePoolList. -func (in *StoragePoolList) DeepCopy() *StoragePoolList { - if in == nil { - return nil - } - out := new(StoragePoolList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *StoragePoolList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoragePoolSpec) DeepCopyInto(out *StoragePoolSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoragePoolSpec. -func (in *StoragePoolSpec) DeepCopy() *StoragePoolSpec { - if in == nil { - return nil - } - out := new(StoragePoolSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoragePoolStatus) DeepCopyInto(out *StoragePoolStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoragePoolStatus. -func (in *StoragePoolStatus) DeepCopy() *StoragePoolStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageStatus. +func (in *StorageStatus) DeepCopy() *StorageStatus { if in == nil { return nil } - out := new(StoragePoolStatus) + out := new(StorageStatus) in.DeepCopyInto(out) return out } diff --git a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storagepools.yaml b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storagepools.yaml deleted file mode 100644 index aedab31ed..000000000 --- a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storagepools.yaml +++ /dev/null @@ -1,67 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: storagepools.dws.cray.hpe.com -spec: - group: dws.cray.hpe.com - names: - kind: StoragePool - listKind: StoragePoolList - plural: storagepools - singular: storagepool - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: StoragePool is the Schema for the storagepools API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: StoragePoolSpec defines the desired state of StoragePool - properties: - free: - type: integer - granularity: - type: string - poolID: - type: string - quantity: - type: integer - units: - type: string - required: - - free - - granularity - - poolID - - quantity - - units - type: object - status: - description: StoragePoolStatus defines the observed state of StoragePool - properties: - state: - type: string - required: - - state - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storages.yaml b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storages.yaml index c017e557d..5bc7bda3d 100644 --- a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storages.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_storages.yaml @@ -15,7 +15,19 @@ spec: singular: storage scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: State of the storage resource + jsonPath: .spec.state + name: State + type: string + - description: Status of the storage resource + jsonPath: .status.status + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: description: Storage is the Schema for the storages API @@ -25,7 +37,26 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string - data: + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: StorageSpec defines the desired specifications of Storage + resource + properties: + state: + default: Enabled + description: State describes the desired state of the Storage resource. + enum: + - Enabled + - Disabled + type: string + type: object + status: description: StorageData contains the data about the storage properties: access: @@ -51,6 +82,8 @@ spec: - NotPresent - Offline - Failed + - Degraded + - Unknown type: string type: object type: array @@ -78,6 +111,8 @@ spec: - NotPresent - Offline - Failed + - Degraded + - Unknown type: string type: object type: array @@ -123,6 +158,8 @@ spec: - NotPresent - Offline - Failed + - Degraded + - Unknown type: string wearLevel: description: WearLevel in percent for SSDs. A value of 100 indicates @@ -132,6 +169,15 @@ spec: type: integer type: object type: array + message: + description: Message provides additional details on the current status + of the resource + type: string + rebootRequired: + description: Reboot Required is true if the node requires a reboot + and false otherwise. A reboot my be necessary to recover from certain + hardware failures or high-availability clustering events. + type: boolean status: description: Status is the overall status of the storage enum: @@ -141,6 +187,8 @@ spec: - NotPresent - Offline - Failed + - Degraded + - Unknown type: string type: description: Type describes what type of storage this is @@ -150,13 +198,10 @@ spec: required: - capacity type: object - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object + required: + - spec type: object served: true storage: true + subresources: + status: {} diff --git a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml index f4586a6f0..cfc5b2395 100644 --- a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml @@ -88,8 +88,6 @@ spec: type: string type: description: Type is the type of server - enum: - - Rabbit type: string required: - name diff --git a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_workflows.yaml b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_workflows.yaml index 426b5cd54..8464e7969 100644 --- a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_workflows.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_workflows.yaml @@ -261,6 +261,8 @@ spec: format: int64 type: integer message: + description: Message provides additional details on the current + status of the resource type: string status: description: 'User readable reason. For the CDS driver, this @@ -279,6 +281,14 @@ spec: watchState: description: WorkflowState is the enumeration of the state of the workflow + enum: + - Proposal + - Setup + - DataIn + - PreRun + - PostRun + - DataOut + - Teardown type: string required: - completed @@ -300,6 +310,8 @@ spec: - DW_JOB_LDBAL_CACHE - DW_PERSISTENT_STRIPED_{resname} type: object message: + description: Message provides additional details on the current status + of the resource type: string ready: description: Ready can be 'True', 'False' Indicates whether State @@ -313,6 +325,14 @@ spec: state: description: The state the resource is currently transitioning to. Updated by the controller once started. + enum: + - Proposal + - Setup + - DataIn + - PreRun + - PostRun + - DataOut + - Teardown type: string status: description: User readable reason and status message @@ -323,7 +343,6 @@ spec: type: string required: - ready - - state type: object type: object served: true diff --git a/vendor/github.com/HewlettPackard/dws/utils/dwdparse/zz_generated.deepcopy.go b/vendor/github.com/HewlettPackard/dws/utils/dwdparse/zz_generated.deepcopy.go index 5d73dae99..d27486be1 100644 --- a/vendor/github.com/HewlettPackard/dws/utils/dwdparse/zz_generated.deepcopy.go +++ b/vendor/github.com/HewlettPackard/dws/utils/dwdparse/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* - * Copyright 2022 Hewlett Packard Enterprise Development LP + * Copyright 2023 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, diff --git a/vendor/modules.txt b/vendor/modules.txt index f58221982..f76c9ce3c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.6.0 ## explicit; go 1.12 github.com/Azure/go-autorest/tracing -# github.com/HewlettPackard/dws v0.0.0-20221110142740-abf44bba3b47 +# github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5 ## explicit; go 1.19 github.com/HewlettPackard/dws/api/v1alpha1 github.com/HewlettPackard/dws/config/crd/bases From 201d3471f978d37bdebb90c9cb32228d1cfa91e0 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Tue, 10 Jan 2023 10:01:26 -0600 Subject: [PATCH 08/71] Add known controller-manager secret Signed-off-by: Nate Thornton --- config/rbac/kustomization.yaml | 3 +++ config/rbac/kustomizeconfig.yaml | 4 ++++ config/rbac/service_account.yaml | 10 ++++++++++ 3 files changed, 17 insertions(+) create mode 100644 config/rbac/kustomizeconfig.yaml diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 731832a6a..8ac967430 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -16,3 +16,6 @@ resources: - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml + +configurations: +- kustomizeconfig.yaml \ No newline at end of file diff --git a/config/rbac/kustomizeconfig.yaml b/config/rbac/kustomizeconfig.yaml new file mode 100644 index 000000000..a8cc96ae9 --- /dev/null +++ b/config/rbac/kustomizeconfig.yaml @@ -0,0 +1,4 @@ +namePrefix: +- path: metadata/annotations/kubernetes.io\/service-account.name +namespace: +- path: metadata/annotations/kubernetes.io\/service-account.namespace \ No newline at end of file diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 7cd6025bf..7e7080eb7 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -3,3 +3,13 @@ kind: ServiceAccount metadata: name: controller-manager namespace: system +--- +apiVersion: v1 +kind: Secret +metadata: + name: controller-manager + namespace: system + annotations: + kubernetes.io/service-account.name: controller-manager + kubernetes.io/service-account.namespace: system +type: kubernetes.io/service-account-token \ No newline at end of file From a4affd699fa4f706246a25530b10cdfa59eafff7 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Wed, 11 Jan 2023 15:36:03 -0600 Subject: [PATCH 09/71] Disable EC Data Controller for unit tests Signed-off-by: Nate Thornton --- controllers/nnf_node_ec_data_controller.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controllers/nnf_node_ec_data_controller.go b/controllers/nnf_node_ec_data_controller.go index ccad9b0bf..5dec1b73c 100644 --- a/controllers/nnf_node_ec_data_controller.go +++ b/controllers/nnf_node_ec_data_controller.go @@ -78,10 +78,10 @@ func (r *NnfNodeECDataReconciler) Start(ctx context.Context) error { return err } } - } - // This resource acts as the storage provider for the NNF Element Controller - persistent.StorageProvider = r + // This resource acts as the storage provider for the NNF Element Controller + persistent.StorageProvider = r + } // Start the NNF Element Controller c := nnfec.NewController(r.Options) From c2534b87ff222a40d333217cfeaebaf91aca53f4 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Wed, 11 Jan 2023 15:55:43 -0600 Subject: [PATCH 10/71] Ignore not found resource on undeploy Signed-off-by: Nate Thornton --- deploy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy.sh b/deploy.sh index ec1af0212..e1bdece0d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -44,6 +44,6 @@ fi if [[ $CMD == 'undeploy' ]] then - kubectl delete -f config/samples/placeholder_nnfstorageprofile.yaml || true - $KUSTOMIZE build config/$OVERLAY | kubectl delete -f - + kubectl delete --ignore-not-found -f config/samples/placeholder_nnfstorageprofile.yaml + $KUSTOMIZE build config/$OVERLAY | kubectl delete --ignore-not-found -f - fi From de89b59a015b6a63b7dcca5bd6b403773ffb39b0 Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Wed, 11 Jan 2023 16:38:34 -0600 Subject: [PATCH 11/71] RABSW-1081: Support multiple MDTs (#117) * RABSW-1081: Support multiple MDTs - Update the DirectiveBreakdown to ask for more than one MDT if necessary - Only use a combined MGT/MDT for the first allocation listed in the mgtmdt allocation set. All other allocations will only be MDTs. - Fix an accounting error in the Servers resource where the allocated capacity was not summed across multiple NnfNodeStorages on the same Rabbit. Signed-off-by: Matt Richerson Signed-off-by: Matt Richerson * Re-vendor Signed-off-by: Matt Richerson Signed-off-by: Matt Richerson Signed-off-by: Matt Richerson Co-authored-by: Matt Richerson --- controllers/directivebreakdown_controller.go | 6 +- controllers/nnf_servers_controller.go | 12 ++- controllers/nnf_storage_controller.go | 9 +- .../nnf_workflow_controller_helpers.go | 16 ++- controllers/nnf_workflow_controller_test.go | 97 +++++++++++++++++++ go.mod | 2 +- go.sum | 4 + .../NearNodeFlash/nnf-ec/pkg/ec/error.go | 26 +++++ .../nnf-ec/pkg/manager-nnf/manager.go | 37 ++++--- .../pkg/manager-server/file_system_lustre.go | 2 +- vendor/modules.txt | 2 +- 11 files changed, 182 insertions(+), 31 deletions(-) diff --git a/controllers/directivebreakdown_controller.go b/controllers/directivebreakdown_controller.go index 179cb888f..b7ad6a631 100644 --- a/controllers/directivebreakdown_controller.go +++ b/controllers/directivebreakdown_controller.go @@ -431,11 +431,11 @@ func (r *DirectiveBreakdownReconciler) populateStorageBreakdown(ctx context.Cont if mdtKey != nil { useKey = mdtKey } - lustreComponents = append(lustreComponents, lustreComponentType{dwsv1alpha1.AllocateSingleServer, mdtCapacity, "mgtmdt", useKey}) + lustreComponents = append(lustreComponents, lustreComponentType{dwsv1alpha1.AllocateAcrossServers, mdtCapacity, "mgtmdt", useKey}) } else if len(nnfStorageProfile.Data.LustreStorage.ExternalMGS) > 0 { - lustreComponents = append(lustreComponents, lustreComponentType{dwsv1alpha1.AllocateSingleServer, mdtCapacity, "mdt", mdtKey}) + lustreComponents = append(lustreComponents, lustreComponentType{dwsv1alpha1.AllocateAcrossServers, mdtCapacity, "mdt", mdtKey}) } else { - lustreComponents = append(lustreComponents, lustreComponentType{dwsv1alpha1.AllocateSingleServer, mdtCapacity, "mdt", mdtKey}) + lustreComponents = append(lustreComponents, lustreComponentType{dwsv1alpha1.AllocateAcrossServers, mdtCapacity, "mdt", mdtKey}) lustreComponents = append(lustreComponents, lustreComponentType{dwsv1alpha1.AllocateSingleServer, mgtCapacity, "mgt", mgtKey}) } diff --git a/controllers/nnf_servers_controller.go b/controllers/nnf_servers_controller.go index 705fc5afb..1c44598c9 100644 --- a/controllers/nnf_servers_controller.go +++ b/controllers/nnf_servers_controller.go @@ -237,6 +237,8 @@ func (r *DWSServersReconciler) updateCapacityUsed(ctx context.Context, servers * ready = false } + capacityAllocatedMap := make(map[string]int64) + for _, nnfNodeStorage := range nnfNodeStorageList.Items { // There can be multiple allocations per Rabbit. Add them all up and present a // single size for the servers resource @@ -248,7 +250,15 @@ func (r *DWSServersReconciler) updateCapacityUsed(ctx context.Context, servers * allocationSize += nnfNodeAllocation.CapacityAllocated } - servers.Status.AllocationSets[serversIndex].Storage[nnfNodeStorage.Namespace] = dwsv1alpha1.ServersStatusStorage{AllocationSize: allocationSize} + if _, exists := capacityAllocatedMap[nnfNodeStorage.Namespace]; exists { + capacityAllocatedMap[nnfNodeStorage.Namespace] += allocationSize + } else { + capacityAllocatedMap[nnfNodeStorage.Namespace] = allocationSize + } + } + + for name, capacityAllocated := range capacityAllocatedMap { + servers.Status.AllocationSets[serversIndex].Storage[name] = dwsv1alpha1.ServersStatusStorage{AllocationSize: capacityAllocated} } for _, storageStatus := range servers.Status.AllocationSets[serversIndex].Storage { diff --git a/controllers/nnf_storage_controller.go b/controllers/nnf_storage_controller.go index 65558a581..aef149154 100644 --- a/controllers/nnf_storage_controller.go +++ b/controllers/nnf_storage_controller.go @@ -247,8 +247,13 @@ func (r *NnfStorageReconciler) createNodeStorage(ctx context.Context, storage *n nnfNodeStorage.Spec.FileSystemType = storage.Spec.FileSystemType nnfNodeStorage.Spec.LustreStorage.StartIndex = startIndex nnfNodeStorage.Spec.LustreStorage.FileSystemName = allocationSet.FileSystemName - nnfNodeStorage.Spec.LustreStorage.TargetType = allocationSet.TargetType nnfNodeStorage.Spec.LustreStorage.BackFs = allocationSet.BackFs + nnfNodeStorage.Spec.LustreStorage.TargetType = allocationSet.TargetType + + // If this isn't the first allocation, then change MGTMDT to MDT so that we only get a single MGT + if allocationSet.TargetType == "MGTMDT" && startIndex != 0 { + nnfNodeStorage.Spec.LustreStorage.TargetType = "MDT" + } // Create the list of client endpoints for each allocation and initialize it with // the rabbit node endpoint @@ -260,7 +265,7 @@ func (r *NnfStorageReconciler) createNodeStorage(ctx context.Context, storage *n } } - if allocationSet.TargetType == "MDT" || allocationSet.TargetType == "OST" { + if nnfNodeStorage.Spec.LustreStorage.TargetType == "MDT" || nnfNodeStorage.Spec.LustreStorage.TargetType == "OST" { if len(allocationSet.ExternalMgsNid) > 0 { nnfNodeStorage.Spec.LustreStorage.MgsNode = allocationSet.ExternalMgsNid } else { diff --git a/controllers/nnf_workflow_controller_helpers.go b/controllers/nnf_workflow_controller_helpers.go index b688e03f2..62399a462 100644 --- a/controllers/nnf_workflow_controller_helpers.go +++ b/controllers/nnf_workflow_controller_helpers.go @@ -401,9 +401,19 @@ func (r *NnfWorkflowReconciler) createNnfStorage(ctx context.Context, workflow * } // Create Nodes for this allocation set. - for _, storage := range s.Spec.AllocationSets[i].Storage { - node := nnfv1alpha1.NnfStorageAllocationNodes{Name: storage.Name, Count: storage.AllocationCount} - nnfAllocSet.Nodes = append(nnfAllocSet.Nodes, node) + for k, storage := range s.Spec.AllocationSets[i].Storage { + if s.Spec.AllocationSets[i].Label == "mgtmdt" && k == 0 && storage.AllocationCount > 1 { + // If there are multiple allocations on the first MGTMDT node, split it out into two seperate + // node entries. The first is a single allocation that will be used for the MGTMDT. The remaining + // allocations on the node will be MDTs only. + node := nnfv1alpha1.NnfStorageAllocationNodes{Name: storage.Name, Count: 1} + nnfAllocSet.Nodes = append(nnfAllocSet.Nodes, node) + node = nnfv1alpha1.NnfStorageAllocationNodes{Name: storage.Name, Count: storage.AllocationCount - 1} + nnfAllocSet.Nodes = append(nnfAllocSet.Nodes, node) + } else { + node := nnfv1alpha1.NnfStorageAllocationNodes{Name: storage.Name, Count: storage.AllocationCount} + nnfAllocSet.Nodes = append(nnfAllocSet.Nodes, node) + } } nnfStorage.Spec.AllocationSets = append(nnfStorage.Spec.AllocationSets, nnfAllocSet) diff --git a/controllers/nnf_workflow_controller_test.go b/controllers/nnf_workflow_controller_test.go index 3983211c0..bb5815fe1 100644 --- a/controllers/nnf_workflow_controller_test.go +++ b/controllers/nnf_workflow_controller_test.go @@ -547,6 +547,8 @@ var _ = Describe("NNF Workflow Unit Tests", func() { ) JustBeforeEach(func() { + Expect(k8sClient.Update(context.TODO(), storageProfile)).To(Succeed()) + Expect(k8sClient.Create(context.TODO(), workflow)).To(Succeed(), "create workflow") Eventually(func(g Gomega) bool { @@ -686,6 +688,37 @@ var _ = Describe("NNF Workflow Unit Tests", func() { }).Should(BeTrue(), "waiting for ready after setup") }) + It("Succeeds with multiple mdts", func() { + Eventually(func(g Gomega) error { + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(servers), servers)).To(Succeed()) + if len(servers.Spec.AllocationSets) == 0 { + return fmt.Errorf("Waiting for cache to update") + } + for i := range servers.Spec.AllocationSets { + if servers.Spec.AllocationSets[i].Label != "mdt" { + continue + } + + // Make four allocations, each with one quarter the size + servers.Spec.AllocationSets[i].AllocationSize /= 4 + servers.Spec.AllocationSets[i].Storage = append(servers.Spec.AllocationSets[i].Storage, servers.Spec.AllocationSets[i].Storage[0]) + servers.Spec.AllocationSets[i].Storage[1].AllocationCount = 3 + } + return k8sClient.Update(context.TODO(), servers) + }).Should(Succeed(), "Set multiple allocations") + + Eventually(func(g Gomega) error { + g.Expect(k8sClient.Get(context.TODO(), key, workflow)).To(Succeed()) + workflow.Spec.DesiredState = dwsv1alpha1.StateSetup + return k8sClient.Update(context.TODO(), workflow) + }).Should(Succeed(), "update to Setup") + + Eventually(func(g Gomega) bool { + g.Expect(k8sClient.Get(context.TODO(), key, workflow)).To(Succeed()) + return workflow.Status.Ready && workflow.Status.State == dwsv1alpha1.StateSetup + }).Should(BeTrue(), "waiting for ready after setup") + }) + It("Fails when allocation is too small", func() { Eventually(func(g Gomega) error { g.Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(servers), servers)).To(Succeed()) @@ -860,6 +893,70 @@ var _ = Describe("NNF Workflow Unit Tests", func() { }) + When("Using a Lustre file system with combined mgtmdt", func() { + BeforeEach(func() { + workflow.Spec.DWDirectives = []string{ + "#DW jobdw name=test type=lustre capacity=1GiB", + } + + storageProfile.Data.LustreStorage.CombinedMGTMDT = true + }) + + It("Succeeds with multiple mdts", func() { + Eventually(func(g Gomega) error { + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(servers), servers)).To(Succeed()) + if len(servers.Spec.AllocationSets) == 0 { + return fmt.Errorf("Waiting for cache to update") + } + for i := range servers.Spec.AllocationSets { + if servers.Spec.AllocationSets[i].Label != "mgtmdt" { + continue + } + + // Make four allocations, each with one quarter the size. 3 allocations are on the first storage + // and one is on a second storage + servers.Spec.AllocationSets[i].AllocationSize /= 4 + servers.Spec.AllocationSets[i].Storage = append(servers.Spec.AllocationSets[i].Storage, servers.Spec.AllocationSets[i].Storage[0]) + servers.Spec.AllocationSets[i].Storage[0].AllocationCount = 3 + } + return k8sClient.Update(context.TODO(), servers) + }).Should(Succeed(), "Set multiple allocations") + + Eventually(func(g Gomega) error { + g.Expect(k8sClient.Get(context.TODO(), key, workflow)).To(Succeed()) + workflow.Spec.DesiredState = dwsv1alpha1.StateSetup + return k8sClient.Update(context.TODO(), workflow) + }).Should(Succeed(), "update to Setup") + + Eventually(func(g Gomega) bool { + g.Expect(k8sClient.Get(context.TODO(), key, workflow)).To(Succeed()) + return workflow.Status.Ready && workflow.Status.State == dwsv1alpha1.StateSetup + }).Should(BeTrue(), "waiting for ready after setup") + + nnfStorage := &nnfv1alpha1.NnfStorage{ + ObjectMeta: metav1.ObjectMeta{ + Name: servers.Name, + Namespace: servers.Namespace, + }, + } + + Eventually(func(g Gomega) bool { + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(nnfStorage), nnfStorage)).To(Succeed()) + for _, allocationSet := range nnfStorage.Spec.AllocationSets { + if allocationSet.Name != "mgtmdt" { + continue + } + + // Check that there are 3 nodes listed in the mgtmdt allocation set. The first node should + // only have a single allocation. + return len(allocationSet.Nodes) == 3 && allocationSet.Nodes[0].Count == 1 + } + + return false + }).Should(BeTrue(), "waiting for ready after setup") + }) + }) + When("Using an XFS file system", func() { BeforeEach(func() { workflow.Spec.DWDirectives = []string{ diff --git a/go.mod b/go.mod index 1c25d6d59..1b440da66 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 + github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 12df7bc60..3f56037d7 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,10 @@ github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 h1:f09Nvmieia github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 h1:Eg/6n0p/qGbFhL1w+zxiPj2jjK+5rAEM3q7+vO0CWy4= github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230105221007-3282bfd76a70 h1:Sycdy92vRX6oSmb/BbvGEKOTcnLd9fWCsbCPOWmGfug= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230105221007-3282bfd76a70/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 h1:j9KD7WrgMVJrCQHvFixazXS6kbxRSpH6oM+puj+MVdw= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go index 6727be18b..4a71d759c 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go @@ -23,10 +23,12 @@ import ( "encoding/json" "fmt" "net/http" + "time" ) type ControllerError struct { statusCode int + retryDelay time.Duration cause string resourceType string err error @@ -39,6 +41,12 @@ func NewControllerError(sc int) *ControllerError { func (e *ControllerError) Error() string { errorString := fmt.Sprintf("Error %d: %s", e.statusCode, http.StatusText(e.statusCode)) + if e.IsRetryable() { + errorString += fmt.Sprintf(", Retry-Delay: %ds", e.retryDelay) + } + if len(e.resourceType) != 0 { + errorString += fmt.Sprintf(", Resource: %s", e.resourceType) + } if len(e.cause) != 0 { errorString += fmt.Sprintf(", Cause: %s", e.cause) } @@ -88,8 +96,26 @@ func (e *ControllerError) WithEvent(event interface{}) *ControllerError { return e } +func (e *ControllerError) WithRetryDelay(delay time.Duration) *ControllerError { + e.retryDelay = delay + return e +} + +func (e *ControllerError) IsRetryable() bool { + return e.statusCode != http.StatusTooManyRequests +} + +func (e *ControllerError) RetryDelay() time.Duration { + return e.retryDelay +} + // NewErr** Functions will allocate a controller error for the specific type of error. // Error details can be added through the use of WithError(err) and WithCause(string) methods +func NewErrorNotReady() *ControllerError { + // In HTTP, Error 429 (Too Many Requests) response indicates how long to wait before making a + // new request. We use that here to include a delay (default 1s) before making a new request. + return NewControllerError(http.StatusTooManyRequests).WithRetryDelay(1 * time.Second) +} func NewErrNotFound() *ControllerError { return NewControllerError(http.StatusNotFound) diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go index e7222769e..efda973ae 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go @@ -23,7 +23,6 @@ import ( "fmt" "strconv" "strings" - "time" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -532,6 +531,15 @@ func (s *StorageService) EventHandler(e event.Event) error { if e.Is(msgreg.FabricReadyNnf("")) { log.Infof("Storage Service: Event Received %+v", e) + if err := s.store.Replay(); err != nil { + log.WithError(err).Errorf("Failed to replay storage database") + return err + } + + // Remove any namespaces that are not part of a Storage Pool + log.Infof("Storage Service: Removing Volumes that are not allocated as part of a Storage Pool") + s.cleanupVolumes() + s.state = sf.ENABLED_RST s.health = sf.OK_RH @@ -549,14 +557,7 @@ func (s *StorageService) EventHandler(e event.Event) error { s.health = f.Status.Health } - if err := s.store.Replay(); err != nil { - log.WithError(err).Errorf("Failed to replay storage database") - return err - } - - // Remove any namespaces that are not part of a Storage Pool - log.Infof("Storage Service: Removing Volumes that are not allocated as part of a Storage Pool") - s.cleanupVolumes() + log.Infof("Storage Service Enabled: Health %s", s.health) } return nil @@ -963,6 +964,8 @@ func (*StorageService) StorageServiceIdStorageGroupPost(storageServiceId string, return ec.NewErrNotAcceptable().WithResourceType(StoragePoolOdataType).WithEvent(msgreg.ResourceNotFoundBase(StoragePoolOdataType, storagePoolId)) } + // TODO RABSW-1110: Ensure storage pool is operational before creating a storage group + fields = strings.Split(model.Links.ServerEndpoint.OdataId, "/") if len(fields) != s.resourceIndex+1 { return ec.NewErrNotAcceptable().WithResourceType(EndpointOdataType).WithEvent(msgreg.InvalidURIBase(model.Links.ServerEndpoint.OdataId)) @@ -1342,16 +1345,12 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedSharesPost(storageSer return ec.NewErrNotAcceptable().WithResourceType(StoragePoolOdataType).WithEvent(msgreg.ResourceNotFoundBase(StorageGroupOdataType, endpointId)) } -refreshState: - switch sg.status().State { - case sf.ENABLED_RST: - break - case sf.STARTING_RST: - log.Infof("Storage group starting, delay 1s") - time.Sleep(time.Second) - goto refreshState - default: - return ec.NewErrNotAcceptable() + // Wait for the storage group to be ready (enabled) to ensure the disks are present on the system + state := sg.status().State + if state == sf.STARTING_RST { + return ec.NewErrorNotReady().WithResourceType(StorageGroupOdataType).WithCause(fmt.Sprintf("Storage group '%s' is starting", sg.id)) + } else if state != sf.ENABLED_RST { + return ec.NewErrNotAcceptable().WithResourceType(StorageGroupOdataType).WithCause(fmt.Sprintf("Storage group '%s' is not ready: state %s", sg.id, state)) } sh := fs.createFileShare(model.Id, sg, model.FileSharePath) diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go index 067f88d25..105d2fa3d 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go @@ -187,7 +187,7 @@ func (f *FileSystemLustre) zfsTargType() string { } func (f *FileSystemLustre) zfsPoolName() string { - return fmt.Sprintf("%s-%spool", f.name, f.zfsTargType()) + return fmt.Sprintf("%s-%spool-%d", f.name, f.zfsTargType(), f.Oem.Index) } func (f *FileSystemLustre) zfsVolName() string { diff --git a/vendor/modules.txt b/vendor/modules.txt index f76c9ce3c..efc21573d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 +# github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From ca84c2414319bdaf1e42ac164dbe6518c0bc42f5 Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Thu, 12 Jan 2023 10:30:35 -0600 Subject: [PATCH 12/71] RABSW-1099: Vendor DWS (#122) Pick up the changes to the PersistentStorage state fields. Signed-off-by: Matt Richerson Signed-off-by: Matt Richerson --- go.mod | 2 +- go.sum | 22 ++----------------- .../persistentstorageinstance_types.go | 10 ++++----- .../dws/api/v1alpha1/workflow_types.go | 2 ++ .../dws/api/v1alpha1/workflow_webhook.go | 2 +- ...ay.hpe.com_persistentstorageinstances.yaml | 10 ++++----- .../dws/config/webhook/manifests.yaml | 1 - vendor/modules.txt | 2 +- 8 files changed, 17 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 1b440da66..417916014 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/NearNodeFlash/nnf-sos go 1.19 require ( - github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5 + github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index 3f56037d7..aa4767210 100644 --- a/go.sum +++ b/go.sum @@ -86,32 +86,14 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/HewlettPackard/dws v0.0.0-20230103152723-c01b98e1d754 h1:ggvsAahh1WyRdSzP2eblyi0ysZAfwsKfS3k55492Osc= -github.com/HewlettPackard/dws v0.0.0-20230103152723-c01b98e1d754/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= -github.com/HewlettPackard/dws v0.0.0-20230103221621-3ddba3d818ed h1:fkTNMUhY9AVYpu6vc7B36pxdhZMEwtXBZuXbs5RtVcM= -github.com/HewlettPackard/dws v0.0.0-20230103221621-3ddba3d818ed/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= -github.com/HewlettPackard/dws v0.0.0-20230104151937-fc882a72469a h1:emYNSAkP0Nb9CRfrMGGMy7PRTqNISTYkbbqEHx/LY5c= -github.com/HewlettPackard/dws v0.0.0-20230104151937-fc882a72469a/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= -github.com/HewlettPackard/dws v0.0.0-20230104161449-b57ee6b0744b h1:3nbllp+Fw6K8oiYg8x3Ctoz3oYEcJLtCwyKxyVPCl9Y= -github.com/HewlettPackard/dws v0.0.0-20230104161449-b57ee6b0744b/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= -github.com/HewlettPackard/dws v0.0.0-20230104182823-40864bb5d259 h1:MysF14Ceu52QJPvAlHp6RQCEfSs2EfiDJl1jgFizgKE= -github.com/HewlettPackard/dws v0.0.0-20230104182823-40864bb5d259/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= -github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5 h1:J+ok11bmTTSAc4ZURVSGstjFYpQg0nN8lFIWoEhUzXA= -github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8 h1:EXF13IM2Zjh4MpOBdiLLR2nl0hOqyO2cJGT/Z3B5hOE= +github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= github.com/HewlettPackard/structex v1.0.2 h1:p2EH/p6zvUd5fSa0onudAUvLWdKsvQSRP0jGKeblA5E= github.com/HewlettPackard/structex v1.0.2/go.mod h1:3frC4RY/cPsP/4+N8rkxsNAGlQwHV+zDC7qvrN+N+rE= github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h1:upOQcpaEDmgLxoK7SZeqzGOQs8raxd9/paJYmBS/3/M= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61 h1:yMRoXhdvJSd+QMfZBv98K82+u9NAvXI5N/dZdxEfo7Q= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221219165212-e4ba0bfa4c61/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1 h1:f09NvmieiaSYE8Gxd6+vhmrtx1PB4W4Sldc+rJAAxZQ= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221220173628-96d6a3bab9d1/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80 h1:Eg/6n0p/qGbFhL1w+zxiPj2jjK+5rAEM3q7+vO0CWy4= -github.com/NearNodeFlash/nnf-ec v0.0.0-20221221152437-83d47b38ec80/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230105221007-3282bfd76a70 h1:Sycdy92vRX6oSmb/BbvGEKOTcnLd9fWCsbCPOWmGfug= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230105221007-3282bfd76a70/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 h1:j9KD7WrgMVJrCQHvFixazXS6kbxRSpH6oM+puj+MVdw= github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/persistentstorageinstance_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/persistentstorageinstance_types.go index 901cf8fe1..3263908d5 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/persistentstorageinstance_types.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/persistentstorageinstance_types.go @@ -41,15 +41,15 @@ type PersistentStorageInstanceState string // State enumerations const ( // The PSI resource exists in k8s, but the storage and filesystem that it represents has not been created yet - PSIStateCreating PersistentStorageInstanceState = "creating" + PSIStateCreating PersistentStorageInstanceState = "Creating" // The storage and filesystem represented by the PSI exists and is ready for use - PSIStateActive PersistentStorageInstanceState = "active" + PSIStateActive PersistentStorageInstanceState = "Active" // A #DW destroy_persistent directive has been issued in a workflow. // Once all other workflows with persistent_dw reservations on the PSI complete, the PSI will be destroyed. // New #DW persistent_dw requests after the PSI enters the 'destroying' state will fail. - PSIStateDestroying PersistentStorageInstanceState = "destroying" + PSIStateDestroying PersistentStorageInstanceState = "Destroying" ) // PersistentStorageInstanceSpec defines the desired state of PersistentStorageInstance @@ -68,7 +68,7 @@ type PersistentStorageInstanceSpec struct { UserID uint32 `json:"userID"` // Desired state of the PersistentStorageInstance - // +kubebuilder:validation:Enum:=active;destroying + // +kubebuilder:validation:Enum:=Active;Destroying State PersistentStorageInstanceState `json:"state"` // List of consumers using this persistent storage @@ -81,7 +81,7 @@ type PersistentStorageInstanceStatus struct { Servers corev1.ObjectReference `json:"servers,omitempty"` // Current state of the PersistentStorageInstance - // +kubebuilder:validation:Enum:=creating;active;destroying + // +kubebuilder:validation:Enum:=Creating;Active;Destroying State PersistentStorageInstanceState `json:"state"` // Error information diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go index 86c5be191..0462bf895 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_types.go @@ -51,6 +51,8 @@ const ( // Next reports the next state after state s func (s WorkflowState) next() WorkflowState { switch s { + case "": + return StateProposal case StateProposal: return StateSetup case StateSetup: diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go index 5298697b2..1b7532b8d 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go @@ -51,7 +51,7 @@ func (w *Workflow) SetupWebhookWithManager(mgr ctrl.Manager) error { Complete() } -//+kubebuilder:webhook:path=/mutate-dws-cray-hpe-com-v1alpha1-workflow,mutating=true,failurePolicy=fail,sideEffects=None,groups=dws.cray.hpe.com,resources=workflows,verbs=create;update,versions=v1alpha1,name=mworkflow.kb.io,admissionReviewVersions={v1,v1beta1} +//+kubebuilder:webhook:path=/mutate-dws-cray-hpe-com-v1alpha1-workflow,mutating=true,failurePolicy=fail,sideEffects=None,groups=dws.cray.hpe.com,resources=workflows,verbs=create,versions=v1alpha1,name=mworkflow.kb.io,admissionReviewVersions={v1,v1beta1} var _ webhook.Defaulter = &Workflow{} diff --git a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_persistentstorageinstances.yaml b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_persistentstorageinstances.yaml index f51031e9a..82443bbc4 100644 --- a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_persistentstorageinstances.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_persistentstorageinstances.yaml @@ -119,8 +119,8 @@ spec: state: description: Desired state of the PersistentStorageInstance enum: - - active - - destroying + - Active + - Destroying type: string userID: description: User ID of the user that created the persistent storage @@ -196,9 +196,9 @@ spec: state: description: Current state of the PersistentStorageInstance enum: - - creating - - active - - destroying + - Creating + - Active + - Destroying type: string required: - state diff --git a/vendor/github.com/HewlettPackard/dws/config/webhook/manifests.yaml b/vendor/github.com/HewlettPackard/dws/config/webhook/manifests.yaml index 67105836a..d7c8803df 100644 --- a/vendor/github.com/HewlettPackard/dws/config/webhook/manifests.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/webhook/manifests.yaml @@ -22,7 +22,6 @@ webhooks: - v1alpha1 operations: - CREATE - - UPDATE resources: - workflows sideEffects: None diff --git a/vendor/modules.txt b/vendor/modules.txt index efc21573d..76b44b870 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.6.0 ## explicit; go 1.12 github.com/Azure/go-autorest/tracing -# github.com/HewlettPackard/dws v0.0.0-20230104194155-1f78117736f5 +# github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8 ## explicit; go 1.19 github.com/HewlettPackard/dws/api/v1alpha1 github.com/HewlettPackard/dws/config/crd/bases From ba2114ab83c87227210661d3bb7de82dd73b1480 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Wed, 18 Jan 2023 15:28:58 -0600 Subject: [PATCH 13/71] Refactor Job/Persistent directive references to use "DW" prefix Signed-off-by: Nate Thornton --- controllers/integration_test.go | 12 +++++------ controllers/nnf_workflow_controller.go | 18 ++++++++--------- .../nnf_workflow_controller_helpers.go | 20 +++++++++---------- controllers/nnf_workflow_controller_test.go | 14 ++++++------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/controllers/integration_test.go b/controllers/integration_test.go index 9369a52ee..ef8928e62 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -1096,7 +1096,7 @@ var _ = Describe("Integration Test", func() { BeforeEach(func() { workflow.Spec.DWDirectives = []string{ "#DW jobdw name=test-dm-0 type=gfs2 capacity=1GiB", - "#DW copy_in source=/lus/maui/file destination=$JOB_DW_test-dm-0/file", + "#DW copy_in source=/lus/maui/file destination=$DW_JOB_test-dm-0/file", } }) @@ -1119,7 +1119,7 @@ var _ = Describe("Integration Test", func() { BeforeEach(func() { workflow.Spec.DWDirectives = []string{ "#DW jobdw name=test-dm-0 type=gfs2 capacity=1GiB", - "#DW copy_out source=$JOB_DW_test-dm-0/file destination=/lus/maui/file", + "#DW copy_out source=$DW_JOB_test-dm-0/file destination=/lus/maui/file", } }) @@ -1146,8 +1146,8 @@ var _ = Describe("Integration Test", func() { BeforeEach(func() { workflow.Spec.DWDirectives = []string{ "#DW jobdw name=test-dm-0 type=gfs2 capacity=1GiB", - "#DW copy_in source=/lus/maui/file destination=$JOB_DW_test-dm-0/file", - "#DW copy_out source=$JOB_DW_test-dm-0/file destination=/lus/maui/file", + "#DW copy_in source=/lus/maui/file destination=$DW_JOB_test-dm-0/file", + "#DW copy_out source=$DW_JOB_test-dm-0/file destination=/lus/maui/file", } }) @@ -1555,7 +1555,7 @@ var _ = Describe("Integration Test", func() { }) BeforeEach(func() { directives = []string{ - "#DW copy_in source=/lus/maui/my-file.in destination=$JOB_DW_notThere/my-file.out", + "#DW copy_in source=/lus/maui/my-file.in destination=$DW_JOB_notThere/my-file.out", } }) @@ -1592,7 +1592,7 @@ var _ = Describe("Integration Test", func() { BeforeEach(func() { directives = []string{ - "#DW copy_in source=/lus/maui/my-file.in destination=$PERSISTENT_DW_mpi/my-persistent-file.out", + "#DW copy_in source=/lus/maui/my-file.in destination=$DW_PERSISTENT_mpi/my-persistent-file.out", } }) diff --git a/controllers/nnf_workflow_controller.go b/controllers/nnf_workflow_controller.go index fdb510a2e..05a8bbecc 100644 --- a/controllers/nnf_workflow_controller.go +++ b/controllers/nnf_workflow_controller.go @@ -450,13 +450,13 @@ func (r *NnfWorkflowReconciler) startDataInOutState(ctx context.Context, workflo } // NOTE: We don't need to check for the occurrence of a source or destination parameters since these are required fields and validated through the webhook - // NOTE: We don't need to validate destination has $JOB_DW_ since this is done during the proposal stage + // NOTE: We don't need to validate destination has $DW_JOB_ since this is done during the proposal stage // copy_in needs to handle four cases - // 1. Lustre to JobStorageInstance #DW copy_in source=[path] destination=$JOB_DW_[name]/[path] - // 2. Lustre to PersistentStorageInstance #DW copy_in source=[path] destination=$PERSISTENT_DW_[name]/[path] - // 3. PersistentStorageInstance to JobStorageInstance #DW copy_in source=$PERSISTENT_DW_[name]/[path] destination=$JOB_DW_[name]/[path] - // 4. PersistentStorageInstance to PersistentStorageInstance #DW copy_in source=$PERSISTENT_DW_[name]/[path] destination=$PERSISTENT_DW_[name]/[path] + // 1. Lustre to JobStorageInstance #DW copy_in source=[path] destination=$DW_JOB_[name]/[path] + // 2. Lustre to PersistentStorageInstance #DW copy_in source=[path] destination=$DW_PERSISTENT_[name]/[path] + // 3. PersistentStorageInstance to JobStorageInstance #DW copy_in source=$DW_PERSISTENT_[name]/[path] destination=$DW_JOB_[name]/[path] + // 4. PersistentStorageInstance to PersistentStorageInstance #DW copy_in source=$DW_PERSISTENT_[name]/[path] destination=$DW_PERSISTENT_[name]/[path] // copy_out is the same, but typically source and destination are reversed // Prepare the provided staging parameter for data-movement. Param is the source/destination value from the #DW copy_in/copy_out directive; based @@ -468,7 +468,7 @@ func (r *NnfWorkflowReconciler) startDataInOutState(ctx context.Context, workflo // If param refers to a Job or Persistent storage type, find the NNF Storage that is backing // this directive line. - if strings.HasPrefix(param, "$JOB_DW_") || strings.HasPrefix(param, "$PERSISTENT_DW_") { + if strings.HasPrefix(param, "$DW_JOB_") || strings.HasPrefix(param, "$DW_PERSISTENT_") { // Find the parent directive index that corresponds to this copy_in/copy_out directive parentDwIndex := findDirectiveIndexByName(workflow, name) @@ -479,7 +479,7 @@ func (r *NnfWorkflowReconciler) startDataInOutState(ctx context.Context, workflo // If directive specifies a persistent storage instance, `name` will be the nnfStorageName // Otherwise it matches the workflow with the directive index as a suffix var nnfStorageName string - if strings.HasPrefix(param, "$PERSISTENT_DW_") { + if strings.HasPrefix(param, "$DW_PERSISTENT_") { nnfStorageName = name } else { nnfStorageName = indexedResourceName(workflow, parentDwIndex) @@ -739,7 +739,7 @@ func (r *NnfWorkflowReconciler) finishDataInOutState(ctx context.Context, workfl unmountIfNecessary := func(param string) (*result, error) { name, _ := splitStagingArgumentIntoNameAndPath(param) - if strings.HasPrefix(param, "$JOB_DW_") || strings.HasPrefix(param, "$PERSISTENT_DW_") { + if strings.HasPrefix(param, "$DW_JOB_") || strings.HasPrefix(param, "$DW_PERSISTENT_") { parentDwIndex := findDirectiveIndexByName(workflow, name) if parentDwIndex < 0 { return nil, nnfv1alpha1.NewWorkflowErrorf("No directive matching '%s' found in workflow", name).WithFatal() @@ -818,7 +818,7 @@ func (r *NnfWorkflowReconciler) startPreRunState(ctx context.Context, workflow * // Create an NnfAccess for the servers resources if necessary. Shared storage like // that of GFS2 provides access to the Rabbit for the lifetime of the user's job. // NnfAccess may already be present if a data_in directive was specified for the - // particular $JOB_DW_[name]; in this case we only need to recreate the resource + // particular $DW_JOB_[name]; in this case we only need to recreate the resource fsType, err := r.getDirectiveFileSystemType(ctx, workflow, index) if err != nil { diff --git a/controllers/nnf_workflow_controller_helpers.go b/controllers/nnf_workflow_controller_helpers.go index 62399a462..fed2ed6d0 100644 --- a/controllers/nnf_workflow_controller_helpers.go +++ b/controllers/nnf_workflow_controller_helpers.go @@ -127,16 +127,16 @@ func (r *NnfWorkflowReconciler) validateStagingDirective(ctx context.Context, wf // #DW copy_out source=[SOURCE] destination=[DESTINATION] // For each copy_in/copy_out directive - // Make sure all $JOB_DW_ references point to job storage instance names - // Make sure all $PERSISTENT_DW references an existing persistentdw instance + // Make sure all $DW_JOB references point to job storage instance names + // Make sure all $DW_PERSISTENT references an existing persistentdw instance // Otherwise, make sure source/destination is prefixed with a valid global lustre file system validateStagingArgument := func(arg string) error { name, _ := splitStagingArgumentIntoNameAndPath(arg) - if strings.HasPrefix(arg, "$JOB_DW_") { + if strings.HasPrefix(arg, "$DW_JOB_") { if findDirectiveIndexByName(wf, name) == -1 { return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("Job storage instance '%s' not found", name)).WithFatal() } - } else if strings.HasPrefix(arg, "$PERSISTENT_DW_") { + } else if strings.HasPrefix(arg, "$DW_PERSISTENT_") { if err := r.validatePersistentInstanceByName(ctx, name, wf.Namespace); err != nil { return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("Persistent storage instance '%s' not found", name)).WithFatal() } @@ -576,7 +576,7 @@ func findCopyOutDirectiveIndexByName(workflow *dwsv1alpha1.Workflow, name string if strings.HasPrefix(directive, "#DW copy_out") { parameters, _ := dwdparse.BuildArgsMap(directive) // ignore error, directives are validated in proposal - srcName, _ := splitStagingArgumentIntoNameAndPath(parameters["source"]) // i.e. source=$JOB_DW_[name] + srcName, _ := splitStagingArgumentIntoNameAndPath(parameters["source"]) // i.e. source=$DW_JOB_[name] if srcName == name { return idx } @@ -587,14 +587,14 @@ func findCopyOutDirectiveIndexByName(workflow *dwsv1alpha1.Workflow, name string } // Returns a pair for the given staging argument (typically source or destination) -// i.e. $JOB_DW_my-file-system-name/path/to/a/file into "my-file-system-name" and "/path/to/a/file" +// i.e. $DW_JOB_my-file-system-name/path/to/a/file into "my-file-system-name" and "/path/to/a/file" func splitStagingArgumentIntoNameAndPath(arg string) (string, string) { var name = "" - if strings.HasPrefix(arg, "$JOB_DW_") { - name = strings.SplitN(strings.Replace(arg, "$JOB_DW_", "", 1), "/", 2)[0] - } else if strings.HasPrefix(arg, "$PERSISTENT_DW_") { - name = strings.SplitN(strings.Replace(arg, "$PERSISTENT_DW_", "", 1), "/", 2)[0] + if strings.HasPrefix(arg, "$DW_JOB_") { + name = strings.SplitN(strings.Replace(arg, "$DW_JOB_", "", 1), "/", 2)[0] + } else if strings.HasPrefix(arg, "$DW_PERSISTENT_") { + name = strings.SplitN(strings.Replace(arg, "$DW_PERSISTENT_", "", 1), "/", 2)[0] } var path = "/" if strings.Count(arg, "/") >= 1 { diff --git a/controllers/nnf_workflow_controller_test.go b/controllers/nnf_workflow_controller_test.go index bb5815fe1..3bb164c7f 100644 --- a/controllers/nnf_workflow_controller_test.go +++ b/controllers/nnf_workflow_controller_test.go @@ -256,7 +256,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { It("Fails missing or malformed jobdw reference", func() { workflow.Spec.DWDirectives = []string{ "#DW jobdw name=test type=lustre capacity=1GiB", - "#DW copy_in source=/lus/maui/my-file.in destination=$JOB_DW_INCORRECT/my-file.out", + "#DW copy_in source=/lus/maui/my-file.in destination=$DW_JOB_INCORRECT/my-file.out", } Expect(k8sClient.Create(context.TODO(), workflow)).To(Succeed(), "create workflow") @@ -272,7 +272,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { It("Fails missing or malformed persistentdw reference", func() { workflow.Spec.DWDirectives = []string{ "#DW create_persistent name=test type=lustre capacity=1GiB", - "#DW copy_in source=/lus/maui/my-file.in destination=$PERSISTENT_DW_INCORRECT/my-file.out", + "#DW copy_in source=/lus/maui/my-file.in destination=$DW_PERSISTENT_INCORRECT/my-file.out", } Expect(k8sClient.Create(context.TODO(), workflow)).To(Succeed(), "create workflow") @@ -287,7 +287,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { It("Fails missing or malformed global lustre reference", func() { workflow.Spec.DWDirectives = []string{ "#DW jobdw name=test type=lustre capacity=1GiB", - "#DW copy_in source=/lus/INCORRECT/my-file.in destination=$JOB_DW_test/my-file.out", + "#DW copy_in source=/lus/INCORRECT/my-file.in destination=$DW_JOB_test/my-file.out", } Expect(k8sClient.Create(context.TODO(), workflow)).To(Succeed(), "create workflow") @@ -354,12 +354,12 @@ var _ = Describe("NNF Workflow Unit Tests", func() { Expect(k8sClient.Delete(context.TODO(), lustre)).To(Succeed()) }) - When("using $JOB_DW_ references", func() { + When("using $DW_JOB_ references", func() { BeforeEach(func() { workflow.Spec.DWDirectives = []string{ "#DW jobdw name=test type=lustre capacity=1GiB", - "#DW copy_in source=/lus/maui/my-file.in destination=$JOB_DW_test/my-file.out", + "#DW copy_in source=/lus/maui/my-file.in destination=$DW_JOB_test/my-file.out", } }) @@ -404,7 +404,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { }) }) - When("using $PERSISTENT_DW_ references", func() { + When("using $DW_PERSISTENT_ references", func() { persistentStorageName := "my-persistent-storage" createPersistentStorageInstance := func() { @@ -473,7 +473,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { BeforeEach(func() { workflow.Spec.DWDirectives = []string{ fmt.Sprintf("#DW persistentdw name=%s", persistentStorageName), - fmt.Sprintf("#DW copy_in source=/lus/maui/my-file.in destination=$PERSISTENT_DW_%s/my-persistent-file.out", persistentStorageName), + fmt.Sprintf("#DW copy_in source=/lus/maui/my-file.in destination=$DW_PERSISTENT_%s/my-persistent-file.out", persistentStorageName), } createPersistentStorageInstance() From 4abba00be7bf831c0a1206732f2899f653785e76 Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Fri, 20 Jan 2023 14:40:58 -0600 Subject: [PATCH 14/71] RABSW-1097: Pass UserID and GroupID to ClientMount (#129) * RABSW-1097: Pass UserID and GroupID to ClientMount Pass the UserID and GroupID from the workflow, through the NnfAccess, and to the ClientMount. This is used to set the owner/group of Raw devices on the compute node. Signed-off-by: Matt Richerson * re-vendor Signed-off-by: Matt Richerson Signed-off-by: Matt Richerson --- api/v1alpha1/nnf_access_types.go | 6 ++++++ config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml | 10 ++++++++++ controllers/nnf_access_controller.go | 3 +++ controllers/nnf_workflow_controller.go | 2 ++ go.mod | 2 +- go.sum | 4 ++-- .../dws/api/v1alpha1/clientmount_types.go | 9 +++++++++ .../crd/bases/dws.cray.hpe.com_clientmounts.yaml | 13 +++++++++++++ vendor/modules.txt | 2 +- 9 files changed, 47 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/nnf_access_types.go b/api/v1alpha1/nnf_access_types.go index f9c42115a..0ac2970b6 100644 --- a/api/v1alpha1/nnf_access_types.go +++ b/api/v1alpha1/nnf_access_types.go @@ -45,6 +45,12 @@ type NnfAccessSpec struct { // +kubebuilder:validation:Enum=single;all Target string `json:"target"` + // UserID for the new mount. Currently only used for raw + UserID uint32 `json:"userID"` + + // GroupID for the new mount. Currently only used for raw + GroupID uint32 `json:"groupID"` + // ClientReference is for a client resource. (DWS) Computes is the only client // resource type currently supported ClientReference corev1.ObjectReference `json:"clientReference,omitempty"` diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml index 88631932c..d0e274f79 100644 --- a/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml +++ b/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml @@ -96,6 +96,10 @@ spec: - mounted - unmounted type: string + groupID: + description: GroupID for the new mount. Currently only used for raw + format: int32 + type: integer mountPath: description: MountPath for the storage target on the client type: string @@ -166,11 +170,17 @@ spec: description: TeardownState is the desired state of the workflow for this NNF Access resource to be torn down and deleted. type: string + userID: + description: UserID for the new mount. Currently only used for raw + format: int32 + type: integer required: - desiredState + - groupID - storageReference - target - teardownState + - userID type: object status: description: NnfAccessStatus defines the observed state of NnfAccess diff --git a/controllers/nnf_access_controller.go b/controllers/nnf_access_controller.go index 27e7e5a8b..8ebe5ea21 100644 --- a/controllers/nnf_access_controller.go +++ b/controllers/nnf_access_controller.go @@ -553,6 +553,9 @@ func (r *NnfAccessReconciler) mapClientLocalStorage(ctx context.Context, access mountInfo.Type = "none" mountInfo.TargetType = "file" mountInfo.Options = "bind" + mountInfo.UserID = access.Spec.UserID + mountInfo.GroupID = access.Spec.GroupID + mountInfo.SetPermissions = true } else { mountInfo.TargetType = "directory" mountInfo.Type = nnfStorage.Spec.FileSystemType diff --git a/controllers/nnf_workflow_controller.go b/controllers/nnf_workflow_controller.go index 05a8bbecc..50b92df6b 100644 --- a/controllers/nnf_workflow_controller.go +++ b/controllers/nnf_workflow_controller.go @@ -782,6 +782,8 @@ func (r *NnfWorkflowReconciler) startPreRunState(ctx context.Context, workflow * access.Spec.TeardownState = dwsv1alpha1.StatePostRun access.Spec.DesiredState = "mounted" + access.Spec.UserID = workflow.Spec.UserID + access.Spec.GroupID = workflow.Spec.GroupID access.Spec.Target = "single" access.Spec.MountPath = buildMountPath(workflow, index) access.Spec.ClientReference = corev1.ObjectReference{ diff --git a/go.mod b/go.mod index 417916014..1676e5994 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/NearNodeFlash/nnf-sos go 1.19 require ( - github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8 + github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index aa4767210..40ecfa312 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8 h1:EXF13IM2Zjh4MpOBdiLLR2nl0hOqyO2cJGT/Z3B5hOE= -github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455 h1:LIuI3Z7nSdm+hnNmJY/TLV//nloL6cqMlqWqclq9yNo= +github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= github.com/HewlettPackard/structex v1.0.2 h1:p2EH/p6zvUd5fSa0onudAUvLWdKsvQSRP0jGKeblA5E= github.com/HewlettPackard/structex v1.0.2/go.mod h1:3frC4RY/cPsP/4+N8rkxsNAGlQwHV+zDC7qvrN+N+rE= github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/clientmount_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/clientmount_types.go index 37eae4a4e..9a1600054 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/clientmount_types.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/clientmount_types.go @@ -117,6 +117,15 @@ type ClientMountInfo struct { // Client path for mount target MountPath string `json:"mountPath"` + // UserID to set for the mount + UserID uint32 `json:"userID,omitempty"` + + // GroupID to set for the mount + GroupID uint32 `json:"groupID,omitempty"` + + // SetPermissions will set UserID and GroupID on the mount if true + SetPermissions bool `json:"setPermissions"` + // Options for the file system mount Options string `json:"options"` diff --git a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_clientmounts.yaml b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_clientmounts.yaml index a77e892e2..871247f66 100644 --- a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_clientmounts.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_clientmounts.yaml @@ -170,12 +170,20 @@ spec: required: - type type: object + groupID: + description: GroupID to set for the mount + format: int32 + type: integer mountPath: description: Client path for mount target type: string options: description: Options for the file system mount type: string + setPermissions: + description: SetPermissions will set UserID and GroupID on the + mount if true + type: boolean targetType: description: TargetType determines whether the mount target is a file or a directory @@ -191,10 +199,15 @@ spec: - gfs2 - none type: string + userID: + description: UserID to set for the mount + format: int32 + type: integer required: - device - mountPath - options + - setPermissions - targetType - type type: object diff --git a/vendor/modules.txt b/vendor/modules.txt index 76b44b870..9e4d52ceb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.6.0 ## explicit; go 1.12 github.com/Azure/go-autorest/tracing -# github.com/HewlettPackard/dws v0.0.0-20230112160217-3d1e4e20bfc8 +# github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455 ## explicit; go 1.19 github.com/HewlettPackard/dws/api/v1alpha1 github.com/HewlettPackard/dws/config/crd/bases From c55083c0061166bb20d74c55b6a264edb6a5116c Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Tue, 24 Jan 2023 09:59:44 -0600 Subject: [PATCH 15/71] RABSW-1122: Don't allow staging to Raw allocations (#131) Do some more sanity checks on staging directives: - Don't allow staging to/from raw allocations - Match allocation directives based on name and command "jobdw/persistentdw" since names can collide between the two types. Signed-off-by: Matt Richerson Signed-off-by: Matt Richerson --- controllers/nnf_workflow_controller.go | 17 ++++++- .../nnf_workflow_controller_helpers.go | 48 ++++++++++++++----- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/controllers/nnf_workflow_controller.go b/controllers/nnf_workflow_controller.go index 50b92df6b..37ff10b3e 100644 --- a/controllers/nnf_workflow_controller.go +++ b/controllers/nnf_workflow_controller.go @@ -471,7 +471,14 @@ func (r *NnfWorkflowReconciler) startDataInOutState(ctx context.Context, workflo if strings.HasPrefix(param, "$DW_JOB_") || strings.HasPrefix(param, "$DW_PERSISTENT_") { // Find the parent directive index that corresponds to this copy_in/copy_out directive - parentDwIndex := findDirectiveIndexByName(workflow, name) + parentDwIndex := 0 + + if strings.HasPrefix(param, "$DW_PERSISTENT_") { + parentDwIndex = findDirectiveIndexByName(workflow, name, "persistentdw") + } else { + parentDwIndex = findDirectiveIndexByName(workflow, name, "jobdw") + } + if parentDwIndex < 0 { return nil, nil, nil, nnfv1alpha1.NewWorkflowError("No directive matching '" + name + "' found in workflow").WithFatal() } @@ -740,7 +747,13 @@ func (r *NnfWorkflowReconciler) finishDataInOutState(ctx context.Context, workfl name, _ := splitStagingArgumentIntoNameAndPath(param) if strings.HasPrefix(param, "$DW_JOB_") || strings.HasPrefix(param, "$DW_PERSISTENT_") { - parentDwIndex := findDirectiveIndexByName(workflow, name) + parentDwIndex := 0 + if strings.HasPrefix(param, "$DW_PERSISTENT_") { + parentDwIndex = findDirectiveIndexByName(workflow, name, "persistentdw") + } else { + parentDwIndex = findDirectiveIndexByName(workflow, name, "jobdw") + } + if parentDwIndex < 0 { return nil, nnfv1alpha1.NewWorkflowErrorf("No directive matching '%s' found in workflow", name).WithFatal() } diff --git a/controllers/nnf_workflow_controller_helpers.go b/controllers/nnf_workflow_controller_helpers.go index fed2ed6d0..6fb715307 100644 --- a/controllers/nnf_workflow_controller_helpers.go +++ b/controllers/nnf_workflow_controller_helpers.go @@ -133,14 +133,29 @@ func (r *NnfWorkflowReconciler) validateStagingDirective(ctx context.Context, wf validateStagingArgument := func(arg string) error { name, _ := splitStagingArgumentIntoNameAndPath(arg) if strings.HasPrefix(arg, "$DW_JOB_") { - if findDirectiveIndexByName(wf, name) == -1 { + index := findDirectiveIndexByName(wf, name, "jobdw") + if index == -1 { return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("Job storage instance '%s' not found", name)).WithFatal() } + + args, err := dwdparse.BuildArgsMap(wf.Spec.DWDirectives[index]) + if err != nil { + return nnfv1alpha1.NewWorkflowError("Invalid DW directive: " + wf.Spec.DWDirectives[index]).WithFatal() + } + + fsType, exists := args["type"] + if !exists { + return nnfv1alpha1.NewWorkflowError("Invalid DW directive match for staging argument") + } + + if fsType == "raw" { + return nnfv1alpha1.NewWorkflowError("Data movement can not be used with raw allocations").WithFatal() + } } else if strings.HasPrefix(arg, "$DW_PERSISTENT_") { - if err := r.validatePersistentInstanceByName(ctx, name, wf.Namespace); err != nil { + if err := r.validatePersistentInstanceForStaging(ctx, name, wf.Namespace); err != nil { return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("Persistent storage instance '%s' not found", name)).WithFatal() } - if findDirectiveIndexByName(wf, name) == -1 { + if findDirectiveIndexByName(wf, name, "persistentdw") == -1 { return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("persistentdw directive mentioning '%s' not found", name)).WithFatal() } } else { @@ -168,6 +183,20 @@ func (r *NnfWorkflowReconciler) validateStagingDirective(ctx context.Context, wf return nil } +// validatePersistentInstance validates the persistentdw directive. +func (r *NnfWorkflowReconciler) validatePersistentInstanceForStaging(ctx context.Context, name string, namespace string) error { + psi, err := r.getPersistentStorageInstance(ctx, name, namespace) + if err != nil { + return err + } + + if psi.Spec.FsType == "raw" { + return nnfv1alpha1.NewWorkflowError("Data movement can not be used with raw allocations").WithFatal() + } + + return nil +} + // validatePersistentInstance validates the persistentdw directive. func (r *NnfWorkflowReconciler) validatePersistentInstanceDirective(ctx context.Context, wf *dwsv1alpha1.Workflow, directive string) error { // Validate that the persistent instance is available and not in the process of being deleted @@ -176,18 +205,13 @@ func (r *NnfWorkflowReconciler) validatePersistentInstanceDirective(ctx context. return nnfv1alpha1.NewWorkflowError("Invalid DW directive: " + directive).WithFatal() } - return r.validatePersistentInstanceByName(ctx, args["name"], wf.Namespace) -} - -// validatePersistentInstance validates the persistentdw directive. -func (r *NnfWorkflowReconciler) validatePersistentInstanceByName(ctx context.Context, name string, namespace string) error { - psi, err := r.getPersistentStorageInstance(ctx, name, namespace) + psi, err := r.getPersistentStorageInstance(ctx, args["name"], wf.Namespace) if err != nil { return err } if !psi.DeletionTimestamp.IsZero() { - return nnfv1alpha1.NewWorkflowError("Persistent storage instance " + name + " is deleting").WithFatal() + return nnfv1alpha1.NewWorkflowError("Persistent storage instance " + args["name"] + " is deleting").WithFatal() } return nil @@ -559,10 +583,10 @@ func handleWorkflowError(err error, driverStatus *dwsv1alpha1.WorkflowDriverStat } // Returns the directive index with the 'name' argument matching name, or -1 if not found -func findDirectiveIndexByName(workflow *dwsv1alpha1.Workflow, name string) int { +func findDirectiveIndexByName(workflow *dwsv1alpha1.Workflow, name string, command string) int { for idx, directive := range workflow.Spec.DWDirectives { parameters, _ := dwdparse.BuildArgsMap(directive) - if parameters["name"] == name { + if parameters["name"] == name && parameters["command"] == command { return idx } } From a6780e625731e735cb26647b6a71cbc44a179aac Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Thu, 26 Jan 2023 10:57:36 -0600 Subject: [PATCH 16/71] RABSW-1097: Use "raw" instead of "lvm" for Raw allocation FsType (#132) * RABSW-1097: Use "raw" instead of "lvm" for Raw allocation FsType nnf-ec now understands the "raw" file system type. * re-vendor Signed-off-by: Matt Richerson Signed-off-by: Matt Richerson --- controllers/nnf_node_storage_controller.go | 10 +- go.mod | 2 +- go.sum | 4 +- .../NearNodeFlash/nnf-ec/pkg/ec/error.go | 17 +++ .../nnf-ec/pkg/manager-nnf/file_system.go | 2 +- .../nnf-ec/pkg/manager-nnf/manager.go | 3 +- .../nnf-ec/pkg/manager-nnf/router.go | 2 +- .../pkg/manager-server/file_system_api.go | 90 ++++++++++++--- .../pkg/manager-server/file_system_gfs2.go | 14 ++- .../pkg/manager-server/file_system_lustre.go | 26 ++++- .../pkg/manager-server/file_system_lvm.go | 101 +++++++++-------- .../pkg/manager-server/file_system_raw.go | 104 ++++++++++++++++++ .../pkg/manager-server/file_system_xfs.go | 8 +- .../pkg/manager-server/file_system_zfs.go | 9 +- vendor/modules.txt | 2 +- 15 files changed, 306 insertions(+), 88 deletions(-) create mode 100644 vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_raw.go diff --git a/controllers/nnf_node_storage_controller.go b/controllers/nnf_node_storage_controller.go index 754969d37..147b3eb4d 100644 --- a/controllers/nnf_node_storage_controller.go +++ b/controllers/nnf_node_storage_controller.go @@ -442,14 +442,8 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto condition.LastTransitionTime = metav1.Now() } - var fsType string - if nodeStorage.Spec.FileSystemType == "raw" { - fsType = "lvm" - } else { - fsType = nodeStorage.Spec.FileSystemType - } oem := nnfserver.FileSystemOem{ - Type: fsType, + Type: nodeStorage.Spec.FileSystemType, } if oem.Type == "lustre" { @@ -514,7 +508,7 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto setOpts(&nnfStorageProfile.Data.XFSStorage.Options) } - if oem.Type == "lvm" { + if oem.Type == "raw" { setCmdLines(&nnfStorageProfile.Data.RawStorage.CmdLines) } diff --git a/go.mod b/go.mod index 1676e5994..5e7293e3a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 + github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 40ecfa312..73059e4bf 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h1:upOQcpaEDmgLxoK7SZeqzGOQs8raxd9/paJYmBS/3/M= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 h1:j9KD7WrgMVJrCQHvFixazXS6kbxRSpH6oM+puj+MVdw= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089 h1:ZxXh6YNmnH50yHi0O5uOSNVG/7lI+pqDCB9wGMf9INU= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go index 4a71d759c..532972ce6 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/error.go @@ -21,6 +21,7 @@ package ec import ( "encoding/json" + "errors" "fmt" "net/http" "time" @@ -137,6 +138,22 @@ func NewErrNotImplemented() *ControllerError { return NewControllerError(http.StatusNotImplemented) } +// IsRetryable returns true and the retry delay if ANY errors emanating from the supplied error +// is an ec.ControllerError that is Retryable, and false otherwise. +func IsRetryable(err error) (bool, time.Duration) { + + for err != nil { + var ctrlErr *ControllerError + if errors.As(err, &ctrlErr) && ctrlErr.IsRetryable() { + return true, ctrlErr.RetryDelay() + } + + err = errors.Unwrap(err) + } + + return false, time.Duration(0) +} + type ErrorResponse struct { Status int `json:"status"` Error string `json:"error"` diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/file_system.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/file_system.go index 02a3d566b..85dfc6db4 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/file_system.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/file_system.go @@ -180,7 +180,7 @@ func (rh *fileSystemRecoveryReplyHandler) Metadata(data []byte) error { return err } - fsApi, err := server.FileSystemController.NewFileSystem(metadata.FileSystemOem) + fsApi, err := server.FileSystemController.NewFileSystem(&metadata.FileSystemOem) if err != nil { return fmt.Errorf("File System %s Replay: Failed to create api %s", rh.fileSystemId, err) } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go index efda973ae..62edf04e0 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go @@ -1202,7 +1202,7 @@ func (*StorageService) StorageServiceIdFileSystemsPost(storageServiceId string, return ec.NewErrBadRequest().WithResourceType(FileSystemOdataType).WithError(err).WithEvent(msgreg.MalformedJSONBase()) } - fsApi, err := server.FileSystemController.NewFileSystem(oem) + fsApi, err := server.FileSystemController.NewFileSystem(&oem) if err != nil { return ec.NewErrNotAcceptable().WithResourceType(FileSystemOdataType).WithError(err).WithCause("File system '%s' failed to allocate").WithEvent(msgreg.InternalErrorBase()) } @@ -1408,7 +1408,6 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdPut(storageSer ep := s.findEndpoint(endpointId) if ep == nil { return ec.NewErrNotAcceptable().WithResourceType(EndpointOdataType).WithEvent(msgreg.ResourceNotFoundBase(EndpointOdataType, endpointId)) - } sp := s.findStoragePool(fs.storagePoolId) diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go index 717ff2f02..78a1691db 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go @@ -246,7 +246,7 @@ func (r *DefaultApiRouter) Routes() ec.Routes { }, { Name: "RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemsIdExportedFileSharesExportedFileSharesIdDelete", - Method: ec.GET_METHOD, + Method: ec.DELETE_METHOD, Path: "/redfish/v1/StorageServices/{StorageServiceId}/FileSystems/{FileSystemsId}/ExportedFileShares/{ExportedFileSharesId}", HandlerFunc: s.RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemsIdExportedFileSharesExportedFileSharesIdDelete, }, diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_api.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_api.go index d93f1954a..dc274e22f 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_api.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_api.go @@ -26,6 +26,7 @@ import ( "io/fs" "os" "os/exec" + "reflect" log "github.com/sirupsen/logrus" "k8s.io/mount-utils" @@ -33,8 +34,13 @@ import ( "github.com/NearNodeFlash/nnf-ec/pkg/logging" ) +const ( + UserID = "userID" + GroupID = "groupID" +) + type FileSystemControllerApi interface { - NewFileSystem(oem FileSystemOem) (FileSystemApi, error) + NewFileSystem(oem *FileSystemOem) (FileSystemApi, error) } func NewFileSystemController(config *ConfigFile) FileSystemControllerApi { @@ -46,7 +52,7 @@ type fileSystemController struct { } // NewFileSystem - -func (c *fileSystemController) NewFileSystem(oem FileSystemOem) (FileSystemApi, error) { +func (c *fileSystemController) NewFileSystem(oem *FileSystemOem) (FileSystemApi, error) { return FileSystemRegistry.NewFileSystem(oem) } @@ -74,8 +80,8 @@ type FileSystemOptions = map[string]interface{} type FileSystemApi interface { New(oem FileSystemOem) (FileSystemApi, error) - IsType(oem FileSystemOem) bool // Returns true if the provided oem fields match the file system type, false otherwise - IsMockable() bool // Returns true if the file system can be instantiated by the mock server, false otherwise + IsType(oem *FileSystemOem) bool // Returns true if the provided oem fields match the file system type, false otherwise + IsMockable() bool // Returns true if the file system can be instantiated by the mock server, false otherwise Type() string Name() string @@ -83,6 +89,9 @@ type FileSystemApi interface { Create(devices []string, opts FileSystemOptions) error Delete() error + VgChangeActivateDefault() string + MkfsDefault() string + Mount(mountpoint string) error Unmount(mountpoint string) error @@ -117,6 +126,7 @@ func (f *FileSystem) mount(source string, target string, fstype string, options if !mounted { if err := mounter.Mount(source, target, fstype, options); err != nil { log.Errorf("Mount failed: %v", err) + return err } } @@ -209,6 +219,26 @@ type FileSystemOemLvm struct { // The lvcreate commandline, minus the "lvcreate" command. LvCreate string `json:"LvCreate,omitempty"` + + // The vgchange commandline, minus the "vgchange" command. + VgChange FileSystemOemVgChange `json:"VgChange,omitempty"` + + // The vgremove commandline, minus the "vgremove" command. + VgRemove string `json:"VgRemove,omitempty"` + + // The lvremove commandline, minus the "lvremove" command + LvRemove string `json:"LvRemove,omitempty"` +} + +type FileSystemOemVgChange struct { + // The vgchange commandline, minus the "vgchange" command + Activate string `json:"Activate,omitempty"` + + // The vgchange commandline, minus the "vgchange" command + Deactivate string `json:"Deactivate,omitempty"` + + // The vgchange commandline, minus the "vgchange" command + LockStart string `json:"Lockstart,omitempty"` } type FileSystemOemZfs struct { @@ -232,8 +262,9 @@ type FileSystemOemGfs2 struct { // File System OEM defines the structure that is expected to be included inside a // Redfish / Swordfish FileSystemV122FileSystem type FileSystemOem struct { - Type string `json:"Type"` - Name string `json:"Name"` + Type string `json:"Type"` + Name string `json:"Name"` + Lustre FileSystemOemLustre `json:"Lustre,omitempty"` Gfs2 FileSystemOemGfs2 `json:"Gfs2,omitempty"` @@ -242,6 +273,37 @@ type FileSystemOem struct { ZfsCmd FileSystemOemZfs `json:"Zfs,omitempty"` } +func (oem *FileSystemOem) IsEmpty() bool { + return reflect.DeepEqual(oem, &FileSystemOem{Type: oem.Type, Name: oem.Name}) +} + +func (oem *FileSystemOem) LoadDefaults(fs FileSystemApi) { + oem.LvmCmd = FileSystemOemLvm{ + PvCreate: "$DEVICE", + VgCreate: "$VG_NAME $DEVICE_LIST", + LvCreate: "--extents 100%VG --stripes $DEVICE_NUM --stripesize 32KiB --name $LV_NAME $VG_NAME", + VgChange: FileSystemOemVgChange{ + Activate: fs.VgChangeActivateDefault(), + Deactivate: "--activate n $VG_NAME", + LockStart: "--lock-start $VG_NAME", + }, + VgRemove: "$VG_NAME", + LvRemove: "$VG_NAME", + } + + oem.Gfs2 = FileSystemOemGfs2{ + ClusterName: "$CLUSTER_NAME", + } + + oem.ZfsCmd = FileSystemOemZfs{ + ZpoolCreate: "-O canmount=off -o cachefile=none $POOL_NAME $DEVICE_LIST", + } + + oem.MkfsMount = FileSystemOemMkfsMount{ + Mkfs: fs.MkfsDefault(), + } +} + // File System Registry - Maintains a list of eligible file systems registered in the system. type fileSystemRegistry []FileSystemApi @@ -266,22 +328,22 @@ func (r *fileSystemRegistry) RegisterFileSystem(fileSystem FileSystemApi) { *r = append(*r, fileSystem) } -func (r *fileSystemRegistry) NewFileSystem(oem FileSystemOem) (FileSystemApi, error) { +func (r *fileSystemRegistry) NewFileSystem(oem *FileSystemOem) (FileSystemApi, error) { for _, fs := range *r { if fs.IsType(oem) { - return fs.New(oem) + + if oem.IsEmpty() { + oem.LoadDefaults(fs) + } + + return fs.New(*oem) } } - return nil, nil + return nil, fmt.Errorf("file systems '%s' not supported", oem.Type) } func setFileSystemPermissions(f FileSystemApi, opts FileSystemOptions) (err error) { - const ( - UserID = "userID" - GroupID = "groupID" - ) - userID := 0 if _, exists := opts[UserID]; exists { userID = opts[UserID].(int) diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_gfs2.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_gfs2.go index 6ff467a86..8a5e78f65 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_gfs2.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_gfs2.go @@ -80,12 +80,20 @@ func (*FileSystemGfs2) New(oem FileSystemOem) (FileSystemApi, error) { }, nil } -func (*FileSystemGfs2) IsType(oem FileSystemOem) bool { return oem.Type == "gfs2" } -func (*FileSystemGfs2) IsMockable() bool { return false } -func (*FileSystemGfs2) Type() string { return "gfs2" } +func (*FileSystemGfs2) IsType(oem *FileSystemOem) bool { return oem.Type == "gfs2" } +func (*FileSystemGfs2) IsMockable() bool { return false } +func (*FileSystemGfs2) Type() string { return "gfs2" } func (f *FileSystemGfs2) Name() string { return f.name } +func (*FileSystemGfs2) VgChangeActivateDefault() string { + return "--activate sy $VG_NAME" +} + +func (*FileSystemGfs2) MkfsDefault() string { + return "-j2 -p $PROTOCOL -t $CLUSTER_NAME:$LOCK_SPACE $DEVICE" +} + func (f *FileSystemGfs2) Create(devices []string, opts FileSystemOptions) (err error) { if err := f.FileSystemLvm.Create(devices, opts); err != nil { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go index 105d2fa3d..71d1ae730 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lustre.go @@ -58,6 +58,8 @@ type FileSystemLustre struct { // Satisfy FileSystemApi interface. FileSystem + TargetType string + Oem FileSystemOemLustre MkfsMount FileSystemOemMkfsMount ZfsArgs FileSystemOemZfs @@ -66,14 +68,14 @@ type FileSystemLustre struct { func (*FileSystemLustre) New(oem FileSystemOem) (FileSystemApi, error) { return &FileSystemLustre{ FileSystem: FileSystem{name: oem.Name}, - // TargetType and BackFs are already verified by IsType() below. - Oem: oem.Lustre, - MkfsMount: oem.MkfsMount, - ZfsArgs: oem.ZfsCmd, + TargetType: targetTypes[oem.Lustre.TargetType], + Oem: oem.Lustre, + MkfsMount: oem.MkfsMount, + ZfsArgs: oem.ZfsCmd, }, nil } -func (*FileSystemLustre) IsType(oem FileSystemOem) bool { +func (*FileSystemLustre) IsType(oem *FileSystemOem) bool { _, ok := targetTypes[oem.Lustre.TargetType] if ok { _, ok = backFsTypes[oem.Lustre.BackFs] @@ -83,7 +85,19 @@ func (*FileSystemLustre) IsType(oem FileSystemOem) bool { func (*FileSystemLustre) IsMockable() bool { return false } func (*FileSystemLustre) Type() string { return "lustre" } -func (f *FileSystemLustre) Name() string { return f.name } + +func (f *FileSystemLustre) Name() string { return f.name } + +func (f *FileSystemLustre) VgChangeActivateDefault() string { return "" } + +func (f *FileSystemLustre) MkfsDefault() string { + return map[string]string{ + TargetMGT: "--mgs $VOL_NAME", + TargetMDT: "--mdt --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $VOL_NAME", + TargetMGTMDT: "--mgs --mdt --fsname=$FS_NAME --index=$INDEX $VOL_NAME", + TargetOST: "--ost --fsname=$FS_NAME --mgsnode=$MGS_NID --index=$INDEX $VOL_NAME", + }[f.TargetType] +} func (f *FileSystemLustre) Create(devices []string, options FileSystemOptions) error { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lvm.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lvm.go index 8178ef1ab..e5424e9c8 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lvm.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_lvm.go @@ -22,8 +22,9 @@ package server import ( "fmt" "os" - "path/filepath" + "path" "strings" + "time" "github.com/NearNodeFlash/nnf-ec/pkg/var_handler" ) @@ -54,12 +55,15 @@ func (*FileSystemLvm) New(oem FileSystemOem) (FileSystemApi, error) { }, nil } -func (*FileSystemLvm) IsType(oem FileSystemOem) bool { return oem.Type == "lvm" } -func (*FileSystemLvm) IsMockable() bool { return false } +func (*FileSystemLvm) IsType(oem *FileSystemOem) bool { return oem.Type == "lvm" } +func (*FileSystemLvm) IsMockable() bool { return false } +func (*FileSystemLvm) Type() string { return "lvm" } -func (*FileSystemLvm) Type() string { return "lvm" } func (f *FileSystemLvm) Name() string { return f.name } +func (f *FileSystemLvm) VgChangeActivateDefault() string { return "--activate y $VG_NAME" } +func (f *FileSystemLvm) MkfsDefault() string { return "" } + func (f *FileSystemLvm) Create(devices []string, opts FileSystemOptions) error { if _, exists := opts[VolumeGroupName]; exists { @@ -83,35 +87,38 @@ func (f *FileSystemLvm) Create(devices []string, opts FileSystemOptions) error { return err } - rsp, _ := f.run(fmt.Sprintf("lvdisplay %s || echo 'not found'", f.vgName)) - if len(rsp) != 0 && !strings.Contains(string(rsp), "not found") { - // Volume Group is present, activate the volume. This is for the case where another - // node created the volume group and we just want to share it. - shared := "" + varHandler := var_handler.NewVarHandler(map[string]string{ + "$DEVICE_NUM": fmt.Sprintf("%d", len(devices)), + "$DEVICE_LIST": strings.Join(devices, " "), + "$LV_NAME": f.lvName, + "$VG_NAME": f.vgName, + }) + if err := varHandler.ListToVars("$DEVICE_LIST", "$DEVICE"); err != nil { + return fmt.Errorf("invalid internal device list: %w", err) + } + + activateVolumeGroup := func() error { if f.shared { // Activate the shared lock - if _, err := f.run(fmt.Sprintf("vgchange --lock-start %s", f.vgName)); err != nil { + vgArgs := varHandler.ReplaceAll(f.CmdArgs.VgChange.LockStart) + if _, err := f.run(fmt.Sprintf("vgchange %s", vgArgs)); err != nil { return err } - - shared = "s" // activate with shared option } - if _, err := f.run(fmt.Sprintf("vgchange --activate %sy %s", shared, f.vgName)); err != nil { + vgArgs := varHandler.ReplaceAll(f.CmdArgs.VgChange.Activate) + if _, err := f.run(fmt.Sprintf("vgchange %s", vgArgs)); err != nil { return err } return nil } - varHandler := var_handler.NewVarHandler(map[string]string{ - "$DEVICE_NUM": fmt.Sprintf("%d", len(devices)), - "$DEVICE_LIST": strings.Join(devices, " "), - "$LV_NAME": f.lvName, - "$VG_NAME": f.vgName, - }) - if err := varHandler.ListToVars("$DEVICE_LIST", "$DEVICE"); err != nil { - return fmt.Errorf("invalid internal device list: %w", err) + rsp, _ := f.run(fmt.Sprintf("lvdisplay %s || echo 'not found'", f.vgName)) + if len(rsp) != 0 && !strings.Contains(string(rsp), "not found") { + // Volume Group is present, activate the volume. This is for the case where another + // node created the volume group and we just want to share it. + return activateVolumeGroup() } // Create the physical volumes. @@ -135,17 +142,12 @@ func (f *FileSystemLvm) Create(devices []string, opts FileSystemOptions) error { return err } - shared = "" - if f.shared { - if _, err := f.run(fmt.Sprintf("vgchange --lockstart %s", f.vgName)); err != nil { - return err - } - - shared = "s" + if err := activateVolumeGroup(); err != nil { + return err } // Create the logical volume - // -Zn - don't zero the volume, it will fail. + // --zero n - don't zero the volume, it will fail. // We are depending on the drive behavior for newly allocated blocks to track // NVM Command Set spec, Section 3.2.3.2.1 Deallocated or Unwritten Logical Blocks // The Kioxia drives support DLFEAT=001b @@ -159,36 +161,49 @@ func (f *FileSystemLvm) Create(devices []string, opts FileSystemOptions) error { // error is not enabled, the values read from a deallocated or unwritten block and its metadata (excluding protection information) // shall be: // • all bytes cleared to 0h if bits 2:0 in the DLFEAT field are set to 001b; - zeroOpt := "-Zn" + zeroOpt := "--zero n" if _, ok := os.LookupEnv("NNF_SUPPLIED_DEVICES"); ok { // On non-NVMe, let the zeroing happen so devices can be reused. - zeroOpt = "--yes" + zeroOpt = "--zero y" } lvArgs := varHandler.ReplaceAll(f.CmdArgs.LvCreate) - if _, err := f.run(fmt.Sprintf("lvcreate %s %s", zeroOpt, lvArgs)); err != nil { + if _, err := f.run(fmt.Sprintf("lvcreate --yes %s %s", zeroOpt, lvArgs)); err != nil { return err } - // Activate the volume group. - if _, err := f.run(fmt.Sprintf("vgchange --activate %sy %s", shared, f.vgName)); err != nil { - return err + // Ensure that the Logical Volume exists prior to returning; otherwise a future 'mkfs' command may fail. + /* + NJT: Can't do this yet since to Persistent() controller all errors are fatal and perform a Rollback. + if _, err := os.Stat(f.devPath()); os.IsNotExist(err) { + return ec.NewErrorNotReady().WithCause(fmt.Sprintf("logical volume '%s' does not exist", f.devPath())) + } + */ + + // HACK: Wait up to 10 seconds for the path to come ready + for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(time.Second) { + if _, err := os.Stat(f.devPath()); !os.IsNotExist(err) { + return nil + } } - return nil + return fmt.Errorf("logical volume path '%s' does not exist after 10s", f.devPath()) } func (f *FileSystemLvm) Delete() error { - if _, err := f.run(fmt.Sprintf("lvremove --yes %s", f.devPath())); err != nil { - return err - } + varHandler := var_handler.NewVarHandler(map[string]string{ + "$LV_NAME": f.lvName, + "$VG_NAME": f.vgName, + }) - if _, err := f.run(fmt.Sprintf("vgchange --activate n %s", f.vgName)); err != nil { + vgArgs := varHandler.ReplaceAll(f.CmdArgs.VgChange.Deactivate) + if _, err := f.run(fmt.Sprintf("vgchange %s", vgArgs)); err != nil { return err } - if _, err := f.run(fmt.Sprintf("vgremove --yes %s", f.vgName)); err != nil { + vgArgs = varHandler.ReplaceAll(f.CmdArgs.VgRemove) + if _, err := f.run(fmt.Sprintf("vgremove --yes %s", vgArgs)); err != nil { return err } @@ -202,7 +217,7 @@ func (f *FileSystemLvm) Delete() error { } func (f *FileSystemLvm) Mount(mountpoint string) error { - return f.mount(f.devPath(), mountpoint, "", []string{"--bind"}) + return nil } func (f *FileSystemLvm) GenerateRecoveryData() map[string]string { @@ -218,5 +233,5 @@ func (f *FileSystemLvm) LoadRecoveryData(data map[string]string) { } func (f *FileSystemLvm) devPath() string { - return filepath.Join("/dev", f.vgName, f.lvName) + return path.Join("/dev", f.vgName, f.lvName) } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_raw.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_raw.go new file mode 100644 index 000000000..f1831c123 --- /dev/null +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_raw.go @@ -0,0 +1,104 @@ +/* + * Copyright 2020, 2021, 2022, 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" + "k8s.io/mount-utils" +) + +type FileSystemRaw struct { + FileSystemLvm + + uid int + gid int +} + +func init() { + FileSystemRegistry.RegisterFileSystem(&FileSystemRaw{}) +} + +func (*FileSystemRaw) New(oem FileSystemOem) (FileSystemApi, error) { + return &FileSystemRaw{ + FileSystemLvm: FileSystemLvm{ + FileSystem: FileSystem{name: oem.Name}, + CmdArgs: oem.LvmCmd, + shared: false, + }, + }, nil +} + +func (*FileSystemRaw) IsType(oem *FileSystemOem) bool { return oem.Type == "raw" } +func (*FileSystemRaw) Type() string { return "raw" } + + +func (f *FileSystemRaw) Create(devices []string, opts FileSystemOptions) error { + if err := f.FileSystemLvm.Create(devices, opts); err != nil { + return err + } + + if _, exists := opts[UserID]; exists { + f.uid = opts[UserID].(int) + } else { + f.uid = 0 + } + + if _, exists := opts[GroupID]; exists { + f.gid = opts[GroupID].(int) + } else { + f.gid = 0 + } + + return nil +} + +func (f *FileSystemRaw) Mount(mountpoint string) error { + // Make the parent directory + if err := os.MkdirAll(filepath.Dir(mountpoint), 0755); err != nil { + return err + } + + // Create an empty file to bind mount the device onto + if err := os.WriteFile(mountpoint, []byte(""), 0644); err != nil { + return err + } + + mounter := mount.New("") + mounted, err := mounter.IsMountPoint(mountpoint) + if err != nil { + return err + } + + if !mounted { + if err := mounter.Mount(f.devPath(), mountpoint, "none", []string{"bind"}); err != nil { + log.Errorf("Mount failed: %v", err) + return err + } + + if err := os.Chown(f.devPath(), f.uid, f.gid); err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_xfs.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_xfs.go index 4c02a14c4..6e27bc919 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_xfs.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_xfs.go @@ -46,12 +46,14 @@ func (*FileSystemXfs) New(oem FileSystemOem) (FileSystemApi, error) { }, nil } -func (*FileSystemXfs) IsType(oem FileSystemOem) bool { return oem.Type == "xfs" } -func (*FileSystemXfs) IsMockable() bool { return false } +func (*FileSystemXfs) IsType(oem *FileSystemOem) bool { return oem.Type == "xfs" } +func (*FileSystemXfs) IsMockable() bool { return false } +func (*FileSystemXfs) Type() string { return "xfs" } -func (*FileSystemXfs) Type() string { return "xfs" } func (f *FileSystemXfs) Name() string { return f.name } +func (f *FileSystemXfs) MkfsDefault() string { return "$DEVICE" } + func (f *FileSystemXfs) Create(devices []string, opts FileSystemOptions) (err error) { if err := f.FileSystemLvm.Create(devices, opts); err != nil { return err diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_zfs.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_zfs.go index ebb9188d5..9a8f8535b 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_zfs.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/file_system_zfs.go @@ -37,12 +37,15 @@ func (*FileSystemZfs) New(oem FileSystemOem) (FileSystemApi, error) { return &FileSystemZfs{FileSystem: FileSystem{name: oem.Name}}, nil } -func (*FileSystemZfs) IsType(oem FileSystemOem) bool { return oem.Type == "zfs" } -func (*FileSystemZfs) IsMockable() bool { return false } +func (*FileSystemZfs) IsType(oem *FileSystemOem) bool { return oem.Type == "zfs" } +func (*FileSystemZfs) IsMockable() bool { return false } +func (*FileSystemZfs) Type() string { return "zfs" } -func (*FileSystemZfs) Type() string { return "zfs" } func (f *FileSystemZfs) Name() string { return f.name } +func (f *FileSystemZfs) VgChangeActivateDefault() string { return "" } +func (f *FileSystemZfs) MkfsDefault() string { return "" } + func (f *FileSystemZfs) Create(devices []string, options FileSystemOptions) error { f.devices = devices diff --git a/vendor/modules.txt b/vendor/modules.txt index 9e4d52ceb..2c121306d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20230111193528-18bafb87b268 +# github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089 ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From 5e390c04be6963c596e28f403b6a51a05898ba0e Mon Sep 17 00:00:00 2001 From: Dean Roehrich Date: Thu, 26 Jan 2023 15:57:49 -0600 Subject: [PATCH 17/71] Allow builds on all branches. (#134) Loosen the branch filter for pushes. Print some event context, for future debugging. Remove an unused variable from verify_tag. Rename some jobs to give them unique names, to help with debugging. Signed-off-by: Dean Roehrich --- .github/workflows/main.yml | 8 ++++++-- .github/workflows/verify_tag.yml | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2e79507c..90c2c21ec 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,8 +3,7 @@ name: Docker build and push on: push: branches: - - 'master' - - 'releases/v*' + - '*' tags: - 'v*' pull_request: @@ -27,6 +26,11 @@ jobs: runs-on: ubuntu-latest steps: + - name: "Build context" + run: | + echo "ref is ${{ github.ref }}" + echo "ref_type is ${{ github.ref_type }}" + - name: "Checkout repository" id: checkout_repo uses: actions/checkout@v3 diff --git a/.github/workflows/verify_tag.yml b/.github/workflows/verify_tag.yml index d4b6fe973..2a75d9109 100644 --- a/.github/workflows/verify_tag.yml +++ b/.github/workflows/verify_tag.yml @@ -7,13 +7,15 @@ on: tags: - "v*" -env: - IMAGE_NAME: ${{ github.repository }} - jobs: - build: + verify_tag: runs-on: ubuntu-latest steps: + - name: "Verify context" + run: | + echo "ref is ${{ github.ref }}" + echo "ref_type is ${{ github.ref_type }}" + - uses: actions/checkout@v3 # actions/checkout@v3 breaks annotated tags by converting them into # lightweight tags, so we need to force fetch the tag again From 33b79a66ef4117b8ccb36febb3fd30e27bd01269 Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Mon, 30 Jan 2023 16:24:44 -0600 Subject: [PATCH 18/71] RABSW-1124: Change NnfAccess TeardownState for servers (#136) * RABSW-1124: Change NnfAccess TeardownState for servers The data movement code was mounting and unmounting the Rabbit nodes during the DataIn and DataOut phases of the workflow. A stale workflow resource in the client cache could cause the NnfAccess to be re-mounted after it had already been unmounted. This commit changes the NnfAccess Teardown state logic to do the unmounts in PreRun and Teardown instead of DataIn and DataOut. Signed-off-by: Matt Richerson * review comments Signed-off-by: Matt Richerson --------- Signed-off-by: Matt Richerson --- api/v1alpha1/nnf_access_types.go | 2 +- .../bases/nnf.cray.hpe.com_nnfaccesses.yaml | 3 +- controllers/integration_test.go | 6 +- controllers/nnf_access_controller_test.go | 2 +- controllers/nnf_workflow_controller.go | 56 ++++++------------- .../nnf_workflow_controller_helpers.go | 10 +++- 6 files changed, 30 insertions(+), 49 deletions(-) diff --git a/api/v1alpha1/nnf_access_types.go b/api/v1alpha1/nnf_access_types.go index 0ac2970b6..f45c8acfd 100644 --- a/api/v1alpha1/nnf_access_types.go +++ b/api/v1alpha1/nnf_access_types.go @@ -35,7 +35,7 @@ type NnfAccessSpec struct { // TeardownState is the desired state of the workflow for this NNF Access resource to // be torn down and deleted. - // +kubebuilder:validation:Enum:=DataIn;PreRun;PostRun;DataOut + // +kubebuilder:validation:Enum:=PreRun;PostRun;Teardown // +kubebuilder:validation:Type:=string TeardownState dwsv1alpha1.WorkflowState `json:"teardownState"` diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml index d0e274f79..6c019eb36 100644 --- a/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml +++ b/config/crd/bases/nnf.cray.hpe.com_nnfaccesses.yaml @@ -163,10 +163,9 @@ spec: - DataOut - Teardown - enum: - - DataIn - PreRun - PostRun - - DataOut + - Teardown description: TeardownState is the desired state of the workflow for this NNF Access resource to be torn down and deleted. type: string diff --git a/controllers/integration_test.go b/controllers/integration_test.go index ef8928e62..8b235c46a 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -1130,11 +1130,11 @@ var _ = Describe("Integration Test", func() { advanceStateAndCheckReady(dwsv1alpha1.StatePreRun, workflow) By("Validate NNF Access is created, with deletion in data-out") - validateNnfAccessHasCorrectTeardownState(dwsv1alpha1.StateDataOut) + validateNnfAccessHasCorrectTeardownState(dwsv1alpha1.StateTeardown) By("Advancing to post run, ensure NNF Access is still set for deletion in data-out") advanceStateAndCheckReady(dwsv1alpha1.StatePostRun, workflow) - validateNnfAccessHasCorrectTeardownState(dwsv1alpha1.StateDataOut) + validateNnfAccessHasCorrectTeardownState(dwsv1alpha1.StateTeardown) By("Advancing to data-out, ensure NNF Access is deleted") advanceStateAndCheckReady(dwsv1alpha1.StateDataOut, workflow) @@ -1155,7 +1155,7 @@ var _ = Describe("Integration Test", func() { Expect(workflow.Status.State).To(Equal(dwsv1alpha1.StateDataIn)) By("Validate NNF Access is created, with deletion in data-out") - validateNnfAccessHasCorrectTeardownState(dwsv1alpha1.StateDataOut) + validateNnfAccessHasCorrectTeardownState(dwsv1alpha1.StateTeardown) advanceStateAndCheckReady(dwsv1alpha1.StatePreRun, workflow) advanceStateAndCheckReady(dwsv1alpha1.StatePostRun, workflow) diff --git a/controllers/nnf_access_controller_test.go b/controllers/nnf_access_controller_test.go index 6570c3645..c5e98f719 100644 --- a/controllers/nnf_access_controller_test.go +++ b/controllers/nnf_access_controller_test.go @@ -110,7 +110,7 @@ var _ = Describe("Access Controller Test", func() { }, Spec: nnfv1alpha1.NnfAccessSpec{ DesiredState: "mounted", - TeardownState: dwsv1alpha1.StateDataIn, + TeardownState: dwsv1alpha1.StatePreRun, Target: "all", ClientReference: corev1.ObjectReference{}, MountPath: "./", diff --git a/controllers/nnf_workflow_controller.go b/controllers/nnf_workflow_controller.go index 37ff10b3e..48261e6d3 100644 --- a/controllers/nnf_workflow_controller.go +++ b/controllers/nnf_workflow_controller.go @@ -515,17 +515,17 @@ func (r *NnfWorkflowReconciler) startDataInOutState(ctx context.Context, workflo // when to teardown an NNF Access for the servers var teardownState dwsv1alpha1.WorkflowState if dwArgs["command"] == "copy_in" { - teardownState = dwsv1alpha1.StateDataIn + teardownState = dwsv1alpha1.StatePreRun if fsType == "gfs2" || fsType == "lustre" { teardownState = dwsv1alpha1.StatePostRun if findCopyOutDirectiveIndexByName(workflow, name) >= 0 { - teardownState = dwsv1alpha1.StateDataOut + teardownState = dwsv1alpha1.StateTeardown } } } else if dwArgs["command"] == "copy_out" { - teardownState = dwsv1alpha1.StateDataOut + teardownState = dwsv1alpha1.StateTeardown } // Setup NNF Access for the NNF Servers so we can run data movement on them. @@ -736,42 +736,6 @@ func (r *NnfWorkflowReconciler) finishDataInOutState(ctx context.Context, workfl } } - // Unmount any NNF Accesses that are no longer needed and cannot be shared (i.e. an XFS volume) - // These NNF Accesses should have a teardown label with the current data_in/data_out state. - dwArgs, err := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) - if err != nil { - return nil, nnfv1alpha1.NewWorkflowErrorf("Invalid DW directive index %d: %s", index, workflow.Spec.DWDirectives[index]).WithFatal() - } - - unmountIfNecessary := func(param string) (*result, error) { - name, _ := splitStagingArgumentIntoNameAndPath(param) - - if strings.HasPrefix(param, "$DW_JOB_") || strings.HasPrefix(param, "$DW_PERSISTENT_") { - parentDwIndex := 0 - if strings.HasPrefix(param, "$DW_PERSISTENT_") { - parentDwIndex = findDirectiveIndexByName(workflow, name, "persistentdw") - } else { - parentDwIndex = findDirectiveIndexByName(workflow, name, "jobdw") - } - - if parentDwIndex < 0 { - return nil, nnfv1alpha1.NewWorkflowErrorf("No directive matching '%s' found in workflow", name).WithFatal() - } - - return r.unmountNnfAccessIfNecessary(ctx, workflow, parentDwIndex, "servers") - } - - return nil, nil - } - - if result, err := unmountIfNecessary(dwArgs["source"]); result != nil || err != nil { - return result, err - } - - if result, err := unmountIfNecessary(dwArgs["destination"]); result != nil || err != nil { - return result, err - } - return nil, nil } @@ -779,6 +743,18 @@ func (r *NnfWorkflowReconciler) startPreRunState(ctx context.Context, workflow * log := r.Log.WithValues("Workflow", client.ObjectKeyFromObject(workflow), "Index", index) dwArgs, _ := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) + // If there's an NnfAccess for the servers that needs to be unmounted, do it now. This is + // required for XFS mounts where the compute node and Rabbit can't be mounted at the same + // time. + unmountResult, err := r.unmountNnfAccessIfNecessary(ctx, workflow, index, "servers") + if err != nil { + return nil, err + } + + if unmountResult != nil { + return unmountResult, nil + } + access := &nnfv1alpha1.NnfAccess{ ObjectMeta: metav1.ObjectMeta{ Name: indexedResourceName(workflow, index) + "-computes", @@ -855,7 +831,7 @@ func (r *NnfWorkflowReconciler) startPreRunState(ctx context.Context, workflow * // DataOut. teardownState := dwsv1alpha1.StatePostRun if findCopyOutDirectiveIndexByName(workflow, dwArgs["name"]) >= 0 { - teardownState = dwsv1alpha1.StateDataOut + teardownState = dwsv1alpha1.StateTeardown } _, err := r.setupNnfAccessForServers(ctx, storage, workflow, index, index, teardownState, log) diff --git a/controllers/nnf_workflow_controller_helpers.go b/controllers/nnf_workflow_controller_helpers.go index 6fb715307..db5a68e6d 100644 --- a/controllers/nnf_workflow_controller_helpers.go +++ b/controllers/nnf_workflow_controller_helpers.go @@ -691,9 +691,11 @@ func (r *NnfWorkflowReconciler) unmountNnfAccessIfNecessary(ctx context.Context, }, } + // Check if the NnfAccess exists. If it doesn't, then we don't need to unmount anything. If we didn't + // find it in the cache and it really does exist, we'll eventually get an event from the API server and + // do the unmount then. if err := r.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { - err = fmt.Errorf("Could not get NnfAccess %v: %w", client.ObjectKeyFromObject(access), err) - return nil, nnfv1alpha1.NewWorkflowError("Unable to find compute node mount information").WithError(err) + return nil, client.IgnoreNotFound(err) } teardownState, found := access.Labels[nnfv1alpha1.DataMovementTeardownStateLabel] @@ -710,6 +712,10 @@ func (r *NnfWorkflowReconciler) unmountNnfAccessIfNecessary(ctx context.Context, return Requeue("conflict").withObject(access), nil } } + + if access.Status.State != "unmounted" || !access.Status.Ready { + return Requeue("pending unmount").withObject(access), nil + } } return nil, nil From 88d43b8aa91e72c6263e764437824142455065fc Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Mon, 30 Jan 2023 17:21:05 -0600 Subject: [PATCH 19/71] Update to latest nnf-ec with support for additional LVM commands (#130) Signed-off-by: Nate Thornton --- api/v1alpha1/nnfstorageprofile_types.go | 21 ++++++ api/v1alpha1/zz_generated.deepcopy.go | 16 ++++ .../nnf.cray.hpe.com_nnfstorageprofiles.yaml | 75 +++++++++++++++++++ .../nnf_v1alpha1_nnfstorageprofile.yaml | 16 ++++ .../placeholder_nnfstorageprofile.yaml | 16 ++++ controllers/nnf_node_storage_controller.go | 7 ++ 6 files changed, 151 insertions(+) diff --git a/api/v1alpha1/nnfstorageprofile_types.go b/api/v1alpha1/nnfstorageprofile_types.go index a1511fcc4..ae78131b6 100644 --- a/api/v1alpha1/nnfstorageprofile_types.go +++ b/api/v1alpha1/nnfstorageprofile_types.go @@ -105,8 +105,29 @@ type NnfStorageProfileCmdLines struct { // VgCreate specifies the vgcreate commandline, minus the "vgcreate". VgCreate string `json:"vgCreate,omitempty"` + // VgChange specifies the various vgchange commandlines, minus the "vgchange" + VgChange NnfStorageProfileLVMVgChangeCmdLines `json:"vgChange,omitempty"` + + // VgCreate specifies the vgcreate commandline, minus the "vgremove". + VgRemove string `json:"vgRemove,omitempty"` + // LvCreate specifies the lvcreate commandline, minus the "lvcreate". LvCreate string `json:"lvCreate,omitempty"` + + // LvRemove specifies the lvcreate commandline, minus the "lvremove". + LvRemove string `json:"lvRemove,omitempty"` +} + +// NnfStorageProfileLVMVgChangeCmdLines +type NnfStorageProfileLVMVgChangeCmdLines struct { + // The vgchange commandline for activation, minus the "vgchange" command + Activate string `json:"activate,omitempty"` + + // The vgchange commandline for deactivation, minus the "vgchange" command + Deactivate string `json:"deactivate,omitempty"` + + // The vgchange commandline for lockStart, minus the "vgchange" command + LockStart string `json:"lockStart,omitempty"` } // NnfStorageProfileMiscOptions defines options to use for the mount library, and other utilities. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 45a0bb485..30b43c54b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -908,6 +908,7 @@ func (in *NnfStorageProfile) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NnfStorageProfileCmdLines) DeepCopyInto(out *NnfStorageProfileCmdLines) { *out = *in + out.VgChange = in.VgChange } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfStorageProfileCmdLines. @@ -956,6 +957,21 @@ func (in *NnfStorageProfileGFS2Data) DeepCopy() *NnfStorageProfileGFS2Data { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfStorageProfileLVMVgChangeCmdLines) DeepCopyInto(out *NnfStorageProfileLVMVgChangeCmdLines) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfStorageProfileLVMVgChangeCmdLines. +func (in *NnfStorageProfileLVMVgChangeCmdLines) DeepCopy() *NnfStorageProfileLVMVgChangeCmdLines { + if in == nil { + return nil + } + out := new(NnfStorageProfileLVMVgChangeCmdLines) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NnfStorageProfileList) DeepCopyInto(out *NnfStorageProfileList) { *out = *in diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfstorageprofiles.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfstorageprofiles.yaml index 98f7d9cb2..18832353b 100644 --- a/config/crd/bases/nnf.cray.hpe.com_nnfstorageprofiles.yaml +++ b/config/crd/bases/nnf.cray.hpe.com_nnfstorageprofiles.yaml @@ -52,6 +52,10 @@ spec: description: LvCreate specifies the lvcreate commandline, minus the "lvcreate". type: string + lvRemove: + description: LvRemove specifies the lvcreate commandline, + minus the "lvremove". + type: string mkfs: description: Mkfs specifies the mkfs commandline, minus the "mkfs". @@ -60,10 +64,31 @@ spec: description: PvCreate specifies the pvcreate commandline, minus the "pvcreate". type: string + vgChange: + description: VgChange specifies the various vgchange commandlines, + minus the "vgchange" + properties: + activate: + description: The vgchange commandline for activation, + minus the "vgchange" command + type: string + deactivate: + description: The vgchange commandline for deactivation, + minus the "vgchange" command + type: string + lockStart: + description: The vgchange commandline for lockStart, minus + the "vgchange" command + type: string + type: object vgCreate: description: VgCreate specifies the vgcreate commandline, minus the "vgcreate". type: string + vgRemove: + description: VgCreate specifies the vgcreate commandline, + minus the "vgremove". + type: string type: object options: description: Options contains options for libraries. @@ -257,6 +282,10 @@ spec: description: LvCreate specifies the lvcreate commandline, minus the "lvcreate". type: string + lvRemove: + description: LvRemove specifies the lvcreate commandline, + minus the "lvremove". + type: string mkfs: description: Mkfs specifies the mkfs commandline, minus the "mkfs". @@ -265,10 +294,31 @@ spec: description: PvCreate specifies the pvcreate commandline, minus the "pvcreate". type: string + vgChange: + description: VgChange specifies the various vgchange commandlines, + minus the "vgchange" + properties: + activate: + description: The vgchange commandline for activation, + minus the "vgchange" command + type: string + deactivate: + description: The vgchange commandline for deactivation, + minus the "vgchange" command + type: string + lockStart: + description: The vgchange commandline for lockStart, minus + the "vgchange" command + type: string + type: object vgCreate: description: VgCreate specifies the vgcreate commandline, minus the "vgcreate". type: string + vgRemove: + description: VgCreate specifies the vgcreate commandline, + minus the "vgremove". + type: string type: object type: object xfsStorage: @@ -282,6 +332,10 @@ spec: description: LvCreate specifies the lvcreate commandline, minus the "lvcreate". type: string + lvRemove: + description: LvRemove specifies the lvcreate commandline, + minus the "lvremove". + type: string mkfs: description: Mkfs specifies the mkfs commandline, minus the "mkfs". @@ -290,10 +344,31 @@ spec: description: PvCreate specifies the pvcreate commandline, minus the "pvcreate". type: string + vgChange: + description: VgChange specifies the various vgchange commandlines, + minus the "vgchange" + properties: + activate: + description: The vgchange commandline for activation, + minus the "vgchange" command + type: string + deactivate: + description: The vgchange commandline for deactivation, + minus the "vgchange" command + type: string + lockStart: + description: The vgchange commandline for lockStart, minus + the "vgchange" command + type: string + type: object vgCreate: description: VgCreate specifies the vgcreate commandline, minus the "vgcreate". type: string + vgRemove: + description: VgCreate specifies the vgcreate commandline, + minus the "vgremove". + type: string type: object options: description: Options contains options for libraries. diff --git a/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml b/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml index 93a1d019c..8c33a7a8b 100644 --- a/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml +++ b/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml @@ -22,13 +22,24 @@ data: commandlines: pvCreate: $DEVICE vgCreate: $VG_NAME $DEVICE_LIST + vgChange: + activate: --activate ys $VG_NAME + deactivate: --activate n $VG_NAME + lockstart: --lock-start $VG_NAME + vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME mkfs: -j2 -p $PROTOCOL -t $CLUSTER_NAME:$LOCK_SPACE $DEVICE xfsStorage: commandlines: pvCreate: $DEVICE vgCreate: $VG_NAME $DEVICE_LIST + vgChange: + activate: --activate y $VG_NAME + deactivate: --activate n $VG_NAME + vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME mkfs: $DEVICE options: mountRabbit: @@ -38,5 +49,10 @@ data: commandlines: pvCreate: $DEVICE vgCreate: $VG_NAME $DEVICE_LIST + vgChange: + activate: --activate y $VG_NAME + deactivate: --activate n $VG_NAME + vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME diff --git a/config/samples/placeholder_nnfstorageprofile.yaml b/config/samples/placeholder_nnfstorageprofile.yaml index ce76193d3..552cb7136 100644 --- a/config/samples/placeholder_nnfstorageprofile.yaml +++ b/config/samples/placeholder_nnfstorageprofile.yaml @@ -23,18 +23,34 @@ data: commandlines: pvCreate: $DEVICE vgCreate: $VG_NAME $DEVICE_LIST + vgChange: + activate: --activate ys $VG_NAME + deactivate: --activate n $VG_NAME + lockstart: --lock-start $VG_NAME + vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME mkfs: -j2 -p $PROTOCOL -t $CLUSTER_NAME:$LOCK_SPACE $DEVICE xfsStorage: commandlines: pvCreate: $DEVICE vgCreate: $VG_NAME $DEVICE_LIST + vgChange: + activate: --activate y $VG_NAME + deactivate: --activate n $VG_NAME + vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME mkfs: $DEVICE rawStorage: commandlines: pvCreate: $DEVICE vgCreate: $VG_NAME $DEVICE_LIST + vgChange: + activate: --activate y $VG_NAME + deactivate: --activate n $VG_NAME + vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvRemove: $VG_NAME diff --git a/controllers/nnf_node_storage_controller.go b/controllers/nnf_node_storage_controller.go index 147b3eb4d..a92665c36 100644 --- a/controllers/nnf_node_storage_controller.go +++ b/controllers/nnf_node_storage_controller.go @@ -482,7 +482,14 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto oem.MkfsMount.Mkfs = c.Mkfs oem.LvmCmd.PvCreate = c.PvCreate oem.LvmCmd.VgCreate = c.VgCreate + oem.LvmCmd.VgChange = nnfserver.FileSystemOemVgChange{ + Activate: c.VgChange.Activate, + Deactivate: c.VgChange.Deactivate, + LockStart: c.VgChange.LockStart, + } + oem.LvmCmd.VgRemove = c.VgRemove oem.LvmCmd.LvCreate = c.LvCreate + oem.LvmCmd.LvRemove = c.LvRemove } setOpts := func(c *nnfv1alpha1.NnfStorageProfileMiscOptions) { From 951a2179dc421bf2d223e415bde44203ffee4eb0 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Mon, 30 Jan 2023 18:02:19 -0600 Subject: [PATCH 20/71] Fix for 'lockStart' typo Signed-off-by: Nate Thornton --- config/samples/nnf_v1alpha1_nnfstorageprofile.yaml | 2 +- config/samples/placeholder_nnfstorageprofile.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml b/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml index 8c33a7a8b..c4ea3603d 100644 --- a/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml +++ b/config/samples/nnf_v1alpha1_nnfstorageprofile.yaml @@ -25,7 +25,7 @@ data: vgChange: activate: --activate ys $VG_NAME deactivate: --activate n $VG_NAME - lockstart: --lock-start $VG_NAME + lockStart: --lock-start $VG_NAME vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME lvRemove: $VG_NAME diff --git a/config/samples/placeholder_nnfstorageprofile.yaml b/config/samples/placeholder_nnfstorageprofile.yaml index 552cb7136..20b6e248a 100644 --- a/config/samples/placeholder_nnfstorageprofile.yaml +++ b/config/samples/placeholder_nnfstorageprofile.yaml @@ -26,7 +26,7 @@ data: vgChange: activate: --activate ys $VG_NAME deactivate: --activate n $VG_NAME - lockstart: --lock-start $VG_NAME + lockStart: --lock-start $VG_NAME vgRemove: $VG_NAME lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME lvRemove: $VG_NAME From ed81994642edccdf2ac8358c9e08223dee53a36d Mon Sep 17 00:00:00 2001 From: Blake Devcich <89158881+bdevcich-hpe@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:03:09 -0600 Subject: [PATCH 21/71] Added PR builds for feature branches (#138) Signed-off-by: Blake Devcich --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 90c2c21ec..4fbd6ec33 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,6 +10,7 @@ on: branches: - 'master' - 'releases/v*' + - 'feature/*' env: # TEST_TARGET: Name of the testing target in the Dockerfile From 0b66626501205edee4565444a30987976d15931c Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:21:53 -0600 Subject: [PATCH 22/71] RABSW-1128: Make fake mounts on kind Rabbit nodes (#139) * RABSW-1128: Make fake mounts on kind Rabbit nodes Create empty directories on the Rabbit nodes in the clientmount reconciler to better fake out data movement and user containers. Signed-off-by: Matt Richerson * review comments Signed-off-by: Matt Richerson --------- Signed-off-by: Matt Richerson --- config/kind/kustomization.yaml | 2 ++ config/kind/manager_volumes_patch.yaml | 19 +++++++++++++++ controllers/nnf_access_controller.go | 13 ++++++++++ controllers/nnf_clientmount_controller.go | 24 +++++++++++++++++++ .../nnf_workflow_controller_helpers.go | 6 +++-- controllers/nnf_workflow_controller_test.go | 4 ++-- main.go | 14 +++++------ 7 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 config/kind/manager_volumes_patch.yaml diff --git a/config/kind/kustomization.yaml b/config/kind/kustomization.yaml index 3305486de..077967bd0 100644 --- a/config/kind/kustomization.yaml +++ b/config/kind/kustomization.yaml @@ -2,5 +2,7 @@ bases: - ../top patchesStrategicMerge: +# Let the node-manager daemonset mount host dir for /mnt +- manager_volumes_patch.yaml # Arguments for the controller manager that are specific to kind - manager_environment_patch.yaml diff --git a/config/kind/manager_volumes_patch.yaml b/config/kind/manager_volumes_patch.yaml new file mode 100644 index 000000000..30cb1001f --- /dev/null +++ b/config/kind/manager_volumes_patch.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /mnt + name: mnt-dir + mountPropagation: Bidirectional + volumes: + - name: mnt-dir + hostPath: + path: /mnt + diff --git a/controllers/nnf_access_controller.go b/controllers/nnf_access_controller.go index 8ebe5ea21..bf70ffea5 100644 --- a/controllers/nnf_access_controller.go +++ b/controllers/nnf_access_controller.go @@ -22,6 +22,7 @@ package controllers import ( "context" "fmt" + "os" "path/filepath" "reflect" "runtime" @@ -475,6 +476,12 @@ func (r *NnfAccessReconciler) mapClientNetworkStorage(ctx context.Context, acces ObjectReference: access.Spec.StorageReference, } + if os.Getenv("ENVIRONMENT") == "kind" { + mountInfo.UserID = access.Spec.UserID + mountInfo.GroupID = access.Spec.GroupID + mountInfo.SetPermissions = true + } + storageMapping[client] = append(storageMapping[client], mountInfo) } @@ -561,6 +568,12 @@ func (r *NnfAccessReconciler) mapClientLocalStorage(ctx context.Context, access mountInfo.Type = nnfStorage.Spec.FileSystemType } + if os.Getenv("ENVIRONMENT") == "kind" { + mountInfo.UserID = access.Spec.UserID + mountInfo.GroupID = access.Spec.GroupID + mountInfo.SetPermissions = true + } + // If no ClientReference exists, then the mounts are for the Rabbit nodes. Use references // to the NnfNodeStorage resource so the client mounter can access the swordfish objects if access.Spec.ClientReference == (corev1.ObjectReference{}) { diff --git a/controllers/nnf_clientmount_controller.go b/controllers/nnf_clientmount_controller.go index 9bc2c8ea9..bf3b9a298 100644 --- a/controllers/nnf_clientmount_controller.go +++ b/controllers/nnf_clientmount_controller.go @@ -195,6 +195,30 @@ func (r *NnfClientMountReconciler) changeMountAll(ctx context.Context, clientMou // changeMount mount or unmounts a single mount point described in the ClientMountInfo object func (r *NnfClientMountReconciler) changeMount(ctx context.Context, clientMountInfo dwsv1alpha1.ClientMountInfo, shouldMount bool, log logr.Logger) error { + if os.Getenv("ENVIRONMENT") == "kind" { + if shouldMount { + if err := os.MkdirAll(clientMountInfo.MountPath, 0755); err != nil { + return dwsv1alpha1.NewResourceError(fmt.Sprintf("Make directory failed: %s", clientMountInfo.MountPath), err) + } + + log.Info("Fake mounted file system", "Mount path", clientMountInfo.MountPath) + } else { + if err := os.Remove(clientMountInfo.MountPath); err != nil { + return dwsv1alpha1.NewResourceError(fmt.Sprintf("Remove directory failed: %s", clientMountInfo.MountPath), err) + } + + log.Info("Fake unmounted file system", "Mount path", clientMountInfo.MountPath) + } + + if clientMountInfo.SetPermissions { + if err := os.Chown(clientMountInfo.MountPath, int(clientMountInfo.UserID), int(clientMountInfo.GroupID)); err != nil { + return dwsv1alpha1.NewResourceError(fmt.Sprintf("Chown failed: %s", clientMountInfo.MountPath), err) + } + } + + return nil + } + switch clientMountInfo.Device.Type { case dwsv1alpha1.ClientMountDeviceTypeLustre: mountPath := clientMountInfo.MountPath diff --git a/controllers/nnf_workflow_controller_helpers.go b/controllers/nnf_workflow_controller_helpers.go index db5a68e6d..978e61d26 100644 --- a/controllers/nnf_workflow_controller_helpers.go +++ b/controllers/nnf_workflow_controller_helpers.go @@ -499,8 +499,10 @@ func (r *NnfWorkflowReconciler) setupNnfAccessForServers(ctx context.Context, st DesiredState: "mounted", TeardownState: teardownState, Target: "all", - MountPath: buildMountPath(workflow, index), - MountPathPrefix: buildMountPath(workflow, index), + UserID: workflow.Spec.UserID, + GroupID: workflow.Spec.GroupID, + MountPath: buildMountPath(workflow, parentDwIndex), + MountPathPrefix: buildMountPath(workflow, parentDwIndex), // NNF Storage is Namespaced Name to the servers object StorageReference: corev1.ObjectReference{ diff --git a/controllers/nnf_workflow_controller_test.go b/controllers/nnf_workflow_controller_test.go index 3bb164c7f..3934b5e8a 100644 --- a/controllers/nnf_workflow_controller_test.go +++ b/controllers/nnf_workflow_controller_test.go @@ -393,7 +393,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { "Namespace": Equal(lustre.Namespace), })) - Expect(dm.Spec.Destination.Path).To(Equal(buildMountPath(workflow, 1) + "/my-file.out")) + Expect(dm.Spec.Destination.Path).To(Equal(buildMountPath(workflow, 0) + "/my-file.out")) Expect(dm.Spec.Destination.StorageReference).ToNot(BeNil()) Expect(dm.Spec.Destination.StorageReference).To(MatchFields(IgnoreExtras, Fields{ @@ -525,7 +525,7 @@ var _ = Describe("NNF Workflow Unit Tests", func() { "Namespace": Equal(lustre.Namespace), })) - Expect(dm.Spec.Destination.Path).To(Equal(buildMountPath(workflow, 1) + "/my-persistent-file.out")) + Expect(dm.Spec.Destination.Path).To(Equal(buildMountPath(workflow, 0) + "/my-persistent-file.out")) Expect(dm.Spec.Destination.StorageReference).ToNot(BeNil()) Expect(dm.Spec.Destination.StorageReference).To(MatchFields(IgnoreExtras, Fields{ diff --git a/main.go b/main.go index 6bc50d7ee..7889214c3 100644 --- a/main.go +++ b/main.go @@ -192,14 +192,12 @@ func (c *nodeLocalController) SetupReconcilers(mgr manager.Manager, opts *nnf.Op return err } - if os.Getenv("ENVIRONMENT") != "kind" { - if err := (&controllers.NnfClientMountReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NnfClientMount"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - return err - } + if err := (&controllers.NnfClientMountReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NnfClientMount"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + return err } return (&controllers.NnfNodeStorageReconciler{ From 44c3847176b96b2fb694e85e725bb11fcad1f1ae Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:01:59 -0600 Subject: [PATCH 23/71] Shorten LVM names (#141) * RABSW-1129: Shorten LVM names The LV and VG names were too long and caused an error during the lvcreate. This commit changes the VG name to use a truncated version of the file share ID which includes the workflow name/namespace, directive index, and allocation index. This string is combined with the UUID of the workflow. The LV name was changed to be "lv" for all logical volumes since there is only ever the single LV in each VG. Signed-off-by: Matt Richerson * review comments Signed-off-by: Matt Richerson --------- Signed-off-by: Matt Richerson --- controllers/nnf_node_storage_controller.go | 52 ++++++++++++++++++---- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/controllers/nnf_node_storage_controller.go b/controllers/nnf_node_storage_controller.go index a92665c36..bdfb2517b 100644 --- a/controllers/nnf_node_storage_controller.go +++ b/controllers/nnf_node_storage_controller.go @@ -560,12 +560,22 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto } shareOptions := make(map[string]interface{}) + var volumeGroupName, logicalVolumeName string if nodeStorage.Spec.FileSystemType == "lustre" { targetIndex := nodeStorage.Spec.LustreStorage.StartIndex + index mountPath = "/mnt/lustre/" + nodeStorage.Spec.LustreStorage.FileSystemName + "/" + nodeStorage.Spec.LustreStorage.TargetType + strconv.Itoa(targetIndex) } else { - shareOptions["volumeGroupName"] = volumeGroupName(fileShareID) - shareOptions["logicalVolumeName"] = logicalVolumeName(fileShareID) + volumeGroupName, logicalVolumeName, err = r.lvmNames(ctx, nodeStorage, fileShareID) + if err != nil { + updateError(condition, &allocationStatus.FileShare, err) + nodeStorage.Status.Error = dwsv1alpha1.NewResourceError("could not get VG/LV names", err).WithFatal() + log.Info(nodeStorage.Status.Error.Error()) + + return &ctrl.Result{RequeueAfter: time.Minute * 2}, nil + } + + shareOptions["volumeGroupName"] = volumeGroupName + shareOptions["logicalVolumeName"] = logicalVolumeName shareOptions["userID"] = int(nodeStorage.Spec.UserID) shareOptions["groupID"] = int(nodeStorage.Spec.GroupID) } @@ -597,8 +607,8 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto if len(allocationStatus.FileShare.ID) == 0 { log.Info("Created file share", "Id", sh.Id) allocationStatus.FileShare.ID = sh.Id - allocationStatus.VolumeGroup = volumeGroupName(fileShareID) - allocationStatus.LogicalVolume = logicalVolumeName(fileShareID) + allocationStatus.VolumeGroup = volumeGroupName + allocationStatus.LogicalVolume = logicalVolumeName condition.LastTransitionTime = metav1.Now() condition.Status = metav1.ConditionFalse condition.Reason = nnfv1alpha1.ConditionSuccess @@ -705,12 +715,36 @@ func (r *NnfNodeStorageReconciler) deleteStorage(nodeStorage *nnfv1alpha1.NnfNod return &ctrl.Result{}, nil } -func volumeGroupName(id string) string { - return fmt.Sprintf("%s_vg", id) -} +func (r *NnfNodeStorageReconciler) lvmNames(ctx context.Context, nodeStorage *nnfv1alpha1.NnfNodeStorage, id string) (string, string, error) { + labels := nodeStorage.GetLabels() + + workflowName, ok := labels[dwsv1alpha1.WorkflowNameLabel] + if !ok { + return "", "", fmt.Errorf("missing Workflow label on NnfNodeStorage") + } + + workflowNamespace, ok := labels[dwsv1alpha1.WorkflowNamespaceLabel] + if !ok { + return "", "", fmt.Errorf("missing Workflow label on NnfNodeStorage") + } + + workflow := &dwsv1alpha1.Workflow{ + ObjectMeta: metav1.ObjectMeta{ + Name: workflowName, + Namespace: workflowNamespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(workflow), workflow); err != nil { + return "", "", dwsv1alpha1.NewResourceError("could get workflow", err) + } + + // Truncate the id to 65 characters since LVM has a name length limit + maxIDLength := 65 + if len(id) > maxIDLength { + id = id[:maxIDLength] + } -func logicalVolumeName(id string) string { - return fmt.Sprintf("%s_lv", id) + return fmt.Sprintf("%s_%s", id, workflow.GetUID()), "lv", nil } func (r *NnfNodeStorageReconciler) isSpecComplete(nodeStorage *nnfv1alpha1.NnfNodeStorage) bool { From 2a36e993c233e6295112253f3010c0d28d6834ed Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Tue, 7 Feb 2023 09:42:34 -0600 Subject: [PATCH 24/71] =?UTF-8?q?Ensure=20the=20NNF=20Node=20resource=20fe?= =?UTF-8?q?ncing=20status=20is=20cleared=20prior=20updating=E2=80=A6=20(#1?= =?UTF-8?q?26)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Ensure the NNF Node resource fencing status is cleared prior to updating the Storage resource * Refactor to use DWS Storage Controller Signed-off-by: Nate Thornton --- config/rbac/role.yaml | 8 - ...ontroller.go => dws_servers_controller.go} | 0 controllers/dws_storage_controller.go | 287 ++++++++++++++++ controllers/integration_test.go | 21 +- controllers/nnf_node_controller.go | 23 ++ controllers/nnf_node_slc.go | 313 ------------------ controllers/nnf_node_slc_test.go | 77 ----- controllers/suite_test.go | 4 +- main.go | 16 +- 9 files changed, 333 insertions(+), 416 deletions(-) rename controllers/{nnf_servers_controller.go => dws_servers_controller.go} (100%) create mode 100644 controllers/dws_storage_controller.go delete mode 100644 controllers/nnf_node_slc.go delete mode 100644 controllers/nnf_node_slc_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0eeb1dfaf..f2427e224 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,14 +5,6 @@ metadata: creationTimestamp: null name: manager-role rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - apiGroups: - "" resources: diff --git a/controllers/nnf_servers_controller.go b/controllers/dws_servers_controller.go similarity index 100% rename from controllers/nnf_servers_controller.go rename to controllers/dws_servers_controller.go diff --git a/controllers/dws_storage_controller.go b/controllers/dws_storage_controller.go new file mode 100644 index 000000000..131f353a9 --- /dev/null +++ b/controllers/dws_storage_controller.go @@ -0,0 +1,287 @@ +/* + * Copyright 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" + "github.com/HewlettPackard/dws/utils/updater" + nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" +) + +const ( + finalizerStorage = "nnf.cray.hpe.com/storage" +) + +type DWSStorageReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=dws.cray.hpe.com,resources=storages,verbs=get;create;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=dws.cray.hpe.com,resources=storages/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodes,verbs=get;list;watch; +// +kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch + +// Reconcile is part of the main Kubernetes reconciliation loop which aims to +// move the current state of the Storage resource to the desired state. +func (r *DWSStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) { + log := r.Log.WithValues("resource", req.NamespacedName.String()) + + storage := &dwsv1alpha1.Storage{} + if err := r.Get(ctx, req.NamespacedName, storage); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Check if the object is being deleted + if !storage.GetDeletionTimestamp().IsZero() { + if !controllerutil.ContainsFinalizer(storage, finalizerStorage) { + return ctrl.Result{}, nil + } + + // TODO: Delete NNF Node resource? + + controllerutil.RemoveFinalizer(storage, finalizerStorage) + if err := r.Update(ctx, storage); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + + // Add a finalizer to ensure resource is properly deleted + if !controllerutil.ContainsFinalizer(storage, finalizerStorage) { + controllerutil.AddFinalizer(storage, finalizerStorage) + + if err := r.Update(ctx, storage); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{Requeue: true}, nil + } + + // Ensure the storage resource is updated with the latest NNF Node resource status + node := &nnfv1alpha1.NnfNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nnf-nlc", + Namespace: storage.GetName(), + }, + } + + if err := r.Get(ctx, client.ObjectKeyFromObject(node), node); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Now that it is confirmed an NNF Node resource exists for this Storage resource, + // ensure the proper labels are set on the resource + labels := storage.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + + const rabbitStorageType = "Rabbit" + if label, found := labels[dwsv1alpha1.StorageTypeLabel]; !found || label != rabbitStorageType { + labels[dwsv1alpha1.StorageTypeLabel] = rabbitStorageType + storage.SetLabels(labels) + + if err := r.Update(ctx, storage); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{Requeue: true}, nil + } + + // Create a new status + + statusUpdater := updater.NewStatusUpdater[*dwsv1alpha1.StorageStatus](storage) + defer func() { err = statusUpdater.CloseWithStatusUpdate(ctx, r, err) }() + + storage.Status.Type = dwsv1alpha1.NVMe + storage.Status.Capacity = node.Status.Capacity + storage.Status.Access.Protocol = dwsv1alpha1.PCIe + + if len(node.Status.Servers) == 0 { + return ctrl.Result{}, nil // Wait until severs array has been filed in with Rabbit info + } + + // Populate server status' - Server 0 is reserved as the Rabbit node. + storage.Status.Access.Servers = []dwsv1alpha1.Node{{ + Name: storage.Name, + Status: node.Status.Servers[0].Status.ConvertToDWSResourceStatus(), + }} + + // Populate compute status' + if len(node.Status.Servers) > 1 { + storage.Status.Access.Computes = make([]dwsv1alpha1.Node, 0) + for _, server := range node.Status.Servers[1:] /*Skip Rabbit*/ { + + // Servers that are unassigned in the system configuration will + // not have a hostname and should be skipped. + if len(server.Hostname) == 0 { + continue + } + + storage.Status.Access.Computes = append(storage.Status.Access.Computes, + dwsv1alpha1.Node{ + Name: server.Hostname, + Status: server.Status.ConvertToDWSResourceStatus(), + }) + } + } + + // Populate storage status' + storage.Status.Devices = make([]dwsv1alpha1.StorageDevice, len(node.Status.Drives)) + for idx, drive := range node.Status.Drives { + device := &storage.Status.Devices[idx] + + device.Slot = drive.Slot + device.Status = drive.Status.ConvertToDWSResourceStatus() + + if drive.Status == nnfv1alpha1.ResourceReady { + wearLevel := drive.WearLevel + device.Model = drive.Model + device.SerialNumber = drive.SerialNumber + device.FirmwareVersion = drive.FirmwareVersion + device.Capacity = drive.Capacity + device.WearLevel = &wearLevel + } else { + device.Model = "" + device.SerialNumber = "" + device.FirmwareVersion = "" + device.Capacity = 0 + device.WearLevel = nil + } + } + + // Handle any state transitions + switch storage.Spec.State { + case dwsv1alpha1.EnabledState: + + // Clear the fence status if the storage resource is enabled from a disabled state + if storage.Status.Status == dwsv1alpha1.DisabledStatus { + + log.WithValues("fenced", node.Status.Fenced).Info("resource disabled") + if node.Status.Fenced { + node.Status.Fenced = false + + if err := r.Status().Update(ctx, node); err != nil { + return ctrl.Result{}, err + } + + log.Info("fenced status cleared") + return ctrl.Result{}, nil + } + + // TODO: Fencing Agent Phase #2: Resume Rabbit NLC pods, wait for the pods to + // resume, then change Node Status to Enabled + + storage.Status.RebootRequired = false + storage.Status.Message = "" + } + + if node.Status.Fenced { + storage.Status.Status = dwsv1alpha1.DegradedStatus + storage.Status.RebootRequired = true + storage.Status.Message = "Storage node requires reboot to recover from STONITH event" + } else { + + ready, err := r.isKubernetesNodeReady(ctx, storage) + if err != nil { + return ctrl.Result{}, err + } + + if !ready { + log.Info("storage node is offline") + storage.Status.Status = dwsv1alpha1.OfflineStatus + storage.Status.Message = "Kubernetes node is offline" + } else { + storage.Status.Status = node.Status.Status.ConvertToDWSResourceStatus() + } + } + + case dwsv1alpha1.DisabledState: + // TODO: Fencing Agent Phase #2: Pause Rabbit NLC pods, wait for pods to be + // removed, then change Node Status to Disabled + + storage.Status.Status = dwsv1alpha1.DisabledStatus + storage.Status.Message = "Storage node manually disabled" + } + + return ctrl.Result{}, nil +} + +func (r *DWSStorageReconciler) isKubernetesNodeReady(ctx context.Context, storage *dwsv1alpha1.Storage) (bool, error) { + // Get the kubernetes node resource corresponding to the same node as the nnfNode resource. + // The kubelet has a heartbeat mechanism, so we can determine node failures from this resource. + node := &corev1.Node{} + if err := r.Get(ctx, types.NamespacedName{Name: storage.Name}, node); err != nil { + return false, err + } + + // Look through the node conditions to determine if the node is up. + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + + return false, nil +} + +func (r *DWSStorageReconciler) SetupWithManager(mgr ctrl.Manager) error { + + // Setup to watch the NNF Node resource + nnfNodeMapFunc := func(o client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: o.GetNamespace(), + Namespace: corev1.NamespaceDefault, + }}} + } + + // Setup to watch the Kubernetes Node resource + nodeMapFunc := func(o client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: o.GetName(), + }}} + } + + return ctrl.NewControllerManagedBy(mgr). + For(&dwsv1alpha1.Storage{}). + Watches(&source.Kind{Type: &nnfv1alpha1.NnfNode{}}, handler.EnqueueRequestsFromMapFunc(nnfNodeMapFunc)). + Watches(&source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(nodeMapFunc)). + Complete(r) +} diff --git a/controllers/integration_test.go b/controllers/integration_test.go index 8b235c46a..d6410f828 100644 --- a/controllers/integration_test.go +++ b/controllers/integration_test.go @@ -352,7 +352,7 @@ var _ = Describe("Integration Test", func() { Expect(k8sClient.Create(context.TODO(), node)).To(Succeed()) - // Create the NNF Node + // Create the NNF Node resource nnfNode := &nnfv1alpha1.NnfNode{ ObjectMeta: metav1.ObjectMeta{ Name: "nnf-nlc", @@ -367,19 +367,24 @@ var _ = Describe("Integration Test", func() { Expect(k8sClient.Create(context.TODO(), nnfNode)).To(Succeed()) - // Check that the DWS storage resource was updated with the compute node information - storage := &dwsv1alpha1.Storage{} - namespacedName := types.NamespacedName{ - Name: nodeName, - Namespace: corev1.NamespaceDefault, + // Create the DWS Storage resource + storage := &dwsv1alpha1.Storage{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Namespace: corev1.NamespaceDefault, + }, } + Expect(k8sClient.Create(context.TODO(), storage)).To(Succeed()) + + // Check that the DWS storage resource was updated with the compute node information + Eventually(func() error { - return k8sClient.Get(context.TODO(), namespacedName, storage) + return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(storage), storage) }).Should(Succeed()) Eventually(func() bool { - Expect(k8sClient.Get(context.TODO(), namespacedName, storage)).To(Succeed()) + Expect(k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(storage), storage)).To(Succeed()) return len(storage.Status.Access.Computes) == 16 }).Should(BeTrue()) diff --git a/controllers/nnf_node_controller.go b/controllers/nnf_node_controller.go index 7e67fd847..2f8713869 100644 --- a/controllers/nnf_node_controller.go +++ b/controllers/nnf_node_controller.go @@ -161,6 +161,29 @@ func (r *NnfNodeReconciler) Start(ctx context.Context) error { return err } } + + storage := &dwsv1alpha1.Storage{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Namespace, + Namespace: corev1.NamespaceDefault, + }, + } + + if err := r.Get(ctx, client.ObjectKeyFromObject(storage), storage); err != nil { + log := r.Log.WithValues("resource", client.ObjectKeyFromObject(storage)) + + if !errors.IsNotFound(err) { + log.Error(err, "get storage resource failed") + return err + } + + if err := r.Create(ctx, storage); err != nil { + log.Error(err, "create storage resource failed") + return err + } + + log.Info("created storage resource") + } } // Subscribe to the NNF Event Manager diff --git a/controllers/nnf_node_slc.go b/controllers/nnf_node_slc.go deleted file mode 100644 index 004bf341a..000000000 --- a/controllers/nnf_node_slc.go +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP - * Other additional copyright holders may be indicated within. - * - * The entirety of this work is licensed under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package controllers - -import ( - "context" - "runtime" - - "github.com/go-logr/logr" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" - "github.com/HewlettPackard/dws/utils/updater" - nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" -) - -// NnfNodeSLCReconciler reconciles a NnfNode object -type NnfNodeSLCReconciler struct { - client.Client - Log logr.Logger - Scheme *kruntime.Scheme -} - -const ( - // finalizerNnfNodeSLC defines the key used to prevent the system from deleting the - // resource until this reconciler has finished doing clean up - finalizerNnfNodeSLC = "nnf.cray.hpe.com/nnf-node-slc" -) - -//+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodes,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodes/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfnodes/finalizers,verbs=update -//+kubebuilder:rbac:groups=dws.cray.hpe.com,resources=storages,verbs=get;create;list;watch;update;patch;delete -//+kubebuilder:rbac:groups=dws.cray.hpe.com,resources=storages/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch -//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the NnfNodeSLC to the desired state. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.9.2/pkg/reconcile -func (r *NnfNodeSLCReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) { - log := r.Log.WithValues("NnfNodeSLC", req.NamespacedName) - - nnfNode := &nnfv1alpha1.NnfNode{} - if err := r.Get(ctx, req.NamespacedName, nnfNode); err != nil { - // ignore not-found errors, since they can't be fixed by an immediate - // requeue (we'll need to wait for a new notification), and we can get them - // on deleted requests. - return ctrl.Result{}, client.IgnoreNotFound(err) - } - - // Check if the object is being deleted - if !nnfNode.GetDeletionTimestamp().IsZero() { - if !controllerutil.ContainsFinalizer(nnfNode, finalizerNnfNodeSLC) { - return ctrl.Result{}, nil - } - - if err := r.deleteStorage(ctx, nnfNode); err != nil { - return ctrl.Result{}, err - } - - controllerutil.RemoveFinalizer(nnfNode, finalizerNnfNodeSLC) - if err := r.Update(ctx, nnfNode); err != nil { - if !apierrors.IsConflict(err) { - return ctrl.Result{}, err - } - - return ctrl.Result{Requeue: true}, nil - } - - return ctrl.Result{}, nil - } - - // Add a finalizer to ensure the DWS Storage resource created by this resource is properly deleted. - if !controllerutil.ContainsFinalizer(nnfNode, finalizerNnfNodeSLC) { - - controllerutil.AddFinalizer(nnfNode, finalizerNnfNodeSLC) - if err := r.Update(ctx, nnfNode); err != nil { - if !apierrors.IsConflict(err) { - return ctrl.Result{}, err - } - - return ctrl.Result{Requeue: true}, nil - } - - return ctrl.Result{}, nil - } - - // Create a DWS Storage resource based on the information from the NNFNode and Node resources - storage := &dwsv1alpha1.Storage{ - ObjectMeta: metav1.ObjectMeta{ - Name: nnfNode.Namespace, - Namespace: corev1.NamespaceDefault, - }, - } - - if err := r.Get(ctx, client.ObjectKeyFromObject(storage), storage); err != nil { - if !apierrors.IsNotFound(err) { - return ctrl.Result{}, err - } - - storage.Spec = dwsv1alpha1.StorageSpec{ - State: dwsv1alpha1.EnabledState, - } - - if err := r.Create(ctx, storage); err != nil { - return ctrl.Result{}, err - } - - log.Info("Created DWS Storage resource", "name", storage.Name) - - return ctrl.Result{Requeue: true}, nil - } - - // Create a pair of updaters for updating the NNF Node status and Storage status - nnfNodeStatusUpdater := updater.NewStatusUpdater[*nnfv1alpha1.NnfNodeStatus](nnfNode) - defer func() { err = nnfNodeStatusUpdater.CloseWithStatusUpdate(ctx, r, err) }() - - storageStatusUpdater := updater.NewStatusUpdater[*dwsv1alpha1.StorageStatus](storage) - defer func() { err = storageStatusUpdater.CloseWithStatusUpdate(ctx, r, err) }() - - // Populate the status of the storage resource. - storage.Status.Type = dwsv1alpha1.NVMe - storage.Status.Capacity = nnfNode.Status.Capacity - - switch storage.Spec.State { - case dwsv1alpha1.EnabledState: - // Clear the fenced status if the node is enabled from a disabled status - if storage.Status.Status == dwsv1alpha1.DisabledStatus { - nnfNode.Status.Fenced = false - - storage.Status.RebootRequired = false - storage.Status.Message = "" - - // TODO: Fencing Agent Phase #2: Resume Rabbit NLC pods, wait for the pods to - // resume, then change Node Status to Enabled - } - - if nnfNode.Status.Fenced { - storage.Status.Status = dwsv1alpha1.DegradedStatus - storage.Status.RebootRequired = true - storage.Status.Message = "Storage node requires reboot to recover from STONITH event" - } else { - - // Check if the kubernetes node itself is offline - ready, err := r.isKubernetesNodeReady(ctx, nnfNode) - if err != nil { - return ctrl.Result{}, err - } - - if !ready { - storage.Status.Status = dwsv1alpha1.OfflineStatus - storage.Status.Message = "Kubernetes node is not ready" - } else { - storage.Status.Status = nnfNode.Status.Status.ConvertToDWSResourceStatus() - } - } - case dwsv1alpha1.DisabledState: - // TODO: Fencing Agent Phase #2: Pause Rabbit NLC pods, wait for pods to be - // removed, then change Node Status to Disabled - - storage.Status.Status = dwsv1alpha1.DisabledStatus - storage.Status.Message = "Storage node was manually disabled" - default: - log.Info("Unhandled storage state", "state", storage.Spec.State) - } - - // Add a label for a storage type of Rabbit. Don't overwrite the - // existing labels map since there may be user applied labels. - labels := storage.GetLabels() - if labels == nil { - labels = make(map[string]string) - } - labels[dwsv1alpha1.StorageTypeLabel] = "Rabbit" - storage.SetLabels(labels) - - // Access information for how to get to the storage - storage.Status.Access.Protocol = dwsv1alpha1.PCIe - - // Wait until the servers array has been filled in with the Rabbit info - if len(nnfNode.Status.Servers) == 0 { - return ctrl.Result{}, nil - } - - storage.Status.Access.Servers = []dwsv1alpha1.Node{{ - Name: nnfNode.Namespace, // The Rabbit node is the name of the nnfNode namespace - Status: nnfNode.Status.Servers[0].Status.ConvertToDWSResourceStatus(), - }} - - computes := []dwsv1alpha1.Node{} - for _, c := range nnfNode.Status.Servers[1:] { - if c.Hostname == "" { - continue - } - - compute := dwsv1alpha1.Node{ - Name: c.Hostname, - Status: c.Status.ConvertToDWSResourceStatus(), - } - computes = append(computes, compute) - } - storage.Status.Access.Computes = computes - - devices := []dwsv1alpha1.StorageDevice{} - for _, d := range nnfNode.Status.Drives { - if d.Status == nnfv1alpha1.ResourceOffline { - continue - } - - wearLevel := d.WearLevel - device := dwsv1alpha1.StorageDevice{ - Model: d.Model, - SerialNumber: d.SerialNumber, - FirmwareVersion: d.FirmwareVersion, - Capacity: d.Capacity, - Status: d.Status.ConvertToDWSResourceStatus(), - WearLevel: &wearLevel, - Slot: d.Slot, - } - devices = append(devices, device) - } - storage.Status.Devices = devices - - return ctrl.Result{}, nil -} - -func (r *NnfNodeSLCReconciler) deleteStorage(ctx context.Context, nnfNode *nnfv1alpha1.NnfNode) error { - log := r.Log.WithValues("NnfNode", types.NamespacedName{Name: nnfNode.Name, Namespace: nnfNode.Namespace}) - - storage := &dwsv1alpha1.Storage{ - ObjectMeta: metav1.ObjectMeta{ - Name: nnfNode.Namespace, - Namespace: "default", - }, - } - - if err := r.Delete(ctx, storage); err != nil { - return client.IgnoreNotFound(err) - } - - log.Info("Deleted DWS Storage resource", "name", storage.Name) - - return nil -} - -func (r *NnfNodeSLCReconciler) isKubernetesNodeReady(ctx context.Context, nnfNode *nnfv1alpha1.NnfNode) (bool, error) { - // Get the kubernetes node resource corresponding to the same node as the nnfNode resource. - // The kubelet has a heartbeat mechanism, so we can determine node failures from this resource. - node := &corev1.Node{} - if err := r.Get(ctx, types.NamespacedName{Name: nnfNode.Namespace}, node); err != nil { - return false, err - } - - // Look through the node conditions to determine if the node is up. - for _, condition := range node.Status.Conditions { - if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { - return true, nil - } - } - - return false, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *NnfNodeSLCReconciler) SetupWithManager(mgr ctrl.Manager) error { - - maxReconciles := runtime.GOMAXPROCS(0) - - mapFunc := func(o client.Object) []reconcile.Request { - return []reconcile.Request{{NamespacedName: types.NamespacedName{ - Name: "nnf-nlc", - Namespace: o.GetName(), - }}} - } - - return ctrl.NewControllerManagedBy(mgr). - WithOptions(controller.Options{MaxConcurrentReconciles: maxReconciles}). - For(&nnfv1alpha1.NnfNode{}). - Watches(&source.Kind{Type: &dwsv1alpha1.Storage{}}, handler.EnqueueRequestsFromMapFunc(mapFunc)). - Watches(&source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(mapFunc)). - Complete(r) -} diff --git a/controllers/nnf_node_slc_test.go b/controllers/nnf_node_slc_test.go deleted file mode 100644 index 1168c98e1..000000000 --- a/controllers/nnf_node_slc_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package controllers - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/client" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" - nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" -) - -var _ = Describe("System Level Controller Test", func() { - It("Creates a Storage object for down node", func() { - - nodeName := "nnf-slc-test-node-0" - - By("Creating the Kubernetes Node") - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - Namespace: corev1.NamespaceDefault, - Labels: map[string]string{ - "cray.nnf.node": "true", - }, - }, - // By default, a node lacking any status block is not ready - } - - Expect(k8sClient.Create(context.TODO(), node)).To(Succeed()) - - By("Creating the Node's Namespace") - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - } - - Expect(k8sClient.Create(context.TODO(), namespace)).To(Succeed()) - - By("Creating the NNF Node") - nnfNode := &nnfv1alpha1.NnfNode{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nnf-nlc", - Namespace: nodeName, - }, - Spec: nnfv1alpha1.NnfNodeSpec{ - Name: nodeName, - State: "Enable", - }, - } - - Expect(k8sClient.Create(context.TODO(), nnfNode)).To(Succeed()) - - storage := &dwsv1alpha1.Storage{} - Eventually(func(g Gomega) dwsv1alpha1.ResourceStatus { - g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: nnfNode.Namespace, Namespace: corev1.NamespaceDefault}, storage)).To(Succeed()) - return storage.Status.Status - }).Should(Equal(dwsv1alpha1.OfflineStatus), "Create the DWS Storage object") - - By("Deleting the NNF Node") - Expect(k8sClient.Delete(context.TODO(), nnfNode)).To(Succeed()) - - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(storage), storage) - }).ShouldNot(Succeed(), "Deletes the DWS Storage object") - - // Test cleanup - Expect(k8sClient.Delete(context.TODO(), node)).To(Succeed()) - Expect(k8sClient.Delete(context.TODO(), namespace)).To(Succeed()) - }) -}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index b16ebe1b8..b38b13deb 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -246,9 +246,9 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) - err = (&NnfNodeSLCReconciler{ + err = (&DWSStorageReconciler{ Client: k8sManager.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NnfNodeSLC"), + Log: ctrl.Log.WithName("controllers").WithName("Storage"), Scheme: testEnv.Scheme, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/main.go b/main.go index 7889214c3..e87f60af8 100644 --- a/main.go +++ b/main.go @@ -222,14 +222,6 @@ func (c *storageController) SetupReconcilers(mgr manager.Manager, opts *nnf.Opti return err } - if err := (&controllers.NnfNodeSLCReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NnfNode"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - return err - } - if err := (&controllers.NnfWorkflowReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("NnfWorkflow"), @@ -254,6 +246,14 @@ func (c *storageController) SetupReconcilers(mgr manager.Manager, opts *nnf.Opti return err } + if err := (&controllers.DWSStorageReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Storage"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + return err + } + if err := (&controllers.DWSServersReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("Servers"), From 7082d393bd7f0d185ca871f242ce1c28769c7257 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Tue, 7 Feb 2023 10:51:13 -0600 Subject: [PATCH 25/71] Remove finalizer on the new DWS Storage Controller (#144) * Remove finalizer on the new DWS Storage Controller Signed-off-by: Nate Thornton --- controllers/dws_storage_controller.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/controllers/dws_storage_controller.go b/controllers/dws_storage_controller.go index 131f353a9..377c139cd 100644 --- a/controllers/dws_storage_controller.go +++ b/controllers/dws_storage_controller.go @@ -31,7 +31,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -41,10 +40,6 @@ import ( nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" ) -const ( - finalizerStorage = "nnf.cray.hpe.com/storage" -) - type DWSStorageReconciler struct { client.Client Log logr.Logger @@ -69,31 +64,9 @@ func (r *DWSStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Check if the object is being deleted if !storage.GetDeletionTimestamp().IsZero() { - if !controllerutil.ContainsFinalizer(storage, finalizerStorage) { - return ctrl.Result{}, nil - } - - // TODO: Delete NNF Node resource? - - controllerutil.RemoveFinalizer(storage, finalizerStorage) - if err := r.Update(ctx, storage); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil } - // Add a finalizer to ensure resource is properly deleted - if !controllerutil.ContainsFinalizer(storage, finalizerStorage) { - controllerutil.AddFinalizer(storage, finalizerStorage) - - if err := r.Update(ctx, storage); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{Requeue: true}, nil - } - // Ensure the storage resource is updated with the latest NNF Node resource status node := &nnfv1alpha1.NnfNode{ ObjectMeta: metav1.ObjectMeta{ From eeac41be3393b64fca62e37f74c56130a607bf48 Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:46:36 -0600 Subject: [PATCH 26/71] RABSW-1139: Fix ClientMount directory create/remove for kind environment (#145) * RABSW-1139: Fix ClientMount directory create/remove for kind environment In the ClientMount controller for kind nodes, check whether the directory exists before creating or removing it. Re-vendor dws Signed-off-by: Matt Richerson * MkdirAll() already handles when the directory exists. Don't check before hand with a Stat() call. Signed-off-by: Matt Richerson * re-vendor Signed-off-by: Matt Richerson --------- Signed-off-by: Matt Richerson --- controllers/nnf_clientmount_controller.go | 7 +- go.mod | 2 +- go.sum | 4 +- .../dws/api/v1alpha1/workflow_webhook.go | 32 +- .../dws/config/webhook/kustomization.yaml | 1 + .../dws/config/webhook/service.yaml | 3 +- .../dws/config/webhook/service_account.yaml | 18 ++ .../dws/controllers/clientmount_controller.go | 17 +- .../dws/utils/dwdparse/dwdparse.go | 281 +++++++++--------- vendor/modules.txt | 2 +- 10 files changed, 179 insertions(+), 188 deletions(-) create mode 100644 vendor/github.com/HewlettPackard/dws/config/webhook/service_account.yaml diff --git a/controllers/nnf_clientmount_controller.go b/controllers/nnf_clientmount_controller.go index bf3b9a298..c905ab175 100644 --- a/controllers/nnf_clientmount_controller.go +++ b/controllers/nnf_clientmount_controller.go @@ -203,7 +203,12 @@ func (r *NnfClientMountReconciler) changeMount(ctx context.Context, clientMountI log.Info("Fake mounted file system", "Mount path", clientMountInfo.MountPath) } else { - if err := os.Remove(clientMountInfo.MountPath); err != nil { + // Return if the directory was already removed + if _, err := os.Stat(clientMountInfo.MountPath); os.IsNotExist(err) { + return nil + } + + if err := os.RemoveAll(clientMountInfo.MountPath); err != nil { return dwsv1alpha1.NewResourceError(fmt.Sprintf("Remove directory failed: %s", clientMountInfo.MountPath), err) } diff --git a/go.mod b/go.mod index 5e7293e3a..b42364b1d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/NearNodeFlash/nnf-sos go 1.19 require ( - github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455 + github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index 73059e4bf..a9ff7424c 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455 h1:LIuI3Z7nSdm+hnNmJY/TLV//nloL6cqMlqWqclq9yNo= -github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 h1:847PiLepVQ10SoPQEc3OKdxWg6LXt70WXcCQj7ih3Bc= +github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= github.com/HewlettPackard/structex v1.0.2 h1:p2EH/p6zvUd5fSa0onudAUvLWdKsvQSRP0jGKeblA5E= github.com/HewlettPackard/structex v1.0.2/go.mod h1:3frC4RY/cPsP/4+N8rkxsNAGlQwHV+zDC7qvrN+N+rE= github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go index 1b7532b8d..69269e3a2 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/workflow_webhook.go @@ -217,38 +217,16 @@ func checkDirectives(workflow *Workflow, ruleParser RuleParser) error { return nil } - err := ruleParser.ReadRules() - if err != nil { + if err := ruleParser.ReadRules(); err != nil { return err } - uniqueMap := make(map[string]bool) - - for i, directive := range workflow.Spec.DWDirectives { - validDirective := false - for _, rule := range ruleParser.GetRuleList() { - // validate #DW syntax - const rejectUnsupportedCommands bool = true - - valid, err := dwdparse.ValidateDWDirective(rule, directive, uniqueMap, rejectUnsupportedCommands) - if err != nil { - // #DW parser validation failed - workflowlog.Info("dwDirective validation failed", "Error", err) - return err - } - - if valid { - validDirective = true - ruleParser.MatchedDirective(workflow, rule.WatchStates, i, rule.DriverLabel) - } - } - - if !validDirective { - return fmt.Errorf("invalid directive found: '%s'", directive) - } + // Forward the rule and directive index to the rule parsers matched directive handling + onValidDirectiveFunc := func(index int, rule dwdparse.DWDirectiveRuleSpec) { + ruleParser.MatchedDirective(workflow, rule.WatchStates, index, rule.DriverLabel) } - return nil + return dwdparse.Validate(ruleParser.GetRuleList(), workflow.Spec.DWDirectives, onValidDirectiveFunc) } // RuleParser defines the interface a rule parser must provide diff --git a/vendor/github.com/HewlettPackard/dws/config/webhook/kustomization.yaml b/vendor/github.com/HewlettPackard/dws/config/webhook/kustomization.yaml index 9cf26134e..d3adc8314 100644 --- a/vendor/github.com/HewlettPackard/dws/config/webhook/kustomization.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/webhook/kustomization.yaml @@ -1,6 +1,7 @@ resources: - manifests.yaml - service.yaml +- service_account.yaml configurations: - kustomizeconfig.yaml diff --git a/vendor/github.com/HewlettPackard/dws/config/webhook/service.yaml b/vendor/github.com/HewlettPackard/dws/config/webhook/service.yaml index 31e0f8295..c6892e79b 100644 --- a/vendor/github.com/HewlettPackard/dws/config/webhook/service.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/webhook/service.yaml @@ -9,4 +9,5 @@ spec: - port: 443 targetPort: 9443 selector: - control-plane: controller-manager + control-plane: webhook + type: LoadBalancer diff --git a/vendor/github.com/HewlettPackard/dws/config/webhook/service_account.yaml b/vendor/github.com/HewlettPackard/dws/config/webhook/service_account.yaml new file mode 100644 index 000000000..70ac2c1ec --- /dev/null +++ b/vendor/github.com/HewlettPackard/dws/config/webhook/service_account.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: webhook + namespace: system +--- +# As of Kubernetes 1.24, ServiceAccount tokens are no longer automatically +# generated. Instead, manually create the secret and the token key in the +# data field will be automatically set. +apiVersion: v1 +kind: Secret +metadata: + name: webhook + namespace: system + annotations: + kubernetes.io/service-account.name: webhook + kubernetes.io/service-account.namespace: system +type: kubernetes.io/service-account-token diff --git a/vendor/github.com/HewlettPackard/dws/controllers/clientmount_controller.go b/vendor/github.com/HewlettPackard/dws/controllers/clientmount_controller.go index 4a0fea457..139bbea3b 100644 --- a/vendor/github.com/HewlettPackard/dws/controllers/clientmount_controller.go +++ b/vendor/github.com/HewlettPackard/dws/controllers/clientmount_controller.go @@ -21,7 +21,6 @@ package controllers import ( "context" - "os" "strings" "github.com/go-logr/logr" @@ -116,20 +115,16 @@ func (r *ClientMountReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } -func filterByNonRabbitNamespacePrefixForTest() predicate.Predicate { +func filterByComputeNamespacePrefix() predicate.Predicate { return predicate.NewPredicateFuncs(func(object client.Object) bool { - return !strings.HasPrefix(object.GetNamespace(), "rabbit") + return strings.HasPrefix(object.GetNamespace(), "compute") }) } // SetupWithManager sets up the controller with the Manager. func (r *ClientMountReconciler) SetupWithManager(mgr ctrl.Manager) error { - builder := ctrl.NewControllerManagedBy(mgr). - For(&dwsv1alpha1.ClientMount{}) - - if _, found := os.LookupEnv("NNF_TEST_ENVIRONMENT"); found { - builder = builder.WithEventFilter(filterByNonRabbitNamespacePrefixForTest()) - } - - return builder.Complete(r) + return ctrl.NewControllerManagedBy(mgr). + For(&dwsv1alpha1.ClientMount{}). + WithEventFilter(filterByComputeNamespacePrefix()). + Complete(r) } diff --git a/vendor/github.com/HewlettPackard/dws/utils/dwdparse/dwdparse.go b/vendor/github.com/HewlettPackard/dws/utils/dwdparse/dwdparse.go index f0a27c350..8fd03bb7c 100644 --- a/vendor/github.com/HewlettPackard/dws/utils/dwdparse/dwdparse.go +++ b/vendor/github.com/HewlettPackard/dws/utils/dwdparse/dwdparse.go @@ -20,7 +20,6 @@ package dwdparse import ( - "errors" "fmt" "regexp" "strconv" @@ -59,174 +58,142 @@ type DWDirectiveRuleSpec struct { RuleDefs []DWDirectiveRuleDef `json:"ruleDefs"` } -type dwUnsupportedCommandErr struct { - command string -} - -// NewUnsupportedCommandErr returns a reference to the unsupported command type -func NewUnsupportedCommandErr(command string) error { - return &dwUnsupportedCommandErr{command} -} +// BuildArgsMap builds a map of the DWDirective's arguments in the form: args["key"] = value +func BuildArgsMap(dwd string) (map[string]string, error) { -func (e *dwUnsupportedCommandErr) Error() string { - return fmt.Sprintf("Unsupported Command: '%s'", e.command) -} + dwdArgs := strings.Fields(dwd) -// IsUnsupportedCommand returns true if the error indicates that the command -// is unsupported -func IsUnsupportedCommand(err error) bool { - if err == nil { - return false + if len(dwdArgs) == 0 || dwdArgs[0] != "#DW" { + return nil, fmt.Errorf("missing '#DW' prefix in directive '%s'", dwd) } - _, ok := err.(*dwUnsupportedCommandErr) - return ok -} - -// BuildRulesMap builds a map of the DWDirectives argument parser rules for the specified command -func BuildRulesMap(rule DWDirectiveRuleSpec, cmd string) (map[string]DWDirectiveRuleDef, error) { - rulesMap := make(map[string]DWDirectiveRuleDef) - for _, rd := range rule.RuleDefs { - rulesMap[rd.Key] = rd - } + argsMap := make(map[string]string) + argsMap["command"] = dwdArgs[1] + for i := 2; i < len(dwdArgs); i++ { + keyValue := strings.Split(dwdArgs[i], "=") + + // Don't allow repeated arguments + _, ok := argsMap[keyValue[0]] + if ok { + return nil, fmt.Errorf("repeated argument '%s' in directive '%s'", keyValue[0], dwd) + } - if len(rulesMap) == 0 { - return nil, NewUnsupportedCommandErr(cmd) + if len(keyValue) == 1 { + argsMap[keyValue[0]] = "true" + } else if len(keyValue) == 2 { + argsMap[keyValue[0]] = keyValue[1] + } else { + keyValue := strings.SplitN(dwdArgs[i], "=", 2) + argsMap[keyValue[0]] = keyValue[1] + } } - return rulesMap, nil + return argsMap, nil } -// BuildArgsMap builds a map of the DWDirective's arguments in the form: args["key"] = value -func BuildArgsMap(dwd string) (map[string]string, error) { - argsMap := make(map[string]string) - dwdArgs := strings.Fields(dwd) +// Compile this regex outside the loop for better performance. +var boolMatcher = regexp.MustCompile(`(?i)^(true|false)$`) // (?i) -> case-insensitve comparison - if len(dwdArgs) == 0 { - return nil, fmt.Errorf("Invalid format for directive '%s'", dwd) +// ValidateArgs validates a map of arguments against the rule specification +func ValidateArgs(spec DWDirectiveRuleSpec, args map[string]string, uniqueMap map[string]bool) error { + + command, found := args["command"] + if !found { + return fmt.Errorf("no command in arguments") } - if dwdArgs[0] == "#DW" { - argsMap["command"] = dwdArgs[1] - for i := 2; i < len(dwdArgs); i++ { - keyValue := strings.Split(dwdArgs[i], "=") + if command != spec.Command { + return fmt.Errorf("command '%s' does not match rule '%s'", command, spec.Command) + } - // Don't allow repeated arguments - _, ok := argsMap[keyValue[0]] - if ok { - return nil, errors.New("repeated argument in directive: " + keyValue[0]) + // Create a map that maps a directive rule definition to an argument that correctly matches it + // key: DWDirectiveRule value: argument that matches that rule + // Required to check that all DWDirectiveRuleDef's have been met + argToRuleMap := map[*DWDirectiveRuleDef]string{} + + findRuleDefinition := func(key string) (*DWDirectiveRuleDef, error) { + for index, rule := range spec.RuleDefs { + re, err := regexp.Compile(rule.Key) + if err != nil { + return nil, fmt.Errorf("invalid rule regular expression '%s'", rule.Key) } - if len(keyValue) == 1 { - argsMap[keyValue[0]] = "true" - } else if len(keyValue) == 2 { - argsMap[keyValue[0]] = keyValue[1] - } else { - keyValue := strings.SplitN(dwdArgs[i], "=", 2) - argsMap[keyValue[0]] = keyValue[1] + if re.MatchString(key) { + return &spec.RuleDefs[index], nil } } - } else { - return nil, errors.New("missing #DW in directive") + + return nil, fmt.Errorf("unsupported argument '%s'", key) } - return argsMap, nil -} -// ValidateArgs validates a map of arguments against the rules -// For cases where an unknown command may be allowed because there may be other handlers for that command -// -// failUnknownCommand = false -func ValidateArgs(args map[string]string, rule DWDirectiveRuleSpec, uniqueMap map[string]bool, failUnknownCommand bool) error { - command := args["command"] + // Iterate over all arguments and validate each based on the associated rule + for k, v := range args { + if k == "command" { + continue + } - // Determine the rules map for command - rulesMap, err := BuildRulesMap(rule, command) - if err != nil { - // If the command is unsupported and we are supposed to fail in that case return error. - // Otherwise just return nil to effectively skip the #DW - // for info on errors.As() below see: - // https://stackoverflow.com/questions/62441960/error-wrap-unwrap-type-checking-with-errors-is#62442136 - var unsupportedCommand *dwUnsupportedCommandErr - if failUnknownCommand && errors.As(err, &unsupportedCommand) { + rule, err := findRuleDefinition(k) + if err != nil { return err } - return nil - } - // Compile this regex outside the loop for better performance. - var boolMatcher = regexp.MustCompile(`(?i)^(true|false)$`) // (?i) -> case-insensitve comparison - - // Create a map that maps a directive rule definition to an argument that correctly matches it - // key: DWDirectiveRule value: argument that matches that rule - // Required to check that all DWDirectiveRuleDef's have been met - argToRuleMap := map[DWDirectiveRuleDef]string{} + if rule.IsValueRequired && len(v) == 0 { + return fmt.Errorf("argument '%s' requires value", k) + } - // Iterate over all arguments and validate each based on the associated rule - for k, v := range args { - if k != "command" { - rule, found := rulesMap[k] - if !found { - return errors.New("unsupported argument - " + k) + switch rule.Type { + case "integer": + i, err := strconv.Atoi(v) + if err != nil { + return fmt.Errorf("argument '%s' invalid integer '%s'", k, v) } - if rule.IsValueRequired && len(v) == 0 { - return errors.New("malformed keyword[=value]: " + k + "=" + v) + if rule.Max != 0 && i > rule.Max { + return fmt.Errorf("argument '%s' specified integer value %d is greather than maximum value %d", k, i, rule.Max) } - switch rule.Type { - case "integer": - // i,err := strconv.ParseInt(v, 10, 64) - i, err := strconv.Atoi(v) - if err != nil { - return errors.New("invalid integer argument: " + k + "=" + v) - } - if rule.Max != 0 && i > rule.Max { - return errors.New("specified integer exceeds maximum " + strconv.Itoa(rule.Max) + ": " + k + "=" + v) - } - if rule.Min != 0 && i < rule.Min { - return errors.New("specified integer smaller than minimum " + strconv.Itoa(rule.Min) + ": " + k + "=" + v) - } - case "bool": - if rule.Pattern != "" { - isok := boolMatcher.MatchString(v) - if !isok { - return errors.New("invalid bool argument: " + k + "=" + v) - } - } - case "string": - if rule.Pattern != "" { - isok, err := regexp.MatchString(rule.Pattern, v) - if !isok { - if err != nil { - return errors.New("invalid regexp in rule: " + rule.Pattern) - } - return errors.New("invalid argument: " + k + "=" + v) - } + if rule.Min != 0 && i < rule.Min { + return fmt.Errorf("argument '%s' specified integer value %d is less than minimum value %d", k, i, rule.Min) + } + case "bool": + if rule.Pattern != "" { + if !boolMatcher.MatchString(v) { + return fmt.Errorf("argument '%s' invalid boolean '%s'", k, v) } - default: - return errors.New("unsupported value type: " + rule.Type) } + case "string": + if rule.Pattern != "" { + re, err := regexp.Compile(rule.Pattern) + if err != nil { + return fmt.Errorf("invalid regular expression '%s'", rule.Pattern) + } - if rule.UniqueWithin != "" { - _, ok := uniqueMap[rule.UniqueWithin+"/"+v] - if ok { - return fmt.Errorf("Value '%s' must be unique within '%s'", v, rule.UniqueWithin) + if !re.MatchString(v) { + return fmt.Errorf("argument '%s' invalid string '%s'", k, v) } + } + default: + return fmt.Errorf("unsupported rule type '%s'", rule.Type) + } - uniqueMap[rule.UniqueWithin+"/"+v] = true + if rule.UniqueWithin != "" { + _, ok := uniqueMap[rule.UniqueWithin+"/"+v] + if ok { + return fmt.Errorf("value '%s' must be unique within '%s'", v, rule.UniqueWithin) } - // NOTE: We know that we don't have repeated arguments here because the arguments - // come to us in a map indexed by the argment name. - argToRuleMap[rule] = k + uniqueMap[rule.UniqueWithin+"/"+v] = true } + + // NOTE: We know that we don't have repeated arguments here because the arguments + // come to us in a map indexed by the argment name. + argToRuleMap[rule] = k } // Iterate over the rules to ensure all required rules have an argument - for k, v := range rulesMap { - // Ensure that each required rule has an argument - if v.IsRequired { - _, ok := argToRuleMap[v] - if !ok { - return errors.New("missing argument: " + k) + for index := range spec.RuleDefs { + rule := &spec.RuleDefs[index] + if rule.IsRequired { + if _, found := argToRuleMap[rule]; !found { + return fmt.Errorf("missing required argument '%v'", rule.Key) } } } @@ -234,8 +201,42 @@ func ValidateArgs(args map[string]string, rule DWDirectiveRuleSpec, uniqueMap ma return nil } -// ValidateDWDirective validates a set of #DW directives against a specified rule set -func ValidateDWDirective(rule DWDirectiveRuleSpec, dwd string, uniqueMap map[string]bool, failUnknownCommand bool) (bool, error) { +// Validate a list of directives against the supplied rules. When a directive is valid +// for a particular rule, the `onValidDirectiveFunc` function is called. +func Validate(rules []DWDirectiveRuleSpec, directives []string, onValidDirectiveFunc func(index int, rule DWDirectiveRuleSpec)) error { + + // Create a map to track argument uniqueness within the directives for + // rules that contain `UniqueWithin` + uniqueMap := make(map[string]bool) + + for index, directive := range directives { + + // A directive is validated against all rules; any one rule that is valid for a directive + // makes that directive valid. + validDirective := false + + for _, rule := range rules { + valid, err := validateDWDirective(rule, directive, uniqueMap) + if err != nil { + return err + } + + if valid { + validDirective = true + onValidDirectiveFunc(index, rule) + } + } + + if !validDirective { + return fmt.Errorf("invalid directive '%s'", directive) + } + } + + return nil +} + +// validateDWDirective validates an individual directive against a rule +func validateDWDirective(rule DWDirectiveRuleSpec, dwd string, uniqueMap map[string]bool) (bool, error) { // Build a map of the #DW commands and arguments argsMap, err := BuildArgsMap(dwd) @@ -243,19 +244,11 @@ func ValidateDWDirective(rule DWDirectiveRuleSpec, dwd string, uniqueMap map[str return false, err } - // If the command doesn't match... if argsMap["command"] != rule.Command { - // If we need to fail unknown commands, return invalid command - if failUnknownCommand { - return false, nil - } - - // Otherwise, we may have a new command that our code doesn't yet know - // Don't bother checking the rest - return true, nil + return false, nil } - err = ValidateArgs(argsMap, rule, uniqueMap, failUnknownCommand) + err = ValidateArgs(rule, argsMap, uniqueMap) if err != nil { return false, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 2c121306d..e502a4906 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.6.0 ## explicit; go 1.12 github.com/Azure/go-autorest/tracing -# github.com/HewlettPackard/dws v0.0.0-20230120203803-669e57232455 +# github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 ## explicit; go 1.19 github.com/HewlettPackard/dws/api/v1alpha1 github.com/HewlettPackard/dws/config/crd/bases From f019c6772aaf94cd920a7385b3348f62dbb42ad2 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Thu, 9 Feb 2023 08:42:57 -0600 Subject: [PATCH 27/71] Ensure 'key' values in the ruleset match against the exact string Signed-off-by: Nate Thornton --- config/dws/nnf-ruleset.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/config/dws/nnf-ruleset.yaml b/config/dws/nnf-ruleset.yaml index b19e56ad8..6146bdf7d 100644 --- a/config/dws/nnf-ruleset.yaml +++ b/config/dws/nnf-ruleset.yaml @@ -8,23 +8,23 @@ spec: - command: "jobdw" watchStates: Proposal,Setup,PreRun,PostRun,Teardown ruleDefs: - - key: "type" + - key: "^type$" type: "string" pattern: "^(raw|xfs|gfs2|lustre)$" isRequired: true isValueRequired: true - - key: "capacity" + - key: "^capacity$" type: "string" pattern: "^\\d+(KiB|KB|MiB|MB|GiB|GB|TiB|TB)$" isRequired: true isValueRequired: true - - key: "name" + - key: "^name$" type: "string" pattern: "^[A-Za-z][A-Za-z0-9-]+$" isRequired: true isValueRequired: true uniqueWithin: "jobdw_name" - - key: "profile" + - key: "^profile$" type: "string" pattern: "^[A-Za-z][A-Za-z0-9-]+$" isRequired: false @@ -32,23 +32,23 @@ spec: - command: "create_persistent" watchStates: Proposal,Setup,Teardown ruleDefs: - - key: "type" + - key: "^type$" type: "string" pattern: "^(raw|xfs|gfs2|lustre)$" isRequired: true isValueRequired: true - - key: "capacity" + - key: "^capacity$" type: "string" pattern: "^\\d+(KiB|KB|MiB|MB|GiB|GB|TiB|TB)$" isRequired: true isValueRequired: true - - key: "name" + - key: "^name$" type: "string" pattern: "^[A-Za-z][A-Za-z0-9-]+$" isRequired: true isValueRequired: true uniqueWithin: "create_persistent_name" - - key: "profile" + - key: "^profile$" type: "string" pattern: "^[A-Za-z][A-Za-z0-9-]+$" isRequired: false @@ -56,7 +56,7 @@ spec: - command: "destroy_persistent" watchStates: Proposal,Teardown ruleDefs: - - key: "name" + - key: "^name$" type: "string" pattern: "^[A-Za-z][A-Za-z0-9-]+$" isRequired: true @@ -65,7 +65,7 @@ spec: - command: "persistentdw" watchStates: Proposal,Setup,PreRun,PostRun,Teardown ruleDefs: - - key: "name" + - key: "^name$" type: "string" pattern: "^[A-Za-z][A-Za-z0-9-]+$" isRequired: true @@ -73,22 +73,22 @@ spec: - command: "copy_in" watchStates: Proposal,DataIn,Teardown ruleDefs: - - key: "source" + - key: "^source$" type: "string" isRequired: true isValueRequired: true - - key: "destination" + - key: "^destination$" type: "string" isRequired: true isValueRequired: true - command: "copy_out" watchStates: Proposal,DataOut,Teardown ruleDefs: - - key: "source" + - key: "^source$" type: "string" isRequired: true isValueRequired: true - - key: "destination" + - key: "^destination$" type: "string" isRequired: true isValueRequired: true From 44b3da2242afb0ad7facb9690bdbc45d0e6c01ce Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Fri, 10 Feb 2023 16:12:09 -0600 Subject: [PATCH 28/71] Handle Retryable EC errors for File Share (#133) * Handle Retryable EC errors Signed-off-by: Nate Thornton --- controllers/nnf_node_storage_controller.go | 46 ++++++++++++++-------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/controllers/nnf_node_storage_controller.go b/controllers/nnf_node_storage_controller.go index bdfb2517b..f4aec7e8c 100644 --- a/controllers/nnf_node_storage_controller.go +++ b/controllers/nnf_node_storage_controller.go @@ -22,6 +22,7 @@ package controllers import ( "context" "crypto/md5" + "errors" "fmt" "net/http" "os" @@ -248,10 +249,7 @@ func (r *NnfNodeStorageReconciler) allocateStorage(nodeStorage *nnfv1alpha1.NnfN sp, err := r.createStoragePool(ss, storagePoolID, nodeStorage.Spec.Capacity) if err != nil { updateError(condition, &allocationStatus.StoragePool, err) - nodeStorage.Status.Error = dwsv1alpha1.NewResourceError("Could not create StoragePool", err) - log.Info(nodeStorage.Status.Error.Error()) - - return &ctrl.Result{Requeue: true}, nil + return r.handleCreateError(nodeStorage, "could not create storage pool", err) } allocationStatus.StoragePool.Status = nnfv1alpha1.ResourceStatus(sp.Status) @@ -377,10 +375,7 @@ func (r *NnfNodeStorageReconciler) createBlockDevice(ctx context.Context, nodeSt sg, err := r.createStorageGroup(ss, storageGroupID, allocationStatus.StoragePool.ID, endpointID) if err != nil { updateError(condition, &allocationStatus.StorageGroup, err) - nodeStorage.Status.Error = dwsv1alpha1.NewResourceError("Could not create storage group", err).WithFatal() - log.Info(nodeStorage.Status.Error.Error()) - - return &ctrl.Result{Requeue: true}, nil + return r.handleCreateError(nodeStorage, "could not create storage group", err) } allocationStatus.StorageGroup.Status = nnfv1alpha1.ResourceStatus(sg.Status) @@ -388,7 +383,7 @@ func (r *NnfNodeStorageReconciler) createBlockDevice(ctx context.Context, nodeSt // If the SF ID is empty then we just created the resource. Save the ID in the NnfNodeStorage if len(allocationStatus.StorageGroup.ID) == 0 { - log.Info("Created storage group", "storageGroupID", storageGroupID) + log.Info("Created storage group", "Id", storageGroupID) allocationStatus.StorageGroup.ID = sg.Id condition.LastTransitionTime = metav1.Now() condition.Status = metav1.ConditionFalse // we are finished with this state @@ -523,10 +518,8 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto fs, err := r.createFileSystem(ss, fileSystemID, allocationStatus.StoragePool.ID, oem) if err != nil { updateError(condition, &allocationStatus.FileSystem, err) - nodeStorage.Status.Error = dwsv1alpha1.NewResourceError("Could not create file system", err).WithFatal() - log.Info(nodeStorage.Status.Error.Error()) - return &ctrl.Result{RequeueAfter: time.Minute * 2}, nil + return r.handleCreateError(nodeStorage, "could not create file system", err) } allocationStatus.FileSystem.Status = nnfv1alpha1.ResourceReady @@ -583,10 +576,7 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto sh, err = r.createFileShare(ss, fileShareID, allocationStatus.FileSystem.ID, os.Getenv("RABBIT_NODE"), mountPath, shareOptions) if err != nil { updateError(condition, &allocationStatus.FileShare, err) - nodeStorage.Status.Error = dwsv1alpha1.NewResourceError("Could not create file share", err).WithFatal() - log.Info(nodeStorage.Status.Error.Error()) - - return &ctrl.Result{RequeueAfter: time.Minute * 2}, nil + return r.handleCreateError(nodeStorage, "could not create file share", err) } nid := "" @@ -937,6 +927,30 @@ func (r *NnfNodeStorageReconciler) getFileSystem(ss nnf.StorageServiceApi, id st return fs, nil } +func (r *NnfNodeStorageReconciler) handleCreateError(storage *nnfv1alpha1.NnfNodeStorage, message string, err error) (*ctrl.Result, error) { + + resourceError := dwsv1alpha1.NewResourceError(message, err) + defer func() { + r.Log.WithValues("NnfNodeStorage", client.ObjectKeyFromObject(storage).String()).Info(resourceError.Error()) + storage.Status.Error = resourceError + }() + + controllerError := &ec.ControllerError{} + if errors.As(err, &controllerError) && controllerError.IsRetryable() { + return &ctrl.Result{RequeueAfter: controllerError.RetryDelay()}, nil + } + + resourceError = resourceError.WithFatal() + + // If this is really Fatal, we should not retry. But not all of nnf-ec supports the + // retryable classification of errors. Instead we mark the error as Fatal() but continue + // to retry with a modest delay. If the resource creation error occurs perpetually, an + // external entity should timeout the operation and therefore prevent future create attempts. + // Once nnf-ec has correctly classified all errors, there should be no need to requeue. + + return &ctrl.Result{RequeueAfter: time.Minute}, nil +} + func updateError(condition *metav1.Condition, status *nnfv1alpha1.NnfResourceStatus, err error) { status.Status = nnfv1alpha1.ResourceFailed condition.Reason = nnfv1alpha1.ConditionFailed From 7431fea354af68d2f23b72482302320c9d9e65b9 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Tue, 14 Feb 2023 14:28:50 -0600 Subject: [PATCH 29/71] Drive Slot information for drives which are offline (#152) * Upgrade nnf-ec to latest (ca4975) * Pull in drive Slot information from storage resource * update go.sum after running 'go mod tidy' --------- Signed-off-by: Nate Thornton --- controllers/nnf_node_controller.go | 2 +- go.mod | 2 +- go.sum | 4 +- .../internal/switchtec/pkg/nvme/nvme.go | 41 +++++++++++++++++++ .../nnf-ec/pkg/manager-fabric/config.go | 1 - .../pkg/manager-fabric/config_default.yaml | 19 +++++++++ .../nnf-ec/pkg/manager-fabric/manager.go | 21 ++++------ .../nnf-ec/pkg/manager-nvme/manager.go | 31 +++++++------- .../models/model_storage_v1_9_0_storage.go | 2 + vendor/modules.txt | 2 +- 10 files changed, 92 insertions(+), 33 deletions(-) diff --git a/controllers/nnf_node_controller.go b/controllers/nnf_node_controller.go index 2f8713869..d4abc8022 100644 --- a/controllers/nnf_node_controller.go +++ b/controllers/nnf_node_controller.go @@ -401,6 +401,7 @@ func updateDrives(node *nnfv1alpha1.NnfNode, log logr.Logger) error { return err } + drive.Slot = fmt.Sprintf("%d", storage.Location.PartLocation.LocationOrdinalValue) drive.NnfResourceStatus = nnfv1alpha1.NnfResourceStatus{ ID: storage.Id, Name: storage.Name, @@ -431,7 +432,6 @@ func updateDrives(node *nnfv1alpha1.NnfNode, log logr.Logger) error { drive.WearLevel = int64(storageController.NVMeControllerProperties.NVMeSMARTPercentageUsage) drive.SerialNumber = storageController.SerialNumber drive.FirmwareVersion = storageController.FirmwareVersion - drive.Slot = fmt.Sprintf("%s%d", storageController.Location.PartLocation.ServiceLabel, storageController.Location.PartLocation.LocationOrdinalValue) } // The Swordfish architecture places capacity information in a Storage device's Storage Pools. For our implementation, diff --git a/go.mod b/go.mod index b42364b1d..ff4975487 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089 + github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index a9ff7424c..b0b5685d7 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h1:upOQcpaEDmgLxoK7SZeqzGOQs8raxd9/paJYmBS/3/M= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089 h1:ZxXh6YNmnH50yHi0O5uOSNVG/7lI+pqDCB9wGMf9INU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c h1:4uTFUTm+upmdfkB1wROa+ApZqqA7s5yxI1yYAUYnIIE= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go index 35e836cc8..da64c61a8 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go @@ -362,6 +362,26 @@ func (dev *Device) IdentifyNamespaceControllerList(namespaceId uint32) (*CtrlLis return ctrlList, nil } +func (dev *Device) IdentifyPrimaryControllerCapabilities(controllerId uint16) (*CtrlCaps, error) { + + caps := new(CtrlCaps) + buf := structex.NewBuffer(caps) + + var cdw10 uint32 + cdw10 = uint32(controllerId) << 16 + cdw10 |= uint32(PrimaryControllerCapabilities_CNS) + + if err := dev.IdentifyRaw(0, cdw10, 0, buf.Bytes()); err != nil { + return nil, err + } + + if err := structex.Decode(buf, caps); err != nil { + return nil, err + } + + return caps, nil +} + // Identify - func (dev *Device) Identify(namespaceId uint32, cns IdentifyControllerOrNamespaceType, data []byte) error { return dev.IdentifyRaw(namespaceId, uint32(cns), 0, data) @@ -641,6 +661,27 @@ type CtrlList struct { Identifiers [2047]uint16 } +type CtrlCaps struct { + ControllerId uint16 // CNTLID + PortId uint16 // PORTID + ControllerResourceType uint8 // CRT + Reserved5 [27]uint8 + VQResourcesFlexibleTotal uint32 // VQFRT + VQResourcesFlexibleAssigned uint32 // VQRFA + VQResourcesFlexibleAllocatedToPrimary uint16 // VQRFAP + VQResourcesPrivateTotal uint16 // VQPRT + VQResourcesFlexibleSecondaryMaximum uint16 // VQFRSM + VQFlexibleResourcePreferredGranularity uint16 // VQGRAN + Reserved48 [16]uint8 + VIResourcesFlexibleTotal uint32 // VIFRT + VIResourcesFlexibleAssigned uint32 // VIRFA + VIResourcesFlexibleAllocatedToPrimary uint16 // VIRFAP + VIResourcesPrivateTotal uint16 // VIPRT + VIResourcesFlexibleSecondaryMaximum uint16 // VIFRSM + VIFlexibleResourcePreferredGranularity uint16 // VIGRAN + Reserved90 [4016]uint8 +} + // CreateNamespace creates a new namespace with the specified parameters func (dev *Device) CreateNamespace(size uint64, capacity uint64, format uint8, dps uint8, sharing uint8, anagrpid uint32, nvmsetid uint16, timeout uint32) (uint32, error) { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go index 6f878f054..220f4adb2 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go @@ -63,7 +63,6 @@ type SwitchConfig struct { Name string } PciGen int32 `yaml:"pciGen"` - SlotPrefix string `yaml:"slotPrefix"` Ports []PortConfig ManagementPortCount int diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml index c2ef23410..b68f1d258 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml @@ -11,6 +11,7 @@ switches: - name: Interswitch Fabric type: InterswitchPort port: 0 + slot: -1 width: 16 - name: Rabbit type: ManagementPort @@ -19,34 +20,42 @@ switches: - name: Compute 0 type: UpstreamPort port: 32 + slot: 0 width: 4 - name: Compute 1 type: UpstreamPort port: 34 + slot: 1 width: 4 - name: Compute 2 type: UpstreamPort port: 36 + slot: 2 width: 4 - name: Compute 3 type: UpstreamPort port: 38 + slot: 3 width: 4 - name: Compute 4 type: UpstreamPort port: 40 + slot: 4 width: 4 - name: Compute 5 type: UpstreamPort port: 42 + slot: 5 width: 4 - name: Compute 6 type: UpstreamPort port: 44 + slot: 6 width: 4 - name: Compute 7 type: UpstreamPort port: 46 + slot: 7 width: 4 - name: SSD 0 type: DownstreamPort @@ -101,42 +110,52 @@ switches: - name: Interswitch Fabric type: InterswitchPort port: 0 + slot: -1 width: 16 - name: Rabbit type: ManagementPort port: 24 + slot: -1 width: 16 - name: Compute 8 type: UpstreamPort port: 32 + slot: 8 width: 4 - name: Compute 9 type: UpstreamPort port: 34 + slot: 9 width: 4 - name: Compute 10 type: UpstreamPort port: 36 + slot: 10 width: 4 - name: Compute 11 type: UpstreamPort port: 38 + slot: 11 width: 4 - name: Compute 12 type: UpstreamPort port: 40 + slot: 12 width: 4 - name: Compute 13 type: UpstreamPort port: 42 + slot: 13 width: 4 - name: Compute 14 type: UpstreamPort port: 44 + slot: 14 width: 4 - name: Compute 15 type: UpstreamPort port: 46 + slot: 15 width: 4 - name: SSD 9 type: DownstreamPort diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go index 2664f21cd..342292c27 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go @@ -85,6 +85,8 @@ type Port struct { id string fabricId string + slot int64 + portType sf.PortV130PortType portStatus @@ -568,6 +570,7 @@ func (s *Switch) findPortByPhysicalPortId(id uint8) *Port { } func (p *Port) GetBaseEndpointIndex() int { return p.endpoints[0].index } +func (p *Port) GetSlot() int64 { return p.slot } func (p *Port) findEndpoint(functionId string) *Endpoint { id, err := strconv.Atoi(functionId) @@ -619,6 +622,10 @@ func (p *Port) Initialize() error { panic(fmt.Sprintf("No EP Functions received for port %+v", port)) } + if len(epPort.Ep.Functions) == 1 { + return fmt.Errorf("Port %s: Single Root I/O Virtualization (SR-IOV) not supported", port.id) + } + f := port.swtch.fabric if len(epPort.Ep.Functions) < 1 /*PF*/ +f.managementEndpointCount+f.upstreamEndpointCount { log.Warnf("Port %s: Insufficient function count %d", port.id, len(epPort.Ep.Functions)) @@ -898,6 +905,7 @@ func Initialize(ctrl SwitchtecControllerInterface) error { s.ports[portIdx] = Port{ id: strconv.Itoa(portIdx), fabricId: strconv.Itoa(fabricPortId), + slot: int64(portConf.Slot), portType: portType, portStatus: portStatus{ cfgLinkWidth: 0, @@ -1550,16 +1558,3 @@ func (f *Fabric) GetSwitchPort(switchId, portId string) (*Port, error) { return p, nil } - -func (f *Fabric) GetPortPartLocation(switchId, portId string) (*openapi.PartLocation, error) { - p, err := f.GetSwitchPort(switchId, portId) - if err != nil { - return nil, err - } - - return &openapi.PartLocation{ - ServiceLabel: p.swtch.config.SlotPrefix, - LocationOrdinalValue: int64(p.config.Slot), - LocationType: sf.SLOT_RV1100LT, - }, nil -} diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go index 7e4beb916..e4115e18a 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go @@ -78,6 +78,8 @@ type Storage struct { id string address string + slot int64 + serialNumber string modelNumber string firmwareRevision string @@ -574,8 +576,8 @@ func (v *Volume) Delete() error { return v.storage.deleteVolume(v.id) } func (v *Volume) AttachController(controllerId uint16) error { return v.attach(controllerId) } func (v *Volume) DetachController(controllerId uint16) error { return v.detach(controllerId) } -func (v *Volume) Format() error { - +func (v *Volume) Format() error { + // According to the NVM Express Base Specification 2.0b section 5.14 Format NVM Command // The scope of the format operation and the scope of the format with secure erase depend // on the attributes that the controller supports for the Format NVM command and the @@ -583,8 +585,8 @@ func (v *Volume) Format() error { // // So the namespace must be attached to a controller. We attach to the controller associated with // the physical function to avoid any noise generated by attaching the the volume to a working host. - - return v.runInAttachDetachBlock(func() error { return v.storage.formatVolume(v.id)}) + + return v.runInAttachDetachBlock(func() error { return v.storage.formatVolume(v.id) }) } func (v *Volume) SetFeature(data []byte) error { @@ -619,7 +621,7 @@ func (v *Volume) WaitFormatComplete() error { for ns.Utilization != 0 { const delay = 100 * time.Millisecond - + // Pause briefly for format to make progress time.Sleep(delay) @@ -789,6 +791,13 @@ func (m *Manager) EventHandler(e event.Event) error { storage.switchId = switchId storage.portId = portId + port, err := fabric.FabricController.GetSwitchPort(switchId, portId) + if err != nil { + return ec.NewErrInternalServerError().WithError(err).WithCause("downstream switch port not found") + } + + storage.slot = port.GetSlot() + if linkEstablished { return storage.LinkEstablishedEventHandler(switchId, portId) } @@ -980,6 +989,9 @@ func (mgr *Manager) StorageIdGet(storageId string, model *sf.StorageV190Storage) }, } + model.Location.PartLocation.LocationOrdinalValue = s.slot + model.Location.PartLocation.LocationType = sf.SLOT_RV1100LT + model.Controllers = s.OdataIdRef("/Controllers") model.StoragePools = s.OdataIdRef("/StoragePools") model.Volumes = s.OdataIdRef("/Volumes") @@ -1068,15 +1080,6 @@ func (mgr *Manager) StorageIdControllersControllerIdGet(storageId, controllerId model.FirmwareVersion = s.firmwareRevision model.Model = s.modelNumber - // Retrieve the slot label and value for this storage controller - location, err := fabric.FabricController.GetPortPartLocation(s.switchId, s.portId) - if err != nil { - return ec.NewErrNotFound().WithError(err).WithCause(fmt.Sprintf("Storage Controller failed to retrieve part location: Storage: %s", storageId)) - } - model.Location.PartLocation.ServiceLabel = location.ServiceLabel - model.Location.PartLocation.LocationOrdinalValue = location.LocationOrdinalValue - model.Location.PartLocation.LocationType = location.LocationType - model.Links.EndpointsodataCount = 1 model.Links.Endpoints = make([]sf.OdataV4IdRef, model.Links.EndpointsodataCount) model.Links.Endpoints[0].OdataId = endpointId diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models/model_storage_v1_9_0_storage.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models/model_storage_v1_9_0_storage.go index 6bb5916b3..f8600ecd7 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models/model_storage_v1_9_0_storage.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models/model_storage_v1_9_0_storage.go @@ -51,6 +51,8 @@ type StorageV190Storage struct { Links StorageV190Links `json:"Links,omitempty"` + Location ResourceLocation `json:"Location,omitempty"` + // The name of the resource or array member. Name string `json:"Name"` diff --git a/vendor/modules.txt b/vendor/modules.txt index e502a4906..af9c83841 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20230126164635-a87929e88089 +# github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From 7fe3715cac9d1da21ed756e08eb1068320564c9e Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Wed, 15 Feb 2023 09:02:21 -0600 Subject: [PATCH 30/71] add --ignore-not-found to uninstall Signed-off-by: Nate Thornton --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8a775740b..63bae4706 100644 --- a/Makefile +++ b/Makefile @@ -254,7 +254,7 @@ install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~ $(KUSTOMIZE) build config/crd | kubectl apply -f - uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl delete -f - + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found -f - deploy: VERSION ?= $(shell cat .version) deploy: .version kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. From 65f11930887e907ff9e17b67d199ef6be13c26c7 Mon Sep 17 00:00:00 2001 From: matthew-richerson <82597529+matthew-richerson@users.noreply.github.com> Date: Fri, 17 Feb 2023 08:52:48 -0600 Subject: [PATCH 31/71] github-151: Fix LVM issues with gfs2 (#157) * github-151: Fix LVM issues with gfs2 This commit fixes two issues that were affecting gfs2 file systems: - The dlm lock manager was failing to lock because the VG name was too long - The lvcreate command needs an "--activate ys" to active a shared volume Signed-off-by: Matt Richerson * use --extents Signed-off-by: Matt Richerson --------- Signed-off-by: Matt Richerson --- .../samples/placeholder_nnfstorageprofile.yaml | 6 +++--- controllers/nnf_node_storage_controller.go | 17 ++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/config/samples/placeholder_nnfstorageprofile.yaml b/config/samples/placeholder_nnfstorageprofile.yaml index 20b6e248a..882dc8aa0 100644 --- a/config/samples/placeholder_nnfstorageprofile.yaml +++ b/config/samples/placeholder_nnfstorageprofile.yaml @@ -28,7 +28,7 @@ data: deactivate: --activate n $VG_NAME lockStart: --lock-start $VG_NAME vgRemove: $VG_NAME - lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvCreate: --activate ys --extents 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME lvRemove: $VG_NAME mkfs: -j2 -p $PROTOCOL -t $CLUSTER_NAME:$LOCK_SPACE $DEVICE xfsStorage: @@ -39,7 +39,7 @@ data: activate: --activate y $VG_NAME deactivate: --activate n $VG_NAME vgRemove: $VG_NAME - lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvCreate: --extents 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME lvRemove: $VG_NAME mkfs: $DEVICE rawStorage: @@ -50,7 +50,7 @@ data: activate: --activate y $VG_NAME deactivate: --activate n $VG_NAME vgRemove: $VG_NAME - lvCreate: -l 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME + lvCreate: --extents 100%VG --stripes $DEVICE_NUM --stripesize=32KiB --name $LV_NAME $VG_NAME lvRemove: $VG_NAME diff --git a/controllers/nnf_node_storage_controller.go b/controllers/nnf_node_storage_controller.go index f4aec7e8c..08996da58 100644 --- a/controllers/nnf_node_storage_controller.go +++ b/controllers/nnf_node_storage_controller.go @@ -558,7 +558,7 @@ func (r *NnfNodeStorageReconciler) formatFileSystem(ctx context.Context, nodeSto targetIndex := nodeStorage.Spec.LustreStorage.StartIndex + index mountPath = "/mnt/lustre/" + nodeStorage.Spec.LustreStorage.FileSystemName + "/" + nodeStorage.Spec.LustreStorage.TargetType + strconv.Itoa(targetIndex) } else { - volumeGroupName, logicalVolumeName, err = r.lvmNames(ctx, nodeStorage, fileShareID) + volumeGroupName, logicalVolumeName, err = r.lvmNames(ctx, nodeStorage, index) if err != nil { updateError(condition, &allocationStatus.FileShare, err) nodeStorage.Status.Error = dwsv1alpha1.NewResourceError("could not get VG/LV names", err).WithFatal() @@ -705,7 +705,7 @@ func (r *NnfNodeStorageReconciler) deleteStorage(nodeStorage *nnfv1alpha1.NnfNod return &ctrl.Result{}, nil } -func (r *NnfNodeStorageReconciler) lvmNames(ctx context.Context, nodeStorage *nnfv1alpha1.NnfNodeStorage, id string) (string, string, error) { +func (r *NnfNodeStorageReconciler) lvmNames(ctx context.Context, nodeStorage *nnfv1alpha1.NnfNodeStorage, index int) (string, string, error) { labels := nodeStorage.GetLabels() workflowName, ok := labels[dwsv1alpha1.WorkflowNameLabel] @@ -718,6 +718,11 @@ func (r *NnfNodeStorageReconciler) lvmNames(ctx context.Context, nodeStorage *nn return "", "", fmt.Errorf("missing Workflow label on NnfNodeStorage") } + directiveIndex, ok := labels[nnfv1alpha1.DirectiveIndexLabel] + if !ok { + return "", "", fmt.Errorf("missing directive index label on NnfNodeStorage") + } + workflow := &dwsv1alpha1.Workflow{ ObjectMeta: metav1.ObjectMeta{ Name: workflowName, @@ -728,13 +733,7 @@ func (r *NnfNodeStorageReconciler) lvmNames(ctx context.Context, nodeStorage *nn return "", "", dwsv1alpha1.NewResourceError("could get workflow", err) } - // Truncate the id to 65 characters since LVM has a name length limit - maxIDLength := 65 - if len(id) > maxIDLength { - id = id[:maxIDLength] - } - - return fmt.Sprintf("%s_%s", id, workflow.GetUID()), "lv", nil + return fmt.Sprintf("%s_%s_%d", workflow.GetUID(), directiveIndex, index), "lv", nil } func (r *NnfNodeStorageReconciler) isSpecComplete(nodeStorage *nnfv1alpha1.NnfNodeStorage) bool { From e0d74544ec415f8d00c8cc318292405121739036 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Tue, 21 Feb 2023 12:12:54 -0600 Subject: [PATCH 32/71] Update nnf-ec to 1dce5b Signed-off-by: Nate Thornton --- go.mod | 2 +- go.sum | 4 +- .../internal/switchtec/pkg/switchtec/csr.go | 138 ++++++++++++++ .../internal/switchtec/pkg/switchtec/pci.go | 100 +++++++++++ .../nnf-ec/pkg/manager-fabric/manager.go | 19 ++ .../pkg/manager-fabric/switchtec_api.go | 1 + .../pkg/manager-fabric/switchtec_cli.go | 4 + .../pkg/manager-fabric/switchtec_device.go | 4 + .../pkg/manager-fabric/switchtec_mock.go | 4 + .../nnf-ec/pkg/manager-nnf/manager.go | 4 +- .../nnf-ec/pkg/manager-nvme/manager.go | 6 + .../nnf-ec/pkg/manager-server/server_local.go | 170 +++++++++--------- .../nnf-ec/pkg/manager-server/storage.go | 27 ++- vendor/modules.txt | 2 +- 14 files changed, 379 insertions(+), 106 deletions(-) create mode 100644 vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/csr.go create mode 100644 vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/pci.go diff --git a/go.mod b/go.mod index ff4975487..11aad4e11 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c + github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index b0b5685d7..cb4a73fdb 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h1:upOQcpaEDmgLxoK7SZeqzGOQs8raxd9/paJYmBS/3/M= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c h1:4uTFUTm+upmdfkB1wROa+ApZqqA7s5yxI1yYAUYnIIE= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f h1:Dr/JRcUA0N8ZKbUnXx/J6pHIPfVT7rrzFFFKyelRe8E= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/csr.go b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/csr.go new file mode 100644 index 000000000..d40283c7d --- /dev/null +++ b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/csr.go @@ -0,0 +1,138 @@ +/* + * Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package switchtec + +import ( + "encoding/binary" + "fmt" +) + +const ( + csrMaxReadWriteLengthInBytes = 4 + + csrReadSubCommand uint8 = 0 + csrWriteSubCommand uint8 = 1 +) + +func (dev *Device) CsrRead(pdfid uint16, addr uint16, length uint8) ([]byte, error) { + + if length > csrMaxReadWriteLengthInBytes { + return nil, fmt.Errorf("csr read of length %d exceeds limit %d", length, csrMaxReadWriteLengthInBytes) + } + + cmd := struct { + SubCmd uint8 + Reserved0 uint8 + Pdfid uint16 + Addr uint16 + Length uint8 + Reserved1 uint8 + }{ + SubCmd: csrReadSubCommand, + Pdfid: pdfid, + Addr: addr, + Length: length, + } + + rsp := struct { + Data [4]byte + }{} + + if err := dev.RunCommand(EpResourceAccessCommand, &cmd, &rsp); err != nil { + return nil, err + } + + + return rsp.Data[0:length], nil +} + +func (dev *Device) CsrRead8(pdfid uint16, addr uint16) (uint8, error) { + data, err := dev.CsrRead(pdfid, addr, 1) + if err != nil { + return 0, err + } + + return data[0], err +} + +func (dev *Device) CsrRead16(pdfid uint16, addr uint16) (uint16, error) { + data, err := dev.CsrRead(pdfid, addr, 2) + if err != nil { + return 0, err + } + + return binary.LittleEndian.Uint16(data), nil +} + +func (dev *Device) CsrRead32(pdfid uint16, addr uint16) (uint32, error) { + data, err := dev.CsrRead(pdfid, addr, 4) + if err != nil { + return 0, err + } + + return binary.LittleEndian.Uint32(data), nil +} + +func (dev *Device) CsrWrite(pdfid uint16, addr uint16, payload []byte) error { + + length := len(payload) + if length > csrMaxReadWriteLengthInBytes { + return fmt.Errorf("csr write of length %d exceeds limit %d", length, csrMaxReadWriteLengthInBytes) + } + + cmd := struct { + SubCmd uint8 + Reserved0 uint8 + Pdfid uint16 + Addr uint16 + Length uint8 + Reserved1 uint8 + Data [4]byte + }{ + SubCmd: csrWriteSubCommand, + Pdfid: pdfid, + Addr: addr, + Length: uint8(length), + } + + copy(cmd.Data[:], payload) + + return dev.RunCommand(EpResourceAccessCommand, &cmd, nil) +} + +func (dev *Device) CsrWrite8(pdfid uint16, addr uint16, data uint8) error { + payload := [1]byte{data} + + return dev.CsrWrite(pdfid, addr, payload[:]) +} + +func (dev *Device) CsrWrite16(pdfid uint16, addr uint16, data uint16) error { + payload := [2]byte{} + binary.LittleEndian.PutUint16(payload[:], data) + + return dev.CsrWrite(pdfid, addr, payload[:]) +} + +func (dev *Device) CsrWrite32(pdfid uint16, addr uint16, data uint32) error { + payload := [4]byte{} + binary.LittleEndian.PutUint32(payload[:], data) + + return dev.CsrWrite(pdfid, addr, payload[:]) +} diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/pci.go b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/pci.go new file mode 100644 index 000000000..f5f7cf505 --- /dev/null +++ b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/pci.go @@ -0,0 +1,100 @@ +/* + * Copyright 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package switchtec + +import "fmt" + +const ( + // PCIe Capabilities Registers - all definitions derived from + // the PCI Express Base Specification, Rev 4.0 Version 1.0 + + // 7.5.1.1.11 Capabilities Pointer (Offset 34h) - points to a linked + // list of capabilities implemented by this function. Bottom two bits + // are reserved and must be set to 00b. + pcieCapabilitiesPointer uint16 = 0x34 + pcieCapabilitiesPointerMask uint8 = 0xFC + + // 7.5.3.1 PCI Express Capability List Register (Offset 00h) + pcieNextCapabilityPointerOffset uint16 = 0x01 + + // 7.5.3.2 PCI Express Capabilities Register (Offset 02h) + pcieCapabilitiesVersionMask uint16 = 0xFC + + // 7.5.3 PCI Express Capability Structure - this structure is a mechanism + // for identifying and managaging PCI Express device. + pcieExpressCapabilitiesId uint8 = 0x10 + + // 7.5.3.4 Device Control Register (Offset 08h) + // PCI Express Base Specification, Rev 4.0 Version 1.0 + pcieDeviceControlRegisterOffset uint16 = 0x08 + pcieDeviceControlRegisterInitiateFunctionLevelResetFlag uint16 = 0x8000 +) + +func (dev *Device) VfReset(pdfid uint16) error { + + // Read the capabilities pointer + capabilityOffset, err := dev.CsrRead8(pdfid, pcieCapabilitiesPointer) + if err != nil { + return fmt.Errorf("vf reset - failed to read register address %#04x", pcieCapabilitiesPointer) + } + + // Bottom two bits are reserved and must be set to 00b. + capabilityOffset &= pcieCapabilitiesPointerMask + + // Loop through the PCI capabilities until the PCI Express Capabilities ID is found + offset := uint16(capabilityOffset) + for offset != 0 { + capId, err := dev.CsrRead8(pdfid, offset) + if err != nil { + return fmt.Errorf("vf reset - failed to read register address %#04x", offset) + } + + if capId == pcieExpressCapabilitiesId { + break + } + + offset += pcieNextCapabilityPointerOffset + nextOffset, err := dev.CsrRead8(pdfid, offset) + if err != nil { + return fmt.Errorf("vf reset - failed to read register address %#04x", offset) + } + + offset = uint16(nextOffset) + } + + if offset == 0 { + return fmt.Errorf("vf reset - cannot find capability register '%#02x'", pcieExpressCapabilitiesId) + } + + // We've located the start of the PCIe Express Capability structure. Read in + // the Device Control register and toggle the reset flag. + offset += pcieDeviceControlRegisterOffset + ctrl, err := dev.CsrRead16(pdfid, offset) + if err != nil { + return fmt.Errorf("vf reset - failed to read register %#04x", offset) + } + + ctrl |= pcieDeviceControlRegisterInitiateFunctionLevelResetFlag + if err := dev.CsrWrite16(pdfid, offset, ctrl); err != nil { + return fmt.Errorf("vf reset - failed to write register %#04x", offset) + } + + return nil +} diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go index 342292c27..99cc4b1c0 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go @@ -1558,3 +1558,22 @@ func (f *Fabric) GetSwitchPort(switchId, portId string) (*Port, error) { return p, nil } + +func (f *Fabric) ResetEndpoint(switchId, portId string, vfIndex int) error { + port, err := f.GetSwitchPort(switchId, portId) + if err != nil { + return err + } + + if port.portType != sf.DOWNSTREAM_PORT_PV130PT { + return fmt.Errorf("port type '%s' is not downstream port", port.portType) + } + + epIndex := vfIndex + 1 // endpoint index zero is reserved for the PF + if epIndex >= len(port.endpoints) { + return fmt.Errorf("virtual function index '%d' exceeds number of endpoints", vfIndex) + } + + ep := port.endpoints[epIndex] + return port.swtch.dev.ResetEndpoint(ep.pdfid) +} diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_api.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_api.go index aef5553ba..1ce1b8bc4 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_api.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_api.go @@ -49,6 +49,7 @@ type SwitchtecDeviceInterface interface { GetEvents() ([]switchtec.GfmsEvent, error) EnumerateEndpoint(uint8, func(epPort *switchtec.DumpEpPortDevice) error) error + ResetEndpoint(uint16) error Bind(uint8, uint8, uint16) error } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_cli.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_cli.go index 2f6b65beb..ae9b04e6a 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_cli.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_cli.go @@ -283,6 +283,10 @@ func (d *SwitchtecCliDevice) EnumerateEndpoint(physPortId uint8, handlerFunc fun }) } +func (d *SwitchtecCliDevice) ResetEndpoint(pdfid uint16) error { + panic("not yet implemented") +} + func (d *SwitchtecCliDevice) Bind(hostPhysPortId, hostLogPortId uint8, pdfid uint16) error { // Usage: switchtec fabric gfms-bind --host_sw_idx= --phys_port_id= --log_port_id= --pdfid= [OPTIONS] rsp, err := d.run(fmt.Sprintf("fabric gfms-bind %s --pax=%d --host_sw_idx=%d --phys_port_id=%d --log_port_id=%d --pdfid=%#04x", d.path, d.id, d.id, hostPhysPortId, hostLogPortId, pdfid)) diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_device.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_device.go index 5039df625..c03c42be5 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_device.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_device.go @@ -113,6 +113,10 @@ func (d *SwitchtecDevice) EnumerateEndpoint(id uint8, f func(epPort *switchtec.D return d.dev.GfmsEpPortDeviceEnumerate(id, f) } +func (d *SwitchtecDevice) ResetEndpoint(pdfid uint16) error { + return d.dev.VfReset(pdfid) +} + func (d *SwitchtecDevice) Bind(hostPhysPortId, hostLogPortId uint8, pdfid uint16) error { return d.dev.Bind(uint8(d.dev.ID()), hostPhysPortId, hostLogPortId, pdfid) } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_mock.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_mock.go index 72060e5b6..67b4886c2 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_mock.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/switchtec_mock.go @@ -275,6 +275,10 @@ func (d *MockSwitchtecDevice) EnumerateEndpoint(physPortId uint8, handlerFunc fu return nil } +func (d *MockSwitchtecDevice) ResetEndpoint(pdfid uint16) error { + return nil +} + func (d *MockSwitchtecDevice) Bind(hostPhysPortId, hostLogPortId uint8, pdfid uint16) error { bindPort := func(hostPort *MockSwitchtecPort, hostLogPortId uint8, pdfid uint16) error { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go index 62edf04e0..6661a3f16 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go @@ -203,8 +203,8 @@ func (s *StorageService) createStorageGroup(id string, sp *StoragePool, endpoint for idx, pv := range sp.providingVolumes { volume := pv.Storage.FindVolume(pv.VolumeId) expectedNamespaces[idx] = server.StorageNamespace{ - Id: volume.GetNamespaceId(), - Guid: volume.GetGloballyUniqueIdentifier(), + SerialNumber: pv.Storage.SerialNumber(), + Id: volume.GetNamespaceId(), } } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go index e4115e18a..d7b49767c 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go @@ -917,6 +917,12 @@ func (s *Storage) LinkEstablishedEventHandler(switchId, portId string) error { ctrl.viResources = uint16(s.config.Resources) } + + if sc.SecondaryControllerState&0x01 == 0 { + if err := fabric.FabricController.ResetEndpoint(switchId, portId, idx); err != nil { + log.WithError(err).Errorf("Secondary Controller %d: Failed to reset controller", sc.SecondaryControllerID) + } + } } if sc.SecondaryControllerState&0x01 == 0 { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/server_local.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/server_local.go index ff2c62488..e9cd37c43 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/server_local.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/server_local.go @@ -20,10 +20,9 @@ package server import ( - "fmt" + "encoding/json" "os" "os/exec" - "strconv" "strings" "syscall" @@ -45,7 +44,7 @@ func (DefaultServerControllerProvider) NewServerController(opts ServerController } type LocalServerController struct { - storage []*Storage + storages []*Storage } func (c *LocalServerController) Connected() bool { return true } @@ -76,14 +75,14 @@ func (c *LocalServerController) NewStorage(pid uuid.UUID, expectedNamespaces []S ctrl: c, } - c.storage = append(c.storage, storage) + c.storages = append(c.storages, storage) return storage } func (c *LocalServerController) Delete(s *Storage) error { - for storageIdx, storage := range c.storage { + for storageIdx, storage := range c.storages { if storage.Id == s.Id { - c.storage = append(c.storage[:storageIdx], c.storage[storageIdx+1:]...) + c.storages = append(c.storages[:storageIdx], c.storages[storageIdx+1:]...) break } } @@ -92,6 +91,7 @@ func (c *LocalServerController) Delete(s *Storage) error { } func (c *LocalServerController) GetStatus(s *Storage) (StorageStatus, error) { + if len(s.discoveredNamespaces) < len(s.expectedNamespaces) { if err := c.Discover(s); err != nil { return StorageStatus_Error, err @@ -138,13 +138,11 @@ func (c *LocalServerController) UnmountFileSystem(s *Storage, fs FileSystemApi, } func (c *LocalServerController) Discover(s *Storage) error { - var err error if devices, ok := os.LookupEnv("NNF_SUPPLIED_DEVICES"); ok { - err = c.discoverUsingSuppliedDevices(s, devices) - } else { - err = c.discoverViaNamespaces(s) + return c.discoverUsingSuppliedDevices(s, devices) } - return err + + return c.discoverViaNamespaces() } func (c *LocalServerController) discoverUsingSuppliedDevices(s *Storage, devices string) error { @@ -153,22 +151,9 @@ func (c *LocalServerController) discoverUsingSuppliedDevices(s *Storage, devices devList := strings.Split(devices, ",") for idx, devSupplied := range devList { log.Info("Supplied device ", " idx ", idx, " dev ", devSupplied) - byBytes := []byte(devSupplied) - // Use the last 8 bytes as the unique Id. - offset := len(byBytes) - 8 - if len(byBytes) < 8 { - offset = 0 - } - var myUuid uuid.UUID - copy(myUuid[:], byBytes[offset:]) - guid := nvme.NamespaceGloballyUniqueIdentifier{} - guid.Parse(myUuid.String()) - s.discoveredNamespaces = append(s.discoveredNamespaces, StorageNamespace{ - Id: nvme.NamespaceIdentifier(idx), - Guid: guid, - path: devSupplied, - debug: false, + Id: nvme.NamespaceIdentifier(idx), + path: devSupplied, }) } @@ -177,52 +162,63 @@ func (c *LocalServerController) discoverUsingSuppliedDevices(s *Storage, devices return nil } -func (c *LocalServerController) discoverViaNamespaces(s *Storage) error { - paths, err := c.namespaces() +// Discover NVMe Namespaces +func (c *LocalServerController) discoverViaNamespaces() error { + + data, err := c.run("nvme list --output-format=json") if err != nil { return err } - for _, path := range paths { - - sns, err := c.newStorageNamespace(path) - if err != nil { - return err - } + devices := nvmeListDevices{} + if err := json.Unmarshal(data, &devices); err != nil { + return err + } - discovered := false - for _, ns := range s.discoveredNamespaces { - if ns.Equals(sns) { - discovered = true - break - } - } + for _, storage := range c.storages { - if discovered { + // Skip storages that already discovered the expected namespaces + if len(storage.expectedNamespaces) == len(storage.discoveredNamespaces) { continue } - expected := false - for _, ns := range s.expectedNamespaces { - if ns.Equals(sns) { - expected = true - break + // For this storage resource, try to build the list of discovered namespaces + // from that which is expected. + for _, ns := range storage.expectedNamespaces { + for _, dev := range devices.Devices { + if dev.equals(ns) { + + discovered := false + for _, ns := range storage.discoveredNamespaces { + if dev.equals(ns) { + discovered = true + break + } + } + + if !discovered { + ns := StorageNamespace{ + SerialNumber: dev.SerialNumber, + Id: nvme.NamespaceIdentifier(dev.Namespace), + path: dev.DevicePath, + } + + storage.discoveredNamespaces = append(storage.discoveredNamespaces, ns) + } + + break + } } } - - if expected { - log.Infof("Discovered namespace %s", sns.String()) - s.discoveredNamespaces = append(s.discoveredNamespaces, *sns) - } } return nil } func (c *LocalServerController) findStorage(pid uuid.UUID) *Storage { - for idx, s := range c.storage { + for idx, s := range c.storages { if s.Id == pid { - return c.storage[idx] + return c.storages[idx] } } @@ -244,37 +240,45 @@ func (c *LocalServerController) namespaces() ([]string, error) { return strings.Fields(string(nss)), err } -func (c *LocalServerController) newStorageNamespace(path string) (*StorageNamespace, error) { - - // First we need to identify the NSID for the provided path. The output is something like - // "NVME Identify Namespace 1:" so we awk the 4th value, which gives us "1:", then trim - // the colon below. - nsidStr, err := c.run(fmt.Sprintf("nvme id-ns %s | awk '/NVME Identify Namespace/{printf $4}'", path)) - if err != nil { - return nil, err - } - - nsid, _ := strconv.Atoi(strings.TrimSuffix(string(nsidStr[:]), ":")) - - // Retrieve the namespace GUID. The output is something like "nguid : 00000000000000008ce38ee205e70401" - // so we awk the third parameter. - guidStr, err := c.run(fmt.Sprintf("nvme id-ns %s --namespace-id=%d | awk '/nguid/{printf $3}'", path, nsid)) - if err != nil { - return nil, err - } - - guid := nvme.NamespaceGloballyUniqueIdentifier{} - guid.Parse(string(guidStr)) - - return &StorageNamespace{ - Id: nvme.NamespaceIdentifier(nsid), - Guid: guid, - path: path, - }, nil -} - func (c *LocalServerController) run(cmd string) ([]byte, error) { return logging.Cli.Trace(cmd, func(cmd string) ([]byte, error) { return exec.Command("bash", "-c", cmd).Output() }) } + +// The NVMe List command, `nvme list --output-format=json`, will output a list of devices +// +// { +// "Devices" : [ +// { +// "NameSpace" : 1, +// "DevicePath" : "/dev/nvme0n1", +// "Firmware" : "1TCRS102", +// "Index" : 0, +// "ModelNumber" : "KIOXIA KCM7DRJE1T92", +// "ProductName" : "Non-Volatile memory controller: KIOXIA Corporation Device 0x0014", +// "SerialNumber" : "YCT0A08H03A1", +// "UsedBytes" : 528384, +// "MaximumLBA" : 15256, +// "PhysicalSize" : 62488576, +// "SectorSize" : 4096 +// }, +// ... +// ] +// } +// +// We use this data to match expected namespaces for a particular drive against what is +// found by the host to grab the Device Path which can be used to create a file system. +type nvmeListDevice struct { + Namespace uint32 `json:"Namespace"` + DevicePath string `json:"DevicePath"` + SerialNumber string `json:"SerialNumber"` +} + +type nvmeListDevices struct { + Devices []nvmeListDevice `json:"Devices"` +} + +func (dev *nvmeListDevice) equals(sn StorageNamespace) bool { + return dev.SerialNumber == sn.SerialNumber && dev.Namespace == uint32(sn.Id) +} diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/storage.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/storage.go index 59b7c7c40..4656217da 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/storage.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-server/storage.go @@ -20,8 +20,6 @@ package server import ( - "fmt" - "github.com/google/uuid" "github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme" @@ -51,16 +49,19 @@ type Storage struct { // Storage Namespace represents an NVMe Namespace present on the host. type StorageNamespace struct { - Id nvme.NamespaceIdentifier `json:"Id"` - Guid nvme.NamespaceGloballyUniqueIdentifier `json:"Guid"` + // The drive serial number + SerialNumber string `json:"SerialNumber"` + + // The NVMe Namespace Identifer + Id nvme.NamespaceIdentifier `json:"Id"` // Path refers to the system path for this particular NVMe Namespace. On unix - // variants, the path is of the form `/dev/nvme[CTRL]n[INDEX]` where CTRL is the - // parent NVMe Controller and INDEX is assigned by the operating system. INDEX - // does _not_ refer to the namespace ID (NSID) + // variants, the path is of the form `/dev/nvme[A]n[B]` where A is a number + // assigned by the OS for any particular NVMe controller, and B is a number + // assigned by the OS for any NVMe Namespace + // "A" does _not_ refer to the Controller ID (CNTL ID) + // "B" does _not_ refer to the Namespace ID (NSID) path string - - debug bool // If this storage namespace is in debug mode } func (s *Storage) GetStatus() (StorageStatus, error) { @@ -97,14 +98,6 @@ func (s *Storage) Devices() []string { return devices } -func (a StorageNamespace) String() string { - return fmt.Sprintf("NSID: %d GUID: %s", a.Id, a.Guid.String()) -} - -func (a StorageNamespace) Equals(b *StorageNamespace) bool { - return a.Id == b.Id && a.Guid == b.Guid -} - type StorageStatus string const ( diff --git a/vendor/modules.txt b/vendor/modules.txt index af9c83841..64d47ec8a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/HewlettPackard/structex ## explicit; go 1.19 github.com/NearNodeFlash/lustre-fs-operator/api/v1alpha1 github.com/NearNodeFlash/lustre-fs-operator/config/crd/bases -# github.com/NearNodeFlash/nnf-ec v0.0.0-20230213215749-ca497539590c +# github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f ## explicit; go 1.16 github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec From 59e3bc22638aeb1b4c7d3ed11befbc4db888da2a Mon Sep 17 00:00:00 2001 From: Blake Devcich <89158881+bdevcich-hpe@users.noreply.github.com> Date: Wed, 22 Feb 2023 09:56:13 -0600 Subject: [PATCH 33/71] Add container support to workflows (#159) Note: This is an experimental feature, but is working end-to-end (i.e. Proposal to Teardown) via the workflow process. This feature adds experimental support for NNF Containers in workflows. Containers workflows are created by using the `#DW container` directive. An `NnfContainerProfile` must be supplied to the directive to instruct the workflow on what containers to create and which volumes to mount inside of the container. Look at the sample container profile in the `config/samples` directory for more information. The `config/examples` directory are deployed with examples profiles on the system, but do not contain the full documentation. The computes resource must also be updated to instruct the workflow on where to place the container pods. The provided compute nodes will be traced back to their local rabbit node, which will be used as the targets for the pods. Containers are created during `PreRun` through the use of Kubernetes Jobs. Each rabbit node will be the target of one kubernetes Job, which will manage the successful completion of the container. `PreRun` will progress to `ready:true` when the pods have started successfully. Each container has volumes mounted inside of it that are defined by the container profile. The mount paths for these volumes are exposed to the container via environment variables that match the storage names provided by the container directive's arguments (e.g. DW_JOB_foo-local-storage). These storages can be considered optional or not. If not, and the storage argument isn't supplied to the directive, the workflow will fail in the `Proposal` state. Once the workflow has progressed to `PostRun`, the workflow will start to check if the pods have finished. Once finished, `PostRun` will progress to `ready:true` if all pods (i.e. k8s jobs) have completed successfully. If not, `PostRun` will remain in ready:false. Example container directive: ``` #DW jobdw name=my-gfs2 type=gfs2 capacity=50GB #DW persistentdw name=my-persistent #DW container name=my-container profile=example-randomly-fail DW_JOB_foo-local-storage=my-gfs2 DW_PERSISTENT_foo-persistent-storage=my-persistent ``` --------- Signed-off-by: Blake Devcich Signed-off-by: Nate Thornton Co-authored-by: Nate Thornton --- PROJECT | 8 + api/v1alpha1/nnfcontainerprofile_types.go | 86 + api/v1alpha1/workflow_helpers.go | 8 + api/v1alpha1/zz_generated.deepcopy.go | 96 +- ...nnf.cray.hpe.com_nnfcontainerprofiles.yaml | 7264 +++++++++++++++++ config/crd/kustomization.yaml | 3 + .../cainjection_in_nnfcontainerprofiles.yaml | 7 + .../webhook_in_nnfcontainerprofiles.yaml | 16 + config/dws/nnf-ruleset.yaml | 15 + config/examples/kustomization.yaml | 5 + .../nnf_v1alpha1_nnfcontainerprofiles.yaml | 59 + .../nnf_v1alpha1_nnfstorageprofile.yaml} | 0 config/manager/kustomization.yaml | 2 +- config/manager/manager.yaml | 2 + .../rbac/nnfcontainerprofile_editor_role.yaml | 24 + .../rbac/nnfcontainerprofile_viewer_role.yaml | 20 + config/rbac/role.yaml | 37 + .../nnf_v1alpha1_nnfcontainerprofile.yaml | 39 + controllers/directivebreakdown_controller.go | 2 +- controllers/nnf_workflow_controller.go | 165 +- .../nnf_workflow_controller_helpers.go | 649 +- deploy.sh | 9 +- hack/boilerplate.go.txt | 2 +- 23 files changed, 8440 insertions(+), 78 deletions(-) create mode 100644 api/v1alpha1/nnfcontainerprofile_types.go create mode 100644 config/crd/bases/nnf.cray.hpe.com_nnfcontainerprofiles.yaml create mode 100644 config/crd/patches/cainjection_in_nnfcontainerprofiles.yaml create mode 100644 config/crd/patches/webhook_in_nnfcontainerprofiles.yaml create mode 100644 config/examples/kustomization.yaml create mode 100644 config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml rename config/{samples/placeholder_nnfstorageprofile.yaml => examples/nnf_v1alpha1_nnfstorageprofile.yaml} (100%) create mode 100644 config/rbac/nnfcontainerprofile_editor_role.yaml create mode 100644 config/rbac/nnfcontainerprofile_viewer_role.yaml create mode 100644 config/samples/nnf_v1alpha1_nnfcontainerprofile.yaml diff --git a/PROJECT b/PROJECT index 252aa4130..693e8cc79 100644 --- a/PROJECT +++ b/PROJECT @@ -105,4 +105,12 @@ resources: kind: NnfNodeECData path: github.com/NearNodeFlash/nnf-sos/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + domain: cray.hpe.com + group: nnf + kind: NnfContainerProfile + path: github.com/NearNodeFlash/nnf-sos/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/nnfcontainerprofile_types.go b/api/v1alpha1/nnfcontainerprofile_types.go new file mode 100644 index 000000000..ee575c9fb --- /dev/null +++ b/api/v1alpha1/nnfcontainerprofile_types.go @@ -0,0 +1,86 @@ +/* + * Copyright 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + ContainerLabel = "nnf.cray.hpe.com/container" +) + +// NnfContainerProfileSpec defines the desired state of NnfContainerProfile +type NnfContainerProfileData struct { + // Pinned is true if this instance is an immutable copy + // +kubebuilder:default:=false + Pinned bool `json:"pinned,omitempty"` + + // List of possible filesystems supported by this container profile + Storages []NnfContainerProfileStorage `json:"storages,omitempty"` + + // Stop any containers after X seconds once a workflow has transitioned to PostRun. Defaults to 0. + // A value of 0 disables this behavior. + PostRunTimeoutSeconds int64 `json:"postRunTimeoutSeconds,omitempty"` + + // Specifies the number of times a container will be retried upon a failure. A new pod is deployed on each retry. + // Defaults to 6 by kubernetes itself and must be set. A value of 0 disables retries. + // +kubebuilder:default:=6 + RetryLimit int32 `json:"retryLimit"` + + // Template defines the containers that will be created from container profile + Template corev1.PodTemplateSpec `json:"template"` +} + +// NnfContainerProfileStorage defines the mount point information that will be available to the +// container +type NnfContainerProfileStorage struct { + // Name specifies the name of the mounted filesystem; must match the user supplied #DW directive + Name string `json:"name"` + + // Optional designates that this filesystem is available to be mounted, but can be ignored by + // the user not supplying this filesystem in the #DW directives + //+kubebuilder:default:=false + Optional bool `json:"optional"` +} + +// +kubebuilder:object:root=true + +// NnfContainerProfile is the Schema for the nnfcontainerprofiles API +type NnfContainerProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Data NnfContainerProfileData `json:"data,omitempty"` +} + +// +kubebuilder:object:root=true + +// NnfContainerProfileList contains a list of NnfContainerProfile +type NnfContainerProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NnfContainerProfile `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NnfContainerProfile{}, &NnfContainerProfileList{}) +} diff --git a/api/v1alpha1/workflow_helpers.go b/api/v1alpha1/workflow_helpers.go index 825401f2d..0ea6b11ed 100644 --- a/api/v1alpha1/workflow_helpers.go +++ b/api/v1alpha1/workflow_helpers.go @@ -33,4 +33,12 @@ const ( // PinnedStorageProfileLabelNameSpace is a label applied to NnfStorage objects to show // which pinned storage profile is being used. PinnedStorageProfileLabelNameSpace = "nnf.cray.hpe.com/pinned_storage_profile_namespace" + + // PinnedContainerProfileLabelName is a label applied to NnfStorage objects to show + // which pinned container profile is being used. + PinnedContainerProfileLabelName = "nnf.cray.hpe.com/pinned_container_profile_name" + + // PinnedContainerProfileLabelNameSpace is a label applied to NnfStorage objects to show + // which pinned container profile is being used. + PinnedContainerProfileLabelNameSpace = "nnf.cray.hpe.com/pinned_container_profile_namespace" ) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 30b43c54b..49df07a38 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* - * Copyright 2022 Hewlett Packard Enterprise Development LP + * Copyright 2023 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -171,6 +171,100 @@ func (in *NnfAccessStatus) DeepCopy() *NnfAccessStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfContainerProfile) DeepCopyInto(out *NnfContainerProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Data.DeepCopyInto(&out.Data) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfContainerProfile. +func (in *NnfContainerProfile) DeepCopy() *NnfContainerProfile { + if in == nil { + return nil + } + out := new(NnfContainerProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NnfContainerProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfContainerProfileData) DeepCopyInto(out *NnfContainerProfileData) { + *out = *in + if in.Storages != nil { + in, out := &in.Storages, &out.Storages + *out = make([]NnfContainerProfileStorage, len(*in)) + copy(*out, *in) + } + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfContainerProfileData. +func (in *NnfContainerProfileData) DeepCopy() *NnfContainerProfileData { + if in == nil { + return nil + } + out := new(NnfContainerProfileData) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfContainerProfileList) DeepCopyInto(out *NnfContainerProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NnfContainerProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfContainerProfileList. +func (in *NnfContainerProfileList) DeepCopy() *NnfContainerProfileList { + if in == nil { + return nil + } + out := new(NnfContainerProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NnfContainerProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfContainerProfileStorage) DeepCopyInto(out *NnfContainerProfileStorage) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfContainerProfileStorage. +func (in *NnfContainerProfileStorage) DeepCopy() *NnfContainerProfileStorage { + if in == nil { + return nil + } + out := new(NnfContainerProfileStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NnfDataMovement) DeepCopyInto(out *NnfDataMovement) { *out = *in diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfcontainerprofiles.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfcontainerprofiles.yaml new file mode 100644 index 000000000..1d4e07b6c --- /dev/null +++ b/config/crd/bases/nnf.cray.hpe.com_nnfcontainerprofiles.yaml @@ -0,0 +1,7264 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: nnfcontainerprofiles.nnf.cray.hpe.com +spec: + group: nnf.cray.hpe.com + names: + kind: NnfContainerProfile + listKind: NnfContainerProfileList + plural: nnfcontainerprofiles + singular: nnfcontainerprofile + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NnfContainerProfile is the Schema for the nnfcontainerprofiles + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + data: + description: NnfContainerProfileSpec defines the desired state of NnfContainerProfile + properties: + pinned: + default: false + description: Pinned is true if this instance is an immutable copy + type: boolean + postRunTimeoutSeconds: + description: Stop any containers after X seconds once a workflow has + transitioned to PostRun. Defaults to 0. A value of 0 disables this + behavior. + format: int64 + type: integer + retryLimit: + default: 6 + description: Specifies the number of times a container will be retried + upon a failure. A new pod is deployed on each retry. Defaults to + 6 by kubernetes itself and must be set. A value of 0 disables retries. + format: int32 + type: integer + storages: + description: List of possible filesystems supported by this container + profile + items: + description: NnfContainerProfileStorage defines the mount point + information that will be available to the container + properties: + name: + description: 'Name specifies the name of the mounted filesystem; + must match the user supplied #DW directive' + type: string + optional: + default: false + description: 'Optional designates that this filesystem is available + to be mounted, but can be ignored by the user not supplying + this filesystem in the #DW directives' + type: boolean + required: + - name + - optional + type: object + type: array + template: + description: Template defines the containers that will be created + from container profile + properties: + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + activeDeadlineSeconds: + description: Optional duration in seconds the pod may be active + on the node relative to StartTime before the system will + actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the + greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if + the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term + matches all objects with implicit weight 0 (i.e. + it's a no-op). A null preferred scheduling term + matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in the + range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the + affinity requirements specified by this field cease + to be met at some point during pod execution (e.g. + due to an update), the system may or may not try + to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them + are ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the + greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if + the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum + are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of + resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set + of namespaces that the term applies to. + The term is applied to the union of the + namespaces selected by this field and + the ones listed in the namespaces field. + null selector and null or empty namespaces + list means "this pod's namespace". An + empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static + list of namespace names that the term + applies to. The term is applied to the + union of the namespaces listed in this + field and the ones selected by namespaceSelector. + null or empty namespaces list and null + namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in the + range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the + affinity requirements specified by this field cease + to be met at some point during pod execution (e.g. + due to a pod label update), the system may or may + not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes + corresponding to each podAffinityTerm are intersected, + i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those + matching the labelSelector relative to the given + namespace(s)) that this pod should be co-located + (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node + whose value of the label with key + matches that of any node on which a pod of the + set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the + greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity + expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to + the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum + are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of + resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set + of namespaces that the term applies to. + The term is applied to the union of the + namespaces selected by this field and + the ones listed in the namespaces field. + null selector and null or empty namespaces + list means "this pod's namespace". An + empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static + list of namespace names that the term + applies to. The term is applied to the + union of the namespaces listed in this + field and the ones selected by namespaceSelector. + null or empty namespaces list and null + namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in the + range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the + anti-affinity requirements specified by this field + cease to be met at some point during pod execution + (e.g. due to a pod label update), the system may + or may not try to eventually evict the pod from + its node. When there are multiple elements, the + lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those + matching the labelSelector relative to the given + namespace(s)) that this pod should be co-located + (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node + whose value of the label with key + matches that of any node on which a pod of the + set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: List of containers belonging to the pod. Containers + cannot currently be added or removed. There must be at least + one container in a Pod. Cannot be updated. + items: + description: A single application container that you want + to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The container + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the + reference in the input string will be unchanged. Double + $$ are reduced to a single $, which allows for escaping + the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within + a shell. The container image''s ENTRYPOINT is used + if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If + a variable cannot be resolved, the reference in the + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in + the container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults + to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: + supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment + variables in the container. The keys defined within + a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is + starting. When a key exists in multiple sources, the + value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images + in workload controllers like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is + specified, or IfNotPresent otherwise. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should + take in response to container lifecycle events. Cannot + be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, + the container is terminated and restarted according + to its restart policy. Other management of the + container blocks until the hook completes. More + info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line + to execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside + a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, + you need to explicitly call out to that + shell. Exit status of 0 is treated as + live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of + this field and lifecycle hooks will fail in + runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup probe + failure, preemption, resource contention, etc. + The handler is not called if the container crashes + or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the + container will eventually terminate within the + Pod''s termination grace period (unless delayed + by finalizers). Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line + to execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside + a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, + you need to explicitly call out to that + shell. Exit status of 0 is treated as + live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of + this field and lifecycle hooks will fail in + runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Not specifying a port here DOES NOT prevent that port + from being exposed. Any port which is listening on + the default "0.0.0.0" address inside a container will + be accessible from the network. Modifying this array + with strategic merge patch may corrupt the data. For + more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port + in a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, + this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the + port that can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if + the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security options + the container should be run with. If set, the fields + of SecurityContext override the equivalent fields + of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges than + its parent process. This bool directly controls + if the no_new_privs flag will be set on the container + process. AllowPrivilegeEscalation is true always + when the container is: 1) run as Privileged 2) + has CAP_SYS_ADMIN Note that this field cannot + be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. Note that this + field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. Note that + this field cannot be set when spec.os.name is + windows. + type: boolean + procMount: + description: procMount denotes the type of proc + mount to use for the containers. The default is + DefaultProcMount which uses the container runtime + defaults for readonly paths and masked paths. + This requires the ProcMountType feature flag to + be enabled. Note that this field cannot be set + when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. Note that this + field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will + validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no + such validation will be performed. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified + in image metadata if unspecified. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. Note that this + field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this + container. If seccomp options are provided at + both the pod & container level, the container + options override the pod options. Note that this + field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. + The profile must be preconfigured on the node + to work. Must be a descending path, relative + to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: + \n Localhost - a profile defined in a file + on the node should be used. RuntimeDefault + - the container runtime default profile should + be used. Unconfined - no profile should be + applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied + to all containers. If unspecified, the options + from the PodSecurityContext will be used. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the + GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. + This field is alpha-level and will only be + honored by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the + feature flag will result in errors when validating + the Pod. All of a Pod's containers must have + the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, + if HostProcess is true then HostNetwork must + also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run + the entrypoint of the container process. Defaults + to the user specified in image metadata if + unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has + successfully initialized. If specified, no other probes + are executed until this completes successfully. If + this probe fails, the Pod will be restarted, just + as if the livenessProbe failed. This can be used to + provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time + to load data or warm a cache, than during steady-state + operation. This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate + a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will + always result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce + is set to true, stdin is opened on container start, + is empty until the first client attaches to stdin, + and then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such + as an assertion failure message. Will be truncated + by the node if greater than 4096 bytes. The total + message length across all containers will be limited + to 12kb. Defaults to /dev/termination-log. Cannot + be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be + updated. + type: string + tty: + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. + items: + description: volumeDevice describes a mapping of a + raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of + the container that the device will be mapped + to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting of a + Volume within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. + type: boolean + subPath: + description: Path within the volume from which + the container's volume should be mounted. Defaults + to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment + variable references $(VAR_NAME) are expanded + using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath + are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot + be updated. + type: string + required: + - name + type: object + type: array + dnsConfig: + description: Specifies the DNS parameters of a pod. Parameters + specified here will be merged to the generated DNS configuration + based on DNSPolicy. + properties: + nameservers: + description: A list of DNS name server IP addresses. This + will be appended to the base nameservers generated from + DNSPolicy. Duplicated nameservers will be removed. + items: + type: string + type: array + options: + description: A list of DNS resolver options. This will + be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options + given in Options will override those that appear in + the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + searches: + description: A list of DNS search domains for host-name + lookup. This will be appended to the base search paths + generated from DNSPolicy. Duplicated search paths will + be removed. + items: + type: string + type: array + type: object + dnsPolicy: + description: Set DNS policy for the pod. Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', + 'Default' or 'None'. DNS parameters given in DNSConfig will + be merged with the policy selected with DNSPolicy. To have + DNS options set along with hostNetwork, you have to specify + DNS policy explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: 'EnableServiceLinks indicates whether information + about services should be injected into pod''s environment + variables, matching the syntax of Docker links. Optional: + Defaults to true.' + type: boolean + ephemeralContainers: + description: List of ephemeral containers run in this pod. + Ephemeral containers may be run in an existing pod to perform + user-initiated actions such as debugging. This list cannot + be specified when creating a pod, and it cannot be modified + by updating the pod spec. In order to add an ephemeral container + to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: "An EphemeralContainer is a temporary container + that you may add to an existing Pod for user-initiated + activities such as debugging. Ephemeral containers have + no resource or scheduling guarantees, and they will not + be restarted when they exit or when a Pod is removed or + restarted. The kubelet may evict a Pod if an ephemeral + container causes the Pod to exceed its resource allocation. + \n To add an ephemeral container, use the ephemeralcontainers + subresource of an existing Pod. Ephemeral containers may + not be removed or restarted." + properties: + args: + description: 'Arguments to the entrypoint. The image''s + CMD is used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s environment. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within + a shell. The image''s ENTRYPOINT is used if this is + not provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. If a + variable cannot be resolved, the reference in the + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in + the container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults + to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: + supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment + variables in the container. The keys defined within + a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is + starting. When a key exists in multiple sources, the + value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is + specified, or IfNotPresent otherwise. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, + the container is terminated and restarted according + to its restart policy. Other management of the + container blocks until the hook completes. More + info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line + to execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside + a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, + you need to explicitly call out to that + shell. Exit status of 0 is treated as + live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of + this field and lifecycle hooks will fail in + runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup probe + failure, preemption, resource contention, etc. + The handler is not called if the container crashes + or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the + container will eventually terminate within the + Pod''s termination grace period (unless delayed + by finalizers). Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line + to execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside + a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, + you need to explicitly call out to that + shell. Exit status of 0 is treated as + live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of + this field and lifecycle hooks will fail in + runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the ephemeral container specified + as a DNS_LABEL. This name must be unique among all + containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral containers. + items: + description: ContainerPort represents a network port + in a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, + this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the + port that can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: Resources are not allowed for ephemeral + containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'Optional: SecurityContext defines the + security options the ephemeral container should be + run with. If set, the fields of SecurityContext override + the equivalent fields of PodSecurityContext.' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges than + its parent process. This bool directly controls + if the no_new_privs flag will be set on the container + process. AllowPrivilegeEscalation is true always + when the container is: 1) run as Privileged 2) + has CAP_SYS_ADMIN Note that this field cannot + be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. Note that this + field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. Note that + this field cannot be set when spec.os.name is + windows. + type: boolean + procMount: + description: procMount denotes the type of proc + mount to use for the containers. The default is + DefaultProcMount which uses the container runtime + defaults for readonly paths and masked paths. + This requires the ProcMountType feature flag to + be enabled. Note that this field cannot be set + when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. Note that this + field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will + validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no + such validation will be performed. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified + in image metadata if unspecified. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. Note that this + field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this + container. If seccomp options are provided at + both the pod & container level, the container + options override the pod options. Note that this + field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. + The profile must be preconfigured on the node + to work. Must be a descending path, relative + to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: + \n Localhost - a profile defined in a file + on the node should be used. RuntimeDefault + - the container runtime default profile should + be used. Unconfined - no profile should be + applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied + to all containers. If unspecified, the options + from the PodSecurityContext will be used. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the + GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. + This field is alpha-level and will only be + honored by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the + feature flag will result in errors when validating + the Pod. All of a Pod's containers must have + the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, + if HostProcess is true then HostNetwork must + also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run + the entrypoint of the container process. Defaults + to the user specified in image metadata if + unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate + a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will + always result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce + is set to true, stdin is opened on container start, + is empty until the first client attaches to stdin, + and then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false + type: boolean + targetContainerName: + description: "If set, the name of the container from + PodSpec that this ephemeral container targets. The + ephemeral container will be run in the namespaces + (IPC, PID, etc) of this container. If not set then + the ephemeral container uses the namespaces configured + in the Pod spec. \n The container runtime must implement + support for this feature. If the runtime does not + support namespace targeting then the result of setting + this field is undefined." + type: string + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such + as an assertion failure message. Will be truncated + by the node if greater than 4096 bytes. The total + message length across all containers will be limited + to 12kb. Defaults to /dev/termination-log. Cannot + be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be + updated. + type: string + tty: + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. + items: + description: volumeDevice describes a mapping of a + raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of + the container that the device will be mapped + to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Subpath mounts are not allowed for ephemeral + containers. Cannot be updated. + items: + description: VolumeMount describes a mounting of a + Volume within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. + type: boolean + subPath: + description: Path within the volume from which + the container's volume should be mounted. Defaults + to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment + variable references $(VAR_NAME) are expanded + using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath + are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot + be updated. + type: string + required: + - name + type: object + type: array + hostAliases: + description: HostAliases is an optional list of hosts and + IPs that will be injected into the pod's hosts file if specified. + This is only valid for non-hostNetwork pods. + items: + description: HostAlias holds the mapping between IP and + hostnames that will be injected as an entry in the pod's + hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + ip: + description: IP address of the host file entry. + type: string + type: object + type: array + hostIPC: + description: 'Use the host''s ipc namespace. Optional: Default + to false.' + type: boolean + hostNetwork: + description: Host networking requested for this pod. Use the + host's network namespace. If this option is set, the ports + that will be used must be specified. Default to false. + type: boolean + hostPID: + description: 'Use the host''s pid namespace. Optional: Default + to false.' + type: boolean + hostUsers: + description: 'Use the host''s user namespace. Optional: Default + to true. If set to true or not present, the pod will be + run in the host user namespace, useful for when the pod + needs a feature only available to the host user namespace, + such as loading a kernel module with CAP_SYS_MODULE. When + set to false, a new userns is created for the pod. Setting + false is useful for mitigating container breakout vulnerabilities + even allowing users to run their containers as root without + actually having root privileges on the host. This field + is alpha-level and is only honored by servers that enable + the UserNamespacesSupport feature.' + type: boolean + hostname: + description: Specifies the hostname of the Pod If not specified, + the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: 'ImagePullSecrets is an optional list of references + to secrets in the same namespace to use for pulling any + of the images used by this PodSpec. If specified, these + secrets will be passed to individual puller implementations + for them to use. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + items: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the same + namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + description: 'List of initialization containers belonging + to the pod. Init containers are executed in order prior + to containers being started. If any init container fails, + the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or + normal container must be unique among all containers. Init + containers may not have Lifecycle actions, Readiness probes, + Liveness probes, or Startup probes. The resourceRequirements + of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, + and then using the max of of that value or the sum of the + normal containers. Limits are applied to init containers + in a similar fashion. Init containers cannot currently be + added or removed. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' + items: + description: A single application container that you want + to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The container + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the + reference in the input string will be unchanged. Double + $$ are reduced to a single $, which allows for escaping + the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within + a shell. The container image''s ENTRYPOINT is used + if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If + a variable cannot be resolved, the reference in the + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in + the container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults + to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: + supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment + variables in the container. The keys defined within + a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is + starting. When a key exists in multiple sources, the + value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images + in workload controllers like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is + specified, or IfNotPresent otherwise. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should + take in response to container lifecycle events. Cannot + be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, + the container is terminated and restarted according + to its restart policy. Other management of the + container blocks until the hook completes. More + info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line + to execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside + a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, + you need to explicitly call out to that + shell. Exit status of 0 is treated as + live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of + this field and lifecycle hooks will fail in + runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup probe + failure, preemption, resource contention, etc. + The handler is not called if the container crashes + or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the + container will eventually terminate within the + Pod''s termination grace period (unless delayed + by finalizers). Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line + to execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside + a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, + you need to explicitly call out to that + shell. Exit status of 0 is treated as + live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward + compatibility. There are no validation of + this field and lifecycle hooks will fail in + runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Not specifying a port here DOES NOT prevent that port + from being exposed. Any port which is listening on + the default "0.0.0.0" address inside a container will + be accessible from the network. Modifying this array + with strategic merge patch may corrupt the data. For + more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port + in a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, + this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the + port that can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if + the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security options + the container should be run with. If set, the fields + of SecurityContext override the equivalent fields + of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges than + its parent process. This bool directly controls + if the no_new_privs flag will be set on the container + process. AllowPrivilegeEscalation is true always + when the container is: 1) run as Privileged 2) + has CAP_SYS_ADMIN Note that this field cannot + be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. Note that this + field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. Note that + this field cannot be set when spec.os.name is + windows. + type: boolean + procMount: + description: procMount denotes the type of proc + mount to use for the containers. The default is + DefaultProcMount which uses the container runtime + defaults for readonly paths and masked paths. + This requires the ProcMountType feature flag to + be enabled. Note that this field cannot be set + when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. Note that this + field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will + validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no + such validation will be performed. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified + in image metadata if unspecified. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. Note that this + field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this + container. If seccomp options are provided at + both the pod & container level, the container + options override the pod options. Note that this + field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. + The profile must be preconfigured on the node + to work. Must be a descending path, relative + to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: + \n Localhost - a profile defined in a file + on the node should be used. RuntimeDefault + - the container runtime default profile should + be used. Unconfined - no profile should be + applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied + to all containers. If unspecified, the options + from the PodSecurityContext will be used. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name + is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the + GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. + This field is alpha-level and will only be + honored by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the + feature flag will result in errors when validating + the Pod. All of a Pod's containers must have + the same effective HostProcess value (it is + not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, + if HostProcess is true then HostNetwork must + also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run + the entrypoint of the container process. Defaults + to the user specified in image metadata if + unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has + successfully initialized. If specified, no other probes + are executed until this completes successfully. If + this probe fails, the Pod will be restarted, just + as if the livenessProbe failed. This can be used to + provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time + to load data or warm a cache, than during steady-state + operation. This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. This is a beta field and requires + enabling GRPCContainerProbe feature gate. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service + to place in the gRPC HealthCheckRequest (see + https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is + 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having + failed. Defaults to 1. Must be 1 for liveness + and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod + needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer + than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides + the value provided by the pod spec. Value must + be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity + to shut down). This is a beta field and requires + enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the + probe times out. Defaults to 1 second. Minimum + value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate + a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will + always result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce + is set to true, stdin is opened on container start, + is empty until the first client attaches to stdin, + and then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such + as an assertion failure message. Will be truncated + by the node if greater than 4096 bytes. The total + message length across all containers will be limited + to 12kb. Defaults to /dev/termination-log. Cannot + be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be + updated. + type: string + tty: + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. + items: + description: volumeDevice describes a mapping of a + raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of + the container that the device will be mapped + to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting of a + Volume within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. + type: boolean + subPath: + description: Path within the volume from which + the container's volume should be mounted. Defaults + to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment + variable references $(VAR_NAME) are expanded + using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath + are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot + be updated. + type: string + required: + - name + type: object + type: array + nodeName: + description: NodeName is a request to schedule this pod onto + a specific node. If it is non-empty, the scheduler simply + schedules this pod onto that node, assuming that it fits + resource requirements. + type: string + nodeSelector: + additionalProperties: + type: string + description: 'NodeSelector is a selector which must be true + for the pod to fit on a node. Selector which must match + a node''s labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + x-kubernetes-map-type: atomic + os: + description: "Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is + set. \n If the OS field is set to linux, the following fields + must be unset: -securityContext.windowsOptions \n If the + OS field is set to windows, following fields must be unset: + - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls + - spec.shareProcessNamespace - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups + - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup" + properties: + name: + description: 'Name is the name of the operating system. + The currently supported values are linux and windows. + Additional value may be defined in future and can be + one of: https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and + treat unrecognized values in this field as os: null' + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Overhead represents the resource overhead associated + with running a pod for a given RuntimeClass. This field + will be autopopulated at admission time by the RuntimeClass + admission controller. If the RuntimeClass admission controller + is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create + requests which have the overhead already set. If RuntimeClass + is configured and selected in the PodSpec, Overhead will + be set to the value defined in the corresponding RuntimeClass, + otherwise it will remain unset and treated as zero. More + info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md' + type: object + preemptionPolicy: + description: PreemptionPolicy is the Policy for preempting + pods with lower priority. One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: The priority value. Various system components + use this field to find the priority of the pod. When Priority + Admission Controller is enabled, it prevents users from + setting this field. The admission controller populates this + field from PriorityClassName. The higher the value, the + higher the priority. + format: int32 + type: integer + priorityClassName: + description: If specified, indicates the pod's priority. "system-node-critical" + and "system-cluster-critical" are two special keywords which + indicate the highest priorities with the former being the + highest priority. Any other name must be defined by creating + a PriorityClass object with that name. If not specified, + the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: 'If specified, all readiness gates will be evaluated + for pod readiness. A pod is ready when all its containers + are ready AND all conditions specified in the readiness + gates have status equal to "True" More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates' + items: + description: PodReadinessGate contains the reference to + a pod condition + properties: + conditionType: + description: ConditionType refers to a condition in + the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + description: 'Restart policy for all containers within the + pod. One of Always, OnFailure, Never. Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' + type: string + runtimeClassName: + description: 'RuntimeClassName refers to a RuntimeClass object + in the node.k8s.io group, which should be used to run this + pod. If no RuntimeClass resource matches the named class, + the pod will not be run. If unset or empty, the "legacy" + RuntimeClass will be used, which is an implicit class with + an empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class' + type: string + schedulerName: + description: If specified, the pod will be dispatched by specified + scheduler. If not specified, the pod will be dispatched + by default scheduler. + type: string + securityContext: + description: 'SecurityContext holds pod-level security attributes + and common container settings. Optional: Defaults to empty. See + type description for default values of each field.' + properties: + fsGroup: + description: "A special supplemental group that applies + to all containers in a pod. Some volume types allow + the Kubelet to change the ownership of that volume to + be owned by the pod: \n 1. The owning GID will be the + FSGroup 2. The setgid bit is set (new files created + in the volume will be owned by FSGroup) 3. The permission + bits are OR'd with rw-rw---- \n If unset, the Kubelet + will not modify the ownership and permissions of any + volume. Note that this field cannot be set when spec.os.name + is windows." + format: int64 + type: integer + fsGroupChangePolicy: + description: 'fsGroupChangePolicy defines behavior of + changing ownership and permission of the volume before + being exposed inside Pod. This field will only apply + to volume types which support fsGroup based ownership(and + permissions). It will have no effect on ephemeral volume + types such as: secret, configmaps and emptydir. Valid + values are "OnRootMismatch" and "Always". If not specified, + "Always" is used. Note that this field cannot be set + when spec.os.name is windows.' + type: string + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. Note that this + field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as + a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not run + as UID 0 (root) and fail to start the container if it + does. If unset or false, no such validation will be + performed. May also be set in SecurityContext. If set + in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence + for that container. Note that this field cannot be set + when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all + containers. If unspecified, the container runtime will + allocate a random SELinux context for each container. May + also be set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. Note that this + field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by the containers + in this pod. Note that this field cannot be set when + spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. The + profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's + configured seccomp profile location. Must only be + set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: \n Localhost + - a profile defined in a file on the node should + be used. RuntimeDefault - the container runtime + default profile should be used. Unconfined - no + profile should be applied." + type: string + required: + - type + type: object + supplementalGroups: + description: A list of groups applied to the first process + run in each container, in addition to the container's + primary GID. If unspecified, no groups will be added + to any container. Note that this field cannot be set + when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls + used for the pod. Pods with unsupported sysctls (by + the container runtime) might fail to launch. Note that + this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be + set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to + all containers. If unspecified, the options within a + container's SecurityContext will be used. If set in + both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec + named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of + the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. This + field is alpha-level and will only be honored by + components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the feature + flag will result in errors when validating the Pod. + All of a Pod's containers must have the same effective + HostProcess value (it is not allowed to have a mix + of HostProcess containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set + in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + serviceAccount: + description: 'DeprecatedServiceAccount is a depreciated alias + for ServiceAccountName. Deprecated: Use serviceAccountName + instead.' + type: string + serviceAccountName: + description: 'ServiceAccountName is the name of the ServiceAccount + to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + type: string + setHostnameAsFQDN: + description: If true the pod's hostname will be configured + as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the + hostname field of the kernel (the nodename field of struct + utsname). In Windows containers, this means setting the + registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters + to FQDN. If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: 'Share a single process namespace between all + of the containers in a pod. When this is set containers + will be able to view and signal processes from other containers + in the same pod, and the first process in each container + will not be assigned PID 1. HostPID and ShareProcessNamespace + cannot both be set. Optional: Default to false.' + type: boolean + subdomain: + description: If specified, the fully qualified Pod hostname + will be "...svc.". If not specified, the pod will not have a domainname + at all. + type: string + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to + terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). If this value is nil, the default grace period + will be used instead. The grace period is the duration in + seconds after the processes running in the pod are sent + a termination signal and the time when the processes are + forcibly halted with a kill signal. Set this value longer + than the expected cleanup time for your process. Defaults + to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period + of time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value should + be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: TopologySpreadConstraints describes how a group + of pods ought to spread across topology domains. Scheduler + will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching + pods. Pods that match this label selector are counted + to determine the number of pods in their corresponding + topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys + to select the pods over which spreading will be calculated. + The keys are used to lookup values from the incoming + pod labels, those key-value labels are ANDed with + labelSelector to select the group of existing pods + over which spreading will be calculated for the incoming + pod. Keys that don't exist in the incoming pod labels + will be ignored. A null or empty list means only match + against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: 'MaxSkew describes the degree to which + pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the + number of matching pods in the target topology and + the global minimum. The global minimum is the minimum + number of matching pods in an eligible domain or zero + if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to + 1, and pods with the same labelSelector spread as + 2/2/1: In this case, the global minimum is 1. | zone1 + | zone2 | zone3 | | P P | P P | P | - if MaxSkew + is 1, incoming pod can only be scheduled to zone3 + to become 2/2/2; scheduling it onto zone1(zone2) would + make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto + any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies + that satisfy it. It''s a required field. Default value + is 1 and 0 is not allowed.' + format: int32 + type: integer + minDomains: + description: "MinDomains indicates a minimum number + of eligible domains. When the number of eligible domains + with matching topology keys is less than minDomains, + Pod Topology Spread treats \"global minimum\" as 0, + and then the calculation of Skew is performed. And + when the number of eligible domains with matching + topology keys equals or greater than minDomains, this + value has no effect on scheduling. As a result, when + the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to + those domains. If value is nil, the constraint behaves + as if MinDomains is equal to 1. Valid values are integers + greater than 0. When value is not nil, WhenUnsatisfiable + must be DoNotSchedule. \n For example, in a 3-zone + cluster, MaxSkew is set to 2, MinDomains is set to + 5 and pods with the same labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | | P P | P P | P P | + The number of domains is less than 5(MinDomains), + so \"global minimum\" is treated as 0. In this situation, + new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod + is scheduled to any of the three zones, it will violate + MaxSkew. \n This is a beta field and requires the + MinDomainsInPodTopologySpread feature gate to be enabled + (enabled by default)." + format: int32 + type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will + treat Pod's nodeAffinity/nodeSelector when calculating + pod topology spread skew. Options are: - Honor: only + nodes matching nodeAffinity/nodeSelector are included + in the calculations. - Ignore: nodeAffinity/nodeSelector + are ignored. All nodes are included in the calculations. + \n If this value is nil, the behavior is equivalent + to the Honor policy. This is a alpha-level feature + enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will + treat node taints when calculating pod topology spread + skew. Options are: - Honor: nodes without taints, + along with tainted nodes for which the incoming pod + has a toleration, are included. - Ignore: node taints + are ignored. All nodes are included. \n If this value + is nil, the behavior is equivalent to the Ignore policy. + This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + topologyKey: + description: TopologyKey is the key of node labels. + Nodes that have a label with this key and identical + values are considered to be in the same topology. + We consider each as a "bucket", and try + to put balanced number of pods into each bucket. We + define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose + nodes meet the requirements of nodeAffinityPolicy + and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain + of that topology. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal + with a pod if it doesn''t satisfy the spread constraint. + - DoNotSchedule (default) tells the scheduler not + to schedule it. - ScheduleAnyway tells the scheduler + to schedule the pod in any location, but giving higher + precedence to topologies that would help reduce the + skew. A constraint is considered "Unsatisfiable" for + an incoming pod if and only if every possible node + assignment for that pod would violate "MaxSkew" on + some topology. For example, in a 3-zone cluster, MaxSkew + is set to 1, and pods with the same labelSelector + spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P + | P | P | If WhenUnsatisfiable is set to DoNotSchedule, + incoming pod can only be scheduled to zone2(zone3) + to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) + satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make + it *more* imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: 'List of volumes that can be mounted by containers + belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS + Disk resource that is attached to a kubelet''s host + machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type of the + volume that you want to mount. Tip: Ensure that + the filesystem type is supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'partition is the partition in the + volume that you want to mount. If omitted, the + default is to mount by volume name. Examples: + For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda + is "0" (or you can leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the + readOnly setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). More + info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk + mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: + None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk + in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in + the blob storage + type: string + fsType: + description: fsType is Filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single + blob disk per storage account Managed: azure + managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that + contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the + host that shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors is + a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted + root, rather than the full Ceph tree, default + is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults to + false (read/write). ReadOnly here will force the + ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile + is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is + reference to the authentication secret for User, + default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is optional: User is the rados + user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a + secret object containing parameters used to connect + to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: 'volumeID used to identify the volume + in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits + used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories + within the path are not affected by this setting. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced ConfigMap + will be projected into the volume as a file whose + name is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the ConfigMap, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. Must + be an octal value between 0000 and 0777 + or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON + requires decimal values for mode bits. If + not specified, the volume defaultMode will + be used. This might be in conflict with + other options that affect the file mode, + like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of + the file to map the key to. May not be an + absolute path. May not contain the path + element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI driver + that handles this volume. Consult with your admin + for the correct name as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed + to the associated CSI driver which will determine + the default filesystem to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference + to the secret object containing sensitive information + to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no + secret is required. If the secret object contains + more than one secret, all secret references are + passed. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific + properties that are passed to the CSI driver. + Consult your driver's documentation for supported + values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about + the pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a Optional: mode bits + used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories + within the path are not affected by this setting. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing the + pod field + properties: + fieldRef: + description: 'Required: Selects a field of + the pod: only annotations, labels, name + and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used to + set permissions on this file, must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both + octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of + the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage + medium should back this directory. The default + is "" which means to use the node''s default medium. + Must be an empty string (default) or Memory. More + info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local + storage required for this EmptyDir volume. The + size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would + be the minimum value between the SizeLimit specified + here and the sum of memory limits of all containers + in a pod. The default is nil which means that + the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is + handled by a cluster storage driver. The volume's + lifecycle is tied to the pod that defines it - it + will be created before the pod starts, and deleted + when the pod is removed. \n Use this if: a) the volume + is only needed while the pod runs, b) features of + normal volumes like restoring from snapshot or capacity + tracking are needed, c) the storage driver is specified + through a storage class, and d) the storage driver + supports dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on + the connection between this volume type and PersistentVolumeClaim). + \n Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the + lifecycle of an individual pod. \n Use CSI for light-weight + local ephemeral volumes if the CSI driver is meant + to be used that way - see the documentation of the + driver for more information. \n A pod can use both + types of ephemeral volumes and persistent volumes + at the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone + PVC to provision the volume. The pod in which + this EphemeralVolumeSource is embedded will be + the owner of the PVC, i.e. the PVC will be deleted + together with the pod. The name of the PVC will + be `-` where `` + is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too + long). \n An existing PVC with that name that + is not owned by the pod will *not* be used for + the pod to avoid using an unrelated volume by + mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created + PVC is meant to be used by the pod, the PVC has + to updated with an owner reference to the pod + once the pod exists. Normally this should not + be necessary, but it may be useful when manually + reconstructing a broken cluster. \n This field + is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, + must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when creating + it. No other fields are allowed and will be + rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into + the PVC that gets created from this template. + The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: 'accessModes contains the desired + access modes the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used + to specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller + can support the specified data source, + it will create a new volume based on the + contents of the specified data source. + If the AnyVolumeDataSource feature gate + is enabled, this field will always have + the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for + the resource being referenced. If + APIGroup is not specified, the specified + Kind must be in the core API group. + For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the + object from which to populate the volume + with data, if a non-empty volume is desired. + This may be any local object from a non-empty + API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, + volume binding will only succeed if the + type of the specified object matches some + installed volume populator or dynamic + provisioner. This field will replace the + functionality of the DataSource field + and as such if both fields are non-empty, + they must have the same value. For backwards + compatibility, both fields (DataSource + and DataSourceRef) will be set to the + same value automatically if one of them + is empty and the other is non-empty. There + are two important differences between + DataSource and DataSourceRef: * While + DataSource only allows two specific types + of objects, DataSourceRef allows any non-core + object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed + values (dropping them), DataSourceRef + preserves all values, and generates an + error if a disallowed value is specified. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for + the resource being referenced. If + APIGroup is not specified, the specified + Kind must be in the core API group. + For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: 'resources represents the minimum + resources the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to + specify resource requirements that are + lower than previous value but must still + be higher than capacity recorded in the + status field of the claim. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the + minimum amount of compute resources + required. If Requests is omitted for + a container, it defaults to Limits + if that is explicitly specified, otherwise + to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over + volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name + of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type + of volume is required by the claim. Value + of Filesystem is implied when not included + in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and then + exposed to the pod. + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. TODO: how + do we prevent errors in the filesystem from compromising + the machine' + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to + false (read/write). ReadOnly here will force the + ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide + identifiers (wwids) Either wwids or combination + of targetWWNs and lun must be set, but not both + simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume + resource that is provisioned/attached using an exec + based plugin. + properties: + driver: + description: driver is the name of the driver to + use for this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". The + default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds + extra command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to + false (read/write). ReadOnly here will force the + ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is + reference to the secret object containing sensitive + information to pass to the plugin scripts. This + may be empty if no secret object is specified. + If the secret object contains more than one secret, + all secrets are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: datasetName is Name of the dataset + stored as metadata -> name on the dataset for + Flocker should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. + This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'partition is the partition in the + volume that you want to mount. If omitted, the + default is to mount by volume name. Examples: + For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda + is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource + in GCE. Used to identify the disk in GCE. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at + a particular revision. DEPRECATED: GitRepo is deprecated. + To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo + using git, then mount the EmptyDir into the Pod''s + container.' + properties: + directory: + description: directory is the target directory name. + Must not contain or start with '..'. If '.' is + supplied, the volume directory will be the git + repository. Otherwise, if specified, the volume + will contain the git repository in the subdirectory + with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the + specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that + details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs + volume to be mounted with read-only permissions. + Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file + or directory on the host machine that is directly + exposed to the container. This is generally used for + system agents or other privileged things that are + allowed to see the host machine. Most containers will + NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use + host directory mounts and who can/can not mount host + directories as read/write.' + properties: + path: + description: 'path of the directory on the host. + If the path is a symlink, it will follow the link + to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults + to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource + that is attached to a kubelet''s host machine and + then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support + iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the + volume that you want to mount. Tip: Ensure that + the filesystem type is supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name + that uses an iSCSI transport. Defaults to 'default' + (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal + List. The portal is either an IP or ip_addr:port + if the port is other than default (typically TCP + ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI + target and initiator authentication + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: targetPortal is iSCSI Target Portal. + The Portal is either an IP or ip_addr:port if + the port is other than default (typically TCP + ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL + and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'nfs represents an NFS mount on the host + that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address + of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same + namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly setting + in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon + Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type + to mount Must be a filesystem type supported by + the host operating system. Ex. "ext4", "xfs". + Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used + to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal + values for mode bits. Directories within the path + are not affected by this setting. This might be + in conflict with other options that affect the + file mode, like fsGroup, and the result can be + other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected + along with other supported volume types + properties: + configMap: + description: configMap information about the + configMap data to project + properties: + items: + description: items if unspecified, each + key-value pair in the Data field of + the referenced ConfigMap will be projected + into the volume as a file whose name + is the key and content is the value. + If specified, the listed keys will be + projected into the specified paths, + and unlisted keys will not be present. + If a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: + mode bits used to set permissions + on this file. Must be an octal + value between 0000 and 0777 or + a decimal value between 0 and + 511. YAML accepts both octal and + decimal values, JSON requires + decimal values for mode bits. + If not specified, the volume defaultMode + will be used. This might be in + conflict with other options that + affect the file mode, like fsGroup, + and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: path is the relative + path of the file to map the key + to. May not be an absolute path. + May not contain the path element + '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits + used to set permissions on this + file, must be an octal value between + 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts + both octal and decimal values, + JSON requires decimal values for + mode bits. If not specified, the + volume defaultMode will be used. + This might be in conflict with + other options that affect the + file mode, like fsGroup, and the + result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file + to be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource + of the container: only resources + limits and requests (limits.cpu, + limits.memory, requests.cpu and + requests.memory) are currently + supported.' + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the + secret data to project + properties: + items: + description: items if unspecified, each + key-value pair in the Data field of + the referenced Secret will be projected + into the volume as a file whose name + is the key and content is the value. + If specified, the listed keys will be + projected into the specified paths, + and unlisted keys will not be present. + If a key is specified which is not present + in the Secret, the volume setup will + error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: + mode bits used to set permissions + on this file. Must be an octal + value between 0000 and 0777 or + a decimal value between 0 and + 511. YAML accepts both octal and + decimal values, JSON requires + decimal values for mode bits. + If not specified, the volume defaultMode + will be used. This might be in + conflict with other options that + affect the file mode, like fsGroup, + and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: path is the relative + path of the file to map the key + to. May not be an absolute path. + May not contain the path element + '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: audience is the intended + audience of the token. A recipient of + a token must identify itself with an + identifier specified in the audience + of the token, and otherwise should reject + the token. The audience defaults to + the identifier of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the + requested duration of validity of the + service account token. As the token + approaches expiration, the kubelet volume + plugin will proactively rotate the service + account token. The kubelet will start + trying to rotate the token if the token + is older than 80 percent of its time + to live or if the token is older than + 24 hours.Defaults to 1 hour and must + be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative + to the mount point of the file to project + the token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the + host that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default + is no group + type: string + readOnly: + description: readOnly here will force the Quobyte + volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: registry represents a single or multiple + Quobyte Registry services specified as a string + as host:port pair (multiple entries are separated + with commas) which acts as the central registry + for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned + Quobyte volumes, value is set by the plugin + type: string + user: + description: user to map volume access to Defaults + to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the + volume that you want to mount. Tip: Ensure that + the filesystem type is supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + image: + description: 'image is the rados image name. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for + RBDUser. Default is /etc/ceph/keyring. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default + is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is the rados user name. Default + is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". Default + is "xfs". + type: string + gateway: + description: gateway is the host address of the + ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the + ScaleIO Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret + for ScaleIO user and other sensitive information. + If this is not provided, Login operation will + fail. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL + communication with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage + for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume + already created in the ScaleIO system that is + associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should + populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits + used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories + within the path are not affected by this setting. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value + pair in the Data field of the referenced Secret + will be projected into the volume as a file whose + name is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the Secret, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. Must + be an octal value between 0000 and 0777 + or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON + requires decimal values for mode bits. If + not specified, the volume defaultMode will + be used. This might be in conflict with + other options that affect the file mode, + like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of + the file to map the key to. May not be an + absolute path. May not contain the path + element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the + Secret or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret + in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use + for obtaining the StorageOS API credentials. If + not specified, default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: volumeName is the human-readable name + of the StorageOS volume. Volume names are only + unique within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope + of the volume within StorageOS. If no namespace + is specified then the Pod's namespace will be + used. This allows the Kubernetes name scoping + to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default + behaviour. Set to "default" if you are not using + namespaces within StorageOS. Namespaces that do + not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume + attached and mounted on kubelets host machine + properties: + fsType: + description: fsType is filesystem type to mount. + Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy + Based Management (SPBM) profile ID associated + with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy + Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - containers + type: object + type: object + required: + - retryLimit + - template + type: object + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 61c4a3336..56fe837f4 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -9,6 +9,7 @@ resources: - bases/nnf.cray.hpe.com_nnfaccesses.yaml - bases/nnf.cray.hpe.com_nnfstorageprofiles.yaml - bases/nnf.cray.hpe.com_nnfnodeecdata.yaml +- bases/nnf.cray.hpe.com_nnfcontainerprofiles.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -21,6 +22,7 @@ patchesStrategicMerge: #- patches/webhook_in_nnfaccesses.yaml #- patches/webhook_in_nnfstorageprofiles.yaml #- patches/webhook_in_nnfnodeecdata.yaml +#- patches/webhook_in_nnfcontainerprofiles.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. @@ -32,6 +34,7 @@ patchesStrategicMerge: #- patches/cainjection_in_nnfaccesses.yaml #- patches/cainjection_in_nnfstorageprofiles.yaml #- patches/cainjection_in_nnfnodeecdata.yaml +#- patches/cainjection_in_nnfcontainerprofiles.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_nnfcontainerprofiles.yaml b/config/crd/patches/cainjection_in_nnfcontainerprofiles.yaml new file mode 100644 index 000000000..700aa412b --- /dev/null +++ b/config/crd/patches/cainjection_in_nnfcontainerprofiles.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: nnfcontainerprofiles.nnf.cray.hpe.com diff --git a/config/crd/patches/webhook_in_nnfcontainerprofiles.yaml b/config/crd/patches/webhook_in_nnfcontainerprofiles.yaml new file mode 100644 index 000000000..241549a58 --- /dev/null +++ b/config/crd/patches/webhook_in_nnfcontainerprofiles.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: nnfcontainerprofiles.nnf.cray.hpe.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/dws/nnf-ruleset.yaml b/config/dws/nnf-ruleset.yaml index 6146bdf7d..9d4742ddb 100644 --- a/config/dws/nnf-ruleset.yaml +++ b/config/dws/nnf-ruleset.yaml @@ -92,3 +92,18 @@ spec: type: "string" isRequired: true isValueRequired: true + - command: "container" + watchStates: Proposal,PreRun,PostRun,Teardown + ruleDefs: + - key: "^name$" + type: "string" + isRequired: true + isValueRequired: true + - key: "^profile$" + type: "string" + isRequired: true + isValueRequired: true + - key: '^(DW_JOB_|DW_PERSISTENT_)[A-Za-z][A-Za-z0-9-]+$' + type: "string" + isRequired: false + isValueRequired: true diff --git a/config/examples/kustomization.yaml b/config/examples/kustomization.yaml new file mode 100644 index 000000000..6cf71071f --- /dev/null +++ b/config/examples/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: nnf-system + +resources: +- nnf_v1alpha1_nnfcontainerprofiles.yaml +- nnf_v1alpha1_nnfstorageprofile.yaml diff --git a/config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml b/config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml new file mode 100644 index 000000000..3b53bfa8f --- /dev/null +++ b/config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml @@ -0,0 +1,59 @@ +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: example-success +data: + storages: + - name: DW_JOB_foo-local-storage + optional: false + - name: DW_PERSISTENT_foo-persistent-storage + optional: true + template: + spec: + containers: + - name: example-success + image: alpine:latest + command: + - /bin/sh + - -c + - "sleep 15 && exit 0" +--- +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: example-randomly-fail +data: + storages: + - name: DW_JOB_foo-local-storage + optional: false + - name: DW_PERSISTENT_foo-persistent-storage + optional: true + template: + spec: + containers: + - name: example-randomly-fail + image: alpine:latest + command: + - /bin/sh + - -c + - | + echo "starting..." + sleep 30 + x=$(($RANDOM % 2)) + echo "exiting: $x" + exit $x +--- +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: example-forever +data: + template: + spec: + containers: + - name: example-forever + image: alpine:latest + command: + - /bin/sh + - -c + - "while true; do date && sleep 5; done" diff --git a/config/samples/placeholder_nnfstorageprofile.yaml b/config/examples/nnf_v1alpha1_nnfstorageprofile.yaml similarity index 100% rename from config/samples/placeholder_nnfstorageprofile.yaml rename to config/examples/nnf_v1alpha1_nnfstorageprofile.yaml diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 1f8de9f86..747990bed 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -17,4 +17,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/nearnodeflash/nnf-sos - newTag: 70d0095f27d713fbd964c101084ef65ae2723553 + newTag: 8d1d7ba8655c7cee86d54a036deffe931038ebdd diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d14b2afcd..eaefddf58 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -113,6 +113,8 @@ spec: value: "2000" - name: NNF_STORAGE_PROFILE_NAMESPACE value: nnf-system + - name: NNF_CONTAINER_PROFILE_NAMESPACE + value: nnf-system ports: - containerPort: 50057 name: nnf-ec diff --git a/config/rbac/nnfcontainerprofile_editor_role.yaml b/config/rbac/nnfcontainerprofile_editor_role.yaml new file mode 100644 index 000000000..4808463a2 --- /dev/null +++ b/config/rbac/nnfcontainerprofile_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit nnfcontainerprofiles. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nnfcontainerprofile-editor-role +rules: +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfcontainerprofiles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfcontainerprofiles/status + verbs: + - get diff --git a/config/rbac/nnfcontainerprofile_viewer_role.yaml b/config/rbac/nnfcontainerprofile_viewer_role.yaml new file mode 100644 index 000000000..31fcc5f3f --- /dev/null +++ b/config/rbac/nnfcontainerprofile_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view nnfcontainerprofiles. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nnfcontainerprofile-viewer-role +rules: +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfcontainerprofiles + verbs: + - get + - list + - watch +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfcontainerprofiles/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f2427e224..ad252ceb2 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,18 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update - apiGroups: - "" resources: @@ -32,6 +44,18 @@ rules: - get - list - update +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update - apiGroups: - cray.hpe.com resources: @@ -274,6 +298,19 @@ rules: - get - patch - update +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfcontainerprofiles + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch - apiGroups: - nnf.cray.hpe.com resources: diff --git a/config/samples/nnf_v1alpha1_nnfcontainerprofile.yaml b/config/samples/nnf_v1alpha1_nnfcontainerprofile.yaml new file mode 100644 index 000000000..c65fd0c4b --- /dev/null +++ b/config/samples/nnf_v1alpha1_nnfcontainerprofile.yaml @@ -0,0 +1,39 @@ +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfContainerProfile +metadata: + name: sample-nnfcontainerprofile + namespace: nnf-system +data: + # Specifies the number of times a container will be retried upon a failure. A + # new pod is deployed on each retry. Defaults to 6 by kubernetes itself and + # must be set. A value of 0 disables retries. + retryLimit: 6 + + # Stop any containers after X seconds once a workflow has transitioned to + # PostRun. Defaults to 0. A value of 0 disables this behavior. + postRunTimeoutSeconds: 0 + + # List of possible filesystems supported by this container profile. These + # storages are mounted inside of the container. Any non-optional storage must + # be supplied with the container directive as an argument and must reference + # a valid jobdw or persistentdw directive's name. + # + # Example: + # DW jobdw name=my-gfs2 type=gfs2 capacity=50GB + # DW container name=my-container profile=nnfcontainerprofile-sample DW_JOB_foo-local-storage=my-gfs2 + storages: + - name: DW_JOB_foo-local-storage + optional: false + - name: DW_PERSISTENT_foo-persistent-storage + optional: true + + # Template defines the containers that will be created from container profile. + template: + spec: + containers: + - name: sample-nnfcontainerprofile + image: alpine:latest + command: + - /bin/sh + - -c + - "sleep 15 && exit 0" diff --git a/controllers/directivebreakdown_controller.go b/controllers/directivebreakdown_controller.go index b7ad6a631..178e455ba 100644 --- a/controllers/directivebreakdown_controller.go +++ b/controllers/directivebreakdown_controller.go @@ -1,5 +1,5 @@ /* - * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP + * Copyright 2021-2023 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, diff --git a/controllers/nnf_workflow_controller.go b/controllers/nnf_workflow_controller.go index 48261e6d3..1970f2c1f 100644 --- a/controllers/nnf_workflow_controller.go +++ b/controllers/nnf_workflow_controller.go @@ -1,5 +1,5 @@ /* - * Copyright 2021, 2022 Hewlett Packard Enterprise Development LP + * Copyright 2021-2023 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, @@ -82,12 +82,12 @@ type NnfWorkflowReconciler struct { //+kubebuilder:rbac:groups=dws.cray.hpe.com,resources=computes,verbs=get;create;list;watch;update;patch //+kubebuilder:rbac:groups=cray.hpe.com,resources=lustrefilesystems,verbs=get;list;watch +//+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfcontainerprofiles,verbs=get;create;list;watch;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;deletecollection +//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;deletecollection + // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Workflow object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.9.2/pkg/reconcile @@ -218,6 +218,7 @@ func (r *NnfWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request) if result != nil { log.Info("Start wait", result.info()...) + driverStatus.Message = result.reason return result.Result, nil } @@ -256,6 +257,7 @@ func (r *NnfWorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request) if result != nil { log.Info("Finish wait", result.info()...) + driverStatus.Message = result.reason return result.Result, nil } @@ -282,7 +284,12 @@ func (r *NnfWorkflowReconciler) startProposalState(ctx context.Context, workflow } // only jobdw, persistentdw, and create_persistent need a directive breakdown - if dwArgs["command"] != "jobdw" && dwArgs["command"] != "persistentdw" && dwArgs["command"] != "create_persistent" { + switch dwArgs["command"] { + case "container": + return nil, r.createPinnedContainerProfileIfNecessary(ctx, workflow, index) + case "jobdw", "persistentdw", "create_persistent": + break + default: return nil, nil } @@ -319,8 +326,11 @@ func (r *NnfWorkflowReconciler) finishProposalState(ctx context.Context, workflo log := r.Log.WithValues("Workflow", client.ObjectKeyFromObject(workflow), "Index", index) dwArgs, _ := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) - // only jobdw, persistentdw, and create_persistent need a directive breakdown - if dwArgs["command"] != "jobdw" && dwArgs["command"] != "persistentdw" && dwArgs["command"] != "create_persistent" { + // only jobdw, persistentdw, and create_persistent have a directive breakdown + switch dwArgs["command"] { + case "jobdw", "persistentdw", "create_persistent": + break + default: return nil, nil } @@ -353,56 +363,54 @@ func (r *NnfWorkflowReconciler) startSetupState(ctx context.Context, workflow *d log := r.Log.WithValues("Workflow", client.ObjectKeyFromObject(workflow), "Index", index) dwArgs, _ := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) - if dwArgs["command"] == "persistentdw" { - return nil, r.addPersistentStorageReference(ctx, workflow, index) - } - - // Only jobdw and create_persistent need to create an NnfStorage resource - if dwArgs["command"] != "create_persistent" && dwArgs["command"] != "jobdw" { + switch dwArgs["command"] { + default: return nil, nil - } - - // Chain through the DirectiveBreakdown to the Servers object - dbd := &dwsv1alpha1.DirectiveBreakdown{ - ObjectMeta: metav1.ObjectMeta{ - Name: indexedResourceName(workflow, index), - Namespace: workflow.Namespace, - }, - } - err := r.Get(ctx, client.ObjectKeyFromObject(dbd), dbd) - if err != nil { - log.Info("Unable to get directiveBreakdown", "dbd", client.ObjectKeyFromObject(dbd), "Message", err) - err = fmt.Errorf("Unable to get DirectiveBreakdown %v: %w", client.ObjectKeyFromObject(dbd), err) - return nil, nnfv1alpha1.NewWorkflowError("Could not read allocation request").WithError(err) - } - - s := &dwsv1alpha1.Servers{ - ObjectMeta: metav1.ObjectMeta{ - Name: dbd.Status.Storage.Reference.Name, - Namespace: dbd.Status.Storage.Reference.Namespace, - }, - } - err = r.Get(ctx, client.ObjectKeyFromObject(s), s) - if err != nil { - log.Info("Unable to get servers", "servers", client.ObjectKeyFromObject(s), "Message", err) - err = fmt.Errorf("Unable to get Servers %v: %w", client.ObjectKeyFromObject(s), err) - return nil, nnfv1alpha1.NewWorkflowError("Could not read allocation request").WithError(err) - } + case "persistentdw": + return nil, r.addPersistentStorageReference(ctx, workflow, index) + case "jobdw", "create_persistent": + // Chain through the DirectiveBreakdown to the Servers object + dbd := &dwsv1alpha1.DirectiveBreakdown{ + ObjectMeta: metav1.ObjectMeta{ + Name: indexedResourceName(workflow, index), + Namespace: workflow.Namespace, + }, + } + err := r.Get(ctx, client.ObjectKeyFromObject(dbd), dbd) + if err != nil { + log.Info("Unable to get directiveBreakdown", "dbd", client.ObjectKeyFromObject(dbd), "Message", err) + err = fmt.Errorf("Unable to get DirectiveBreakdown %v: %w", client.ObjectKeyFromObject(dbd), err) + return nil, nnfv1alpha1.NewWorkflowError("Could not read allocation request").WithError(err) + } - if _, present := os.LookupEnv("RABBIT_TEST_ENV_BYPASS_SERVER_STORAGE_CHECK"); !present { - if err := r.validateServerAllocations(ctx, dbd, s); err != nil { - return nil, err + s := &dwsv1alpha1.Servers{ + ObjectMeta: metav1.ObjectMeta{ + Name: dbd.Status.Storage.Reference.Name, + Namespace: dbd.Status.Storage.Reference.Namespace, + }, + } + err = r.Get(ctx, client.ObjectKeyFromObject(s), s) + if err != nil { + log.Info("Unable to get servers", "servers", client.ObjectKeyFromObject(s), "Message", err) + err = fmt.Errorf("Unable to get Servers %v: %w", client.ObjectKeyFromObject(s), err) + return nil, nnfv1alpha1.NewWorkflowError("Could not read allocation request").WithError(err) } - } - if storage, err := r.createNnfStorage(ctx, workflow, s, index, log); err != nil { - if apierrors.IsConflict(err) { - return Requeue("conflict").withObject(storage), nil + if _, present := os.LookupEnv("RABBIT_TEST_ENV_BYPASS_SERVER_STORAGE_CHECK"); !present { + if err := r.validateServerAllocations(ctx, dbd, s); err != nil { + return nil, err + } } - log.Info("Failed to create nnf storage", "Message", err) - err = fmt.Errorf("Could not create NnfStorage %w", err) - return nil, nnfv1alpha1.NewWorkflowError("Could not create allocation").WithError(err) + if storage, err := r.createNnfStorage(ctx, workflow, s, index, log); err != nil { + if apierrors.IsConflict(err) { + return Requeue("conflict").withObject(storage), nil + } + + log.Info("Failed to create nnf storage", "Message", err) + err = fmt.Errorf("Could not create NnfStorage %w", err) + return nil, nnfv1alpha1.NewWorkflowError("Could not create allocation").WithError(err) + } } return nil, nil @@ -419,8 +427,7 @@ func (r *NnfWorkflowReconciler) finishSetupState(ctx context.Context, workflow * }, } if err := r.Get(ctx, client.ObjectKeyFromObject(nnfStorage), nnfStorage); err != nil { - err = fmt.Errorf("Could not get NnfStorage %v: %w", client.ObjectKeyFromObject(nnfStorage), err) - return nil, nnfv1alpha1.NewWorkflowError("Could not create allocation").WithError(err) + return nil, nnfv1alpha1.NewWorkflowErrorf("failed to get NNF storage resource '%s", client.ObjectKeyFromObject(nnfStorage)).WithError(err) } // If the Status section has not been filled in yet, exit and wait. @@ -430,7 +437,7 @@ func (r *NnfWorkflowReconciler) finishSetupState(ctx context.Context, workflow * } if nnfStorage.Status.Error != nil { - return nil, nnfv1alpha1.NewWorkflowError("Could not create allocation").WithError(nnfStorage.Status.Error) + return nil, nnfv1alpha1.NewWorkflowErrorf("storage resource '%s' has error", client.ObjectKeyFromObject(nnfStorage)).WithError(nnfStorage.Status.Error) } if nnfStorage.Status.Status != nnfv1alpha1.ResourceReady { @@ -762,6 +769,18 @@ func (r *NnfWorkflowReconciler) startPreRunState(ctx context.Context, workflow * }, } + // Create container service and jobs + if dwArgs["command"] == "container" { + if err := r.createContainerService(ctx, workflow); err != nil { + return nil, nnfv1alpha1.NewWorkflowError("Unable to create/update Container Service").WithFatal().WithError(err) + } + if err := r.createContainerJobs(ctx, workflow, dwArgs, index); err != nil { + return nil, nnfv1alpha1.NewWorkflowError("Unable to create/update Container Jobs").WithFatal().WithError(err) + } + + return nil, nil + } + // Create an NNFAccess for the compute clients result, err := ctrl.CreateOrUpdate(ctx, r.Client, access, func() error { @@ -826,11 +845,11 @@ func (r *NnfWorkflowReconciler) startPreRunState(ctx context.Context, workflow * }, } - // Set the teardown state to post run. If there is a copy_out directive that uses - // this storage instance, set the teardown state so NNF Access is preserved through - // DataOut. + // Set the teardown state to post run. If there is a copy_out or container directive that + // uses this storage instance, set the teardown state so NNF Access is preserved up until + // Teardown teardownState := dwsv1alpha1.StatePostRun - if findCopyOutDirectiveIndexByName(workflow, dwArgs["name"]) >= 0 { + if findCopyOutDirectiveIndexByName(workflow, dwArgs["name"]) >= 0 || findContainerDirectiveIndexByName(workflow, dwArgs["name"]) >= 0 { teardownState = dwsv1alpha1.StateTeardown } @@ -845,13 +864,6 @@ func (r *NnfWorkflowReconciler) startPreRunState(ctx context.Context, workflow * func (r *NnfWorkflowReconciler) finishPreRunState(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*result, error) { - result, err := r.waitForNnfAccessStateAndReady(ctx, workflow, index, "mounted") - if err != nil { - return nil, nnfv1alpha1.NewWorkflowError("Failed to achieve NnfAccess 'mounted' state").WithError(err).WithFatal() - } else if result != nil { - return result, nil - } - dwArgs, _ := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) // Add an environment variable to the workflow status section for the location of the @@ -866,17 +878,33 @@ func (r *NnfWorkflowReconciler) finishPreRunState(ctx context.Context, workflow envName = "DW_JOB_" + dwArgs["name"] case "persistentdw": envName = "DW_PERSISTENT_" + dwArgs["name"] + case "container": + return r.waitForContainersToStart(ctx, workflow, index) default: return nil, nnfv1alpha1.NewWorkflowErrorf("Unexpected directive %v", dwArgs["command"]) } workflow.Status.Env[envName] = buildMountPath(workflow, index) + // Containers do not have NNFAccesses, so only do this after r.waitForContainersToStart() would have returned + result, err := r.waitForNnfAccessStateAndReady(ctx, workflow, index, "mounted") + if err != nil { + return nil, nnfv1alpha1.NewWorkflowError("Failed to achieve NnfAccess 'mounted' state").WithError(err).WithFatal() + } else if result != nil { + return result, nil + } + return nil, nil } func (r *NnfWorkflowReconciler) startPostRunState(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*result, error) { + dwArgs, _ := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) + + if dwArgs["command"] == "container" { + return r.waitForContainersToFinish(ctx, workflow, index) + } + // Unmount the NnfAccess for the compute nodes. This will free the compute nodes to be used // in a different job even if there is data movement happening on the Rabbits. if result, err := r.unmountNnfAccessIfNecessary(ctx, workflow, index, "computes"); result != nil || err != nil { @@ -914,6 +942,11 @@ func (r *NnfWorkflowReconciler) startPostRunState(ctx context.Context, workflow } func (r *NnfWorkflowReconciler) finishPostRunState(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*result, error) { + dwArgs, _ := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) + + if dwArgs["command"] == "container" { + return r.checkContainersResults(ctx, workflow, index) + } result, err := r.waitForNnfAccessStateAndReady(ctx, workflow, index, "unmounted") if err != nil { diff --git a/controllers/nnf_workflow_controller_helpers.go b/controllers/nnf_workflow_controller_helpers.go index 978e61d26..e0ef65ac9 100644 --- a/controllers/nnf_workflow_controller_helpers.go +++ b/controllers/nnf_workflow_controller_helpers.go @@ -1,8 +1,28 @@ +/* + * Copyright 2022-2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package controllers import ( "context" "fmt" + "os" "reflect" "strconv" "strings" @@ -14,10 +34,12 @@ import ( nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" "github.com/go-logr/logr" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -32,6 +54,16 @@ type result struct { deleteStatus *dwsv1alpha1.DeleteStatus } +// This struct contains all the necessary information for mounting container storages +type nnfContainerVolume struct { + name string + command string + directiveName string + directiveIndex int + mountPath string + envVarName string +} + // When workflow stages cannot advance they return a Requeue result with a particular reason. func Requeue(reason string) *result { return &result{Result: ctrl.Result{}, reason: reason} @@ -81,8 +113,8 @@ func (r *result) info() []interface{} { // Validate the workflow and return any error found func (r *NnfWorkflowReconciler) validateWorkflow(ctx context.Context, wf *dwsv1alpha1.Workflow) error { - var createPersistentCount, deletePersistentCount, directiveCount int - for _, directive := range wf.Spec.DWDirectives { + var createPersistentCount, deletePersistentCount, directiveCount, containerCount int + for index, directive := range wf.Spec.DWDirectives { directiveCount++ // The webhook validated directives, ignore errors @@ -107,6 +139,13 @@ func (r *NnfWorkflowReconciler) validateWorkflow(ctx context.Context, wf *dwsv1a if err := r.validatePersistentInstanceDirective(ctx, wf, directive); err != nil { return nnfv1alpha1.NewWorkflowError("Could not validate persistent instance: " + directive).WithFatal().WithError(err) } + + case "container": + containerCount++ + + if err := r.validateContainerDirective(ctx, wf, index); err != nil { + return nnfv1alpha1.NewWorkflowError("Could not validate container directive: " + directive).WithFatal().WithError(err) + } } } @@ -115,6 +154,11 @@ func (r *NnfWorkflowReconciler) validateWorkflow(ctx context.Context, wf *dwsv1a if createPersistentCount+deletePersistentCount > 0 { return nnfv1alpha1.NewWorkflowError("Only a single create_persistent or destroy_persistent directive is allowed per workflow").WithFatal() } + + // Only allow 1 container directive (for now) + if containerCount > 1 { + return nnfv1alpha1.NewWorkflowError("Only a single container directive is supported per workflow").WithFatal() + } } return nil @@ -160,7 +204,7 @@ func (r *NnfWorkflowReconciler) validateStagingDirective(ctx context.Context, wf } } else { if r.findLustreFileSystemForPath(ctx, arg, r.Log) == nil { - return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("Global Lustre file system containing '%s' not found", arg)).WithFatal() + return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("global Lustre file system containing '%s' not found", arg)).WithFatal() } } @@ -183,6 +227,93 @@ func (r *NnfWorkflowReconciler) validateStagingDirective(ctx context.Context, wf return nil } +// validateContainerDirective validates the container directive. +func (r *NnfWorkflowReconciler) validateContainerDirective(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) error { + args, err := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) + if err != nil { + return nnfv1alpha1.NewWorkflowError("invalid DW directive: " + workflow.Spec.DWDirectives[index]).WithFatal() + } + + // Ensure the supplied profile exists or use the default + profile, err := r.findContainerProfile(ctx, workflow, index) + if err != nil { + return nnfv1alpha1.NewWorkflowError(err.Error()).WithFatal() + } + + // Check to see if the container storage argument is in the list of storages in the container profile + checkStorageIsInProfile := func(storageName string) error { + for _, storage := range profile.Data.Storages { + if storage.Name == storageName { + return nil + } + } + return fmt.Errorf("storage '%s' not found in container profile '%s'", storageName, profile.Name) + } + + // Find the wildcard DW_JOB and DW_PERSISTENT arguments. Verify that they exist in the + // preceeding directives and in the container profile. For persistent arguments, ensure there + // is a persistent storage instance. + var suppliedStorageArguments []string // use this list later to validate non-optional storages + for arg, storageName := range args { + switch arg { + case "command", "name", "profile": + default: + // TODO: restrict access to xfs and raw filesystem types since they cannot be mounted on both the rabbit and compute nodejk + if strings.HasPrefix(arg, "DW_JOB_") { + if findDirectiveIndexByName(workflow, storageName, "jobdw") == -1 { + return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("jobdw directive mentioning '%s' not found", storageName)).WithFatal() + } + if err := checkStorageIsInProfile(arg); err != nil { + return nnfv1alpha1.NewWorkflowError(err.Error()).WithFatal() + } + suppliedStorageArguments = append(suppliedStorageArguments, arg) + } else if strings.HasPrefix(arg, "DW_PERSISTENT_") { + if err := r.validatePersistentInstanceForStaging(ctx, storageName, workflow.Namespace); err != nil { + return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("persistent storage instance '%s' not found", storageName)).WithFatal() + } + if findDirectiveIndexByName(workflow, storageName, "persistentdw") == -1 { + return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("persistentdw directive mentioning '%s' not found", storageName)).WithFatal() + } + if err := checkStorageIsInProfile(arg); err != nil { + return nnfv1alpha1.NewWorkflowError(err.Error()).WithFatal() + } + suppliedStorageArguments = append(suppliedStorageArguments, arg) + } else { + return nnfv1alpha1.NewWorkflowError(fmt.Sprintf("unrecognized container argument: %s", arg)).WithFatal() + } + } + } + + // Ensure that any storage in the profile that is non-optional is present in the supplied storage arguments + checkNonOptionalStorages := func(storageArguments []string) error { + findInStorageArguments := func(name string) bool { + for _, directive := range storageArguments { + if name == directive { + return true + } + } + return false + } + + for _, storage := range profile.Data.Storages { + if !storage.Optional { + if !findInStorageArguments(storage.Name) { + return fmt.Errorf("storage '%s' in container profile '%s' is not optional: storage argument not found in the supplied arguments", + storage.Name, profile.Name) + } + } + } + + return nil + } + + if err := checkNonOptionalStorages(suppliedStorageArguments); err != nil { + return nnfv1alpha1.NewWorkflowError(err.Error()).WithFatal() + } + + return nil +} + // validatePersistentInstance validates the persistentdw directive. func (r *NnfWorkflowReconciler) validatePersistentInstanceForStaging(ctx context.Context, name string, namespace string) error { psi, err := r.getPersistentStorageInstance(ctx, name, namespace) @@ -243,6 +374,16 @@ func (r *NnfWorkflowReconciler) generateDirectiveBreakdown(ctx context.Context, // "#DW jobdw type=lustre capacity=9TB name=thisIsLustre" // "#DW create_persistent type=lustre capacity=9TB name=thisIsPersistent" // "#DW persistentdw name=thisIsPersistent" + // + // Container directives need to be accompanied by jobdw and/or persistentdw directives prior + // to the container directive. The names of those jobdw/persistentdw directives must match + // what is then supplied to the container directive as key value arguments. + // + // #DW jobdw name=my-gfs2 type=gfs2 capacity=1TB + // #DW persistentdw name=some-lustre + // #DW container name=my-foo profile=foo + // DW_JOB_foo-local-storage=my-gfs2 + // DW_PERSISTENT_foo-persistent-storage=some-lustre // #DW commands that require a dwDirectiveBreakdown breakDownCommands := []string{ @@ -355,7 +496,7 @@ func (r *NnfWorkflowReconciler) validateServerAllocations(ctx context.Context, d } } - if found == false { + if !found { err := fmt.Errorf("Allocation set %s not found in Servers resource", breakdownAllocationSet.Label) return nnfv1alpha1.NewWorkflowError("Allocation request does not meet directive requirements").WithFatal().WithError(err) } @@ -612,6 +753,25 @@ func findCopyOutDirectiveIndexByName(workflow *dwsv1alpha1.Workflow, name string return -1 } +// Returns the directive index matching the container directive whose storage field(s) reference +// the provided name argument, or -1 if not found. +func findContainerDirectiveIndexByName(workflow *dwsv1alpha1.Workflow, name string) int { + for idx, directive := range workflow.Spec.DWDirectives { + parameters, _ := dwdparse.BuildArgsMap(directive) // ignore error, directives are validated in proposal + if parameters["command"] == "container" { + for k, v := range parameters { + if strings.HasPrefix(k, "DW_JOB_") || strings.HasPrefix(k, "DW_PERSISTENT_") { + if v == name { + return idx + } + } + } + } + } + + return -1 +} + // Returns a pair for the given staging argument (typically source or destination) // i.e. $DW_JOB_my-file-system-name/path/to/a/file into "my-file-system-name" and "/path/to/a/file" func splitStagingArgumentIntoNameAndPath(arg string) (string, string) { @@ -849,3 +1009,484 @@ func (r *NnfWorkflowReconciler) removeAllPersistentStorageReferences(ctx context return nil } + +func (r *NnfWorkflowReconciler) createContainerService(ctx context.Context, workflow *dwsv1alpha1.Workflow) error { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: workflow.Name, + Namespace: workflow.Namespace, + }, + } + + service.Spec.Selector = map[string]string{ + nnfv1alpha1.ContainerLabel: workflow.Name, + } + + service.Spec.ClusterIP = corev1.ClusterIPNone + + if err := ctrl.SetControllerReference(workflow, service, r.Scheme); err != nil { + return fmt.Errorf("setting Service controller reference failed for '%s': %w", service.Name, err) + } + + err := r.Create(ctx, service) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + } + + return nil +} + +func (r *NnfWorkflowReconciler) createContainerJobs(ctx context.Context, workflow *dwsv1alpha1.Workflow, dwArgs map[string]string, index int) error { + profile, err := r.getContainerProfile(ctx, workflow, index) + if err != nil { + return err + } + + // Create one master job that will be used for all the jobs on all the NnfNodes. Most of the Job's Data will be the same. + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: workflow.Namespace, + }, + } + + // Apply Job Labels/Owners + dwsv1alpha1.InheritParentLabels(job, workflow) + dwsv1alpha1.AddOwnerLabels(job, workflow) + dwsv1alpha1.AddWorkflowLabels(job, workflow) + + labels := job.GetLabels() + labels[nnfv1alpha1.ContainerLabel] = workflow.Name + labels[nnfv1alpha1.PinnedContainerProfileLabelName] = profile.GetName() + labels[nnfv1alpha1.PinnedContainerProfileLabelNameSpace] = profile.GetNamespace() + labels[nnfv1alpha1.DirectiveIndexLabel] = strconv.Itoa(index) + job.SetLabels(labels) + + if err := ctrl.SetControllerReference(workflow, job, r.Scheme); err != nil { + return nnfv1alpha1.NewWorkflowErrorf("setting Job controller reference failed for '%s':", job.Name).WithError(err) + } + + // This defaults to 6 and is the maximum number of pod retries before considering the + // job failed. I don't believe this can be turned off. + // See the comments below regarding the restartPolicy and also see + // https://github.com/NearNodeFlash/NearNodeFlash.github.io/pull/26#discussion_r1089460308. + job.Spec.BackoffLimit = &profile.Data.RetryLimit + + // Copy the container template from the profile + profile.Data.Template.DeepCopyInto(&job.Spec.Template) + + // Use the same labels as the job for the pods + job.Spec.Template.Labels = job.DeepCopy().Labels + + podSpec := &job.Spec.Template.Spec + + // We want to keep the restart policy to Never. This way, any pods that failed are + // kept around for inspection. The job attempts to retry pods until the number of + // completions are hit or the number of max retries (BackoffLimit) have been hit. + // A retry with a restart policy of Never will not restart the pod, but spin up a new one + // with a new IP (DNS will have to be used to reach pods). A retry is not the same as a restart. + // If we set this to OnFailure, the pods will truly restart but we will lose any log history + // outside of (kubectl logs --previous). + // See https://github.com/NearNodeFlash/NearNodeFlash.github.io/pull/26#discussion_r1089460308 + podSpec.RestartPolicy = corev1.RestartPolicyNever + + podSpec.Subdomain = workflow.Name // service name == workflow name + + podSpec.Tolerations = []corev1.Toleration{ + { + Effect: "NoSchedule", + Key: "cray.nnf.node", + Operator: "Equal", + Value: "true", + }, + } + + // Get the volumes to mount into the containers + volumes, err := r.getContainerVolumes(ctx, workflow, dwArgs) + if err != nil { + return nnfv1alpha1.NewWorkflowErrorf("could not determine the list of volumes need to create container job %s:", job.Name).WithError(err).WithFatal() + } + + // Add Volumes/VolumeMounts + for _, vol := range volumes { + // Volumes + hostPathType := corev1.HostPathDirectory + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: vol.name, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: vol.mountPath, + Type: &hostPathType, + }, + }, + }) + + // Add VolumeMounts and Volume environment variables for all containers + for idx := range podSpec.Containers { + container := &podSpec.Containers[idx] + + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: vol.name, + MountPath: vol.mountPath, + }) + + container.Env = append(container.Env, corev1.EnvVar{ + Name: vol.envVarName, + Value: vol.mountPath, + }) + } + } + + // Get the targeted NNF nodes for the container jobs + nnfNodes, err := r.getNnfNodesFromComputes(ctx, workflow) + if err != nil { + return nnfv1alpha1.NewWorkflowError("error obtaining the target NNF nodes for containers:").WithError(err).WithFatal() + } + + // Add in non-volume environment variables for all containers + for idx := range podSpec.Containers { + container := &podSpec.Containers[idx] + + container.Env = append(container.Env, + corev1.EnvVar{Name: "NNF_CONTAINER_SUBDOMAIN", Value: podSpec.Subdomain}, + corev1.EnvVar{Name: "NNF_CONTAINER_DOMAIN", Value: workflow.Namespace + ".svc.cluster.local"}, + corev1.EnvVar{Name: "NNF_CONTAINER_HOSTNAMES", Value: strings.Join(nnfNodes, " ")}) + } + + // Finally, create a job for each nnfNode. Only the name, hostname, and node selector is different for each node + for _, nnfNode := range nnfNodes { + job.ObjectMeta.Name = workflow.Name + "-" + nnfNode + podSpec.Hostname = nnfNode + + // In our case, the target is only 1 node for the job, so a restartPolicy of Never + // is ok because any retry (i.e. new pod) will land on the same node. + podSpec.NodeSelector = map[string]string{"kubernetes.io/hostname": nnfNode} + + err = r.Create(ctx, job) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + } + } + + return nil +} + +// Retrieve the computes for the workflow and find their local nnf nodes +func (r *NnfWorkflowReconciler) getNnfNodesFromComputes(ctx context.Context, workflow *dwsv1alpha1.Workflow) ([]string, error) { + + var nnfNodes []string + var computeNodes []string + + // Get the compute resources + computes := dwsv1alpha1.Computes{ + ObjectMeta: metav1.ObjectMeta{ + Name: workflow.Name, + Namespace: workflow.Namespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(&computes), &computes); err != nil { + return nnfNodes, nnfv1alpha1.NewWorkflowError("could not find Computes resource for workflow") + } + + // Build the list of computes + for _, c := range computes.Data { + computeNodes = append(computeNodes, c.Name) + } + if len(computeNodes) == 0 { + return computeNodes, nnfv1alpha1.NewWorkflowError("the Computes resources does not specify any compute nodes") + } + + systemConfig := &dwsv1alpha1.SystemConfiguration{} + if err := r.Get(ctx, types.NamespacedName{Name: "default", Namespace: corev1.NamespaceDefault}, systemConfig); err != nil { + return nnfNodes, nnfv1alpha1.NewWorkflowError("could not get system configuration") + } + + // The SystemConfiguration is organized by rabbit. Make a map of computes:rabbit for easy lookup. + computeMap := make(map[string]string) + for _, storageNode := range systemConfig.Spec.StorageNodes { + if storageNode.Type != "Rabbit" { + continue + } + for _, ca := range storageNode.ComputesAccess { + computeMap[ca.Name] = storageNode.Name + } + } + + // For each supplied compute, use the map to lookup its local Rabbit + for _, c := range computeNodes { + nnfNode, found := computeMap[c] + if !found { + return nnfNodes, nnfv1alpha1.NewWorkflowErrorf("supplied compute node '%s' not found in SystemConfiguration", c) + } + nnfNodes = append(nnfNodes, nnfNode) + } + + return nnfNodes, nil +} + +func (r *NnfWorkflowReconciler) waitForContainersToStart(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*result, error) { + jobList, err := r.getContainerJobs(ctx, workflow, index) + if err != nil { + return nil, err + } + + for _, job := range jobList.Items { + // If we have any conditions, the job already finished + if len(job.Status.Conditions) > 0 { + continue + } + + // Ready should be non-zero to indicate the a pod is running for the job + if job.Status.Ready == nil || *job.Status.Ready < 1 { + return Requeue(fmt.Sprintf("pending container start for job '%s'", job.Name)).after(2 * time.Second), nil + } + } + + return nil, nil +} + +func (r *NnfWorkflowReconciler) waitForContainersToFinish(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*result, error) { + jobList, err := r.getContainerJobs(ctx, workflow, index) + if err != nil { + return nil, err + } + + if len(jobList.Items) < 1 { + return nil, nnfv1alpha1.NewWorkflowErrorf("no container jobs found for workflow '%s', index: %d", workflow.Name, index) + } + + // Retrieve the profile to extract the PostRun timeout + profile, err := r.getContainerProfile(ctx, workflow, index) + if err != nil { + return nil, err + } + timeout := time.Duration(profile.Data.PostRunTimeoutSeconds) * time.Second + + // Ensure all the jobs are done running before we check the conditions. + for _, job := range jobList.Items { + // Jobs will have conditions when finished + if len(job.Status.Conditions) <= 0 { + + // If desired, set the ActiveDeadline on the job to kill pods. Use the job's creation + // timestamp to determine how long the job/pod has been running at this point. Then, add + // the desired timeout to that value. k8s Job's ActiveDeadLineSeconds will then + // terminate the pods once the deadline is hit. + if timeout > 0 && job.Spec.ActiveDeadlineSeconds == nil { + deadline := int64((metav1.Now().Sub(job.CreationTimestamp.Time) + timeout).Seconds()) + + // Update the job with the deadline + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + j := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: job.Name, Namespace: job.Namespace}} + if err := r.Get(ctx, client.ObjectKeyFromObject(j), j); err != nil { + return client.IgnoreNotFound(err) + } + + j.Spec.ActiveDeadlineSeconds = &deadline + return r.Update(ctx, j) + }) + + if err != nil { + return nil, nnfv1alpha1.NewWorkflowErrorf("error updating job '%s' activeDeadlineSeconds:", job.Name) + } + } + + return Requeue("pending container finish").after(2 * time.Second).withObject(&job), nil + } + } + + return nil, nil +} + +func (r *NnfWorkflowReconciler) checkContainersResults(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*result, error) { + jobList, err := r.getContainerJobs(ctx, workflow, index) + if err != nil { + return nil, err + } + + if len(jobList.Items) < 1 { + return nil, nnfv1alpha1.NewWorkflowErrorf("no container jobs found for workflow '%s', index: %d", workflow.Name, index) + } + + for _, job := range jobList.Items { + for _, condition := range job.Status.Conditions { + if condition.Type != batchv1.JobComplete { + return nil, nnfv1alpha1.NewWorkflowErrorf("container job %s (%s): %s", condition.Type, condition.Reason, condition.Message) + } + } + } + + return nil, nil +} + +func (r *NnfWorkflowReconciler) getContainerJobs(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*batchv1.JobList, error) { + // Get the jobs for this workflow and directive index + matchLabels := dwsv1alpha1.MatchingWorkflow(workflow) + matchLabels[nnfv1alpha1.DirectiveIndexLabel] = strconv.Itoa(index) + + jobList := &batchv1.JobList{} + if err := r.List(ctx, jobList, matchLabels); err != nil { + return nil, nnfv1alpha1.NewWorkflowErrorf("could not retrieve Jobs for index %d", index).WithError(err) + } + + return jobList, nil +} + +func (r *NnfWorkflowReconciler) getContainerProfile(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*nnfv1alpha1.NnfContainerProfile, error) { + profile, err := r.findPinnedContainerProfile(ctx, workflow, index) + if err != nil { + return nil, err + } + + if profile == nil { + return nil, nnfv1alpha1.NewWorkflowErrorf("container profile '%s' not found", indexedResourceName(workflow, index)).WithFatal() + } + + if !profile.Data.Pinned { + return nil, nnfv1alpha1.NewWorkflowErrorf("expected pinned container profile '%s'", indexedResourceName(workflow, index)).WithFatal() + } + + return profile, nil +} + +func (r *NnfWorkflowReconciler) findPinnedContainerProfile(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*nnfv1alpha1.NnfContainerProfile, error) { + profile := &nnfv1alpha1.NnfContainerProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: indexedResourceName(workflow, index), + Namespace: workflow.Namespace, + }, + } + + if err := r.Get(ctx, client.ObjectKeyFromObject(profile), profile); err != nil { + return nil, err + } + + return profile, nil +} + +func (r *NnfWorkflowReconciler) findContainerProfile(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) (*nnfv1alpha1.NnfContainerProfile, error) { + args, err := dwdparse.BuildArgsMap(workflow.Spec.DWDirectives[index]) + if err != nil { + return nil, err + } + + name, found := args["profile"] + if !found { + return nil, fmt.Errorf("container directive '%s' has no profile key", workflow.Spec.DWDirectives[index]) + } + + profile := &nnfv1alpha1.NnfContainerProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: os.Getenv("NNF_CONTAINER_PROFILE_NAMESPACE"), + }, + } + + if err := r.Get(ctx, client.ObjectKeyFromObject(profile), profile); err != nil { + return nil, err + } + + return profile, nil +} + +func (r *NnfWorkflowReconciler) createPinnedContainerProfileIfNecessary(ctx context.Context, workflow *dwsv1alpha1.Workflow, index int) error { + profile, err := r.findPinnedContainerProfile(ctx, workflow, index) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + + if profile != nil { + if !profile.Data.Pinned { + return fmt.Errorf("expected pinned container profile, but it was not pinned: %s", profile.Name) + } + + return nil + } + + profile, err = r.findContainerProfile(ctx, workflow, index) + if err != nil { + return err + } + + pinnedProfile := &nnfv1alpha1.NnfContainerProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: indexedResourceName(workflow, index), + Namespace: workflow.Namespace, + }, + } + + profile.Data.DeepCopyInto(&pinnedProfile.Data) + + pinnedProfile.Data.Pinned = true + + dwsv1alpha1.AddOwnerLabels(pinnedProfile, workflow) + + if err := controllerutil.SetControllerReference(workflow, pinnedProfile, r.Scheme); err != nil { + r.Log.Error(err, "failed to set controller reference on profile", "profile", pinnedProfile) + return fmt.Errorf("failed to set controller reference on profile %s", client.ObjectKeyFromObject(pinnedProfile)) + } + + r.Log.Info("Creating pinned container profile", "resource", client.ObjectKeyFromObject(pinnedProfile)) + if err := r.Create(ctx, pinnedProfile); err != nil { + return err + } + + return nil +} + +// Create a list of volumes to be mounted inside of the containers based on the DW_JOB/DW_PERSISTENT arguments +func (r *NnfWorkflowReconciler) getContainerVolumes(ctx context.Context, workflow *dwsv1alpha1.Workflow, dwArgs map[string]string) ([]nnfContainerVolume, error) { + volumes := []nnfContainerVolume{} + + // TODO: ssh is necessary for mpi see setupSSHAuthVolumes(manager, podSpec) in nnf-dm + + for arg, val := range dwArgs { + volName, cmd := "", "" + + // Find any DW_(JOB|PERSISTENT) arguments + if strings.HasPrefix(arg, "DW_JOB_") { + volName = strings.TrimPrefix(arg, "DW_JOB_") + cmd = "jobdw" + } else if strings.HasPrefix(arg, "DW_PERSISTENT_") { + volName = strings.TrimPrefix(arg, "DW_PERSISTENT_") + cmd = "persistentdw" + } else { + continue + } + + vol := nnfContainerVolume{ + name: volName, + command: cmd, + directiveName: val, + directiveIndex: -1, + envVarName: strings.ReplaceAll(arg, "-", "_"), // env vars can't have hyphens + } + + // Find the directive index for the given name so we can retrieve its NnfAccess + vol.directiveIndex = findDirectiveIndexByName(workflow, vol.directiveName, vol.command) + if vol.directiveIndex < 0 { + return nil, nnfv1alpha1.NewWorkflowErrorf("could not retrieve the directive breakdown for '%s'", vol.directiveName) + } + + nnfAccess := &nnfv1alpha1.NnfAccess{ + ObjectMeta: metav1.ObjectMeta{ + Name: workflow.Name + "-" + strconv.Itoa(vol.directiveIndex) + "-servers", + Namespace: workflow.Namespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(nnfAccess), nnfAccess); err != nil { + return nil, nnfv1alpha1.NewWorkflowErrorf("could not retrieve the NnfAccess '%s'", nnfAccess.Name) + } + + if !nnfAccess.Status.Ready { + return nil, nnfv1alpha1.NewWorkflowErrorf("NnfAccess '%s' is not ready to be mounted into container", nnfAccess.Name) + } + + vol.mountPath = nnfAccess.Spec.MountPath + volumes = append(volumes, vol) + } + + return volumes, nil +} diff --git a/deploy.sh b/deploy.sh index e1bdece0d..be795cae8 100755 --- a/deploy.sh +++ b/deploy.sh @@ -32,18 +32,19 @@ then $KUSTOMIZE build config/$OVERLAY | kubectl apply -f - - echo "Waiting for the NnfStorageProfile webhook to become ready..." + echo "Waiting for the webhook to become ready..." while : do ready=$(kubectl get pods -n nnf-system -l control-plane=controller-manager --no-headers | awk '{print $2}') [[ $ready == "2/2" ]] && break sleep 1 done - kubectl apply -f config/samples/placeholder_nnfstorageprofile.yaml + + $KUSTOMIZE build config/examples | kubectl apply -f - fi if [[ $CMD == 'undeploy' ]] then - kubectl delete --ignore-not-found -f config/samples/placeholder_nnfstorageprofile.yaml - $KUSTOMIZE build config/$OVERLAY | kubectl delete --ignore-not-found -f - + $KUSTOMIZE build config/examples | kubectl delete --ignore-not-found -f - + $KUSTOMIZE build config/$OVERLAY | kubectl delete --ignore-not-found -f - fi diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index c8ed156a6..f7d37c0b6 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* - * Copyright 2022 Hewlett Packard Enterprise Development LP + * Copyright 2023 Hewlett Packard Enterprise Development LP * Other additional copyright holders may be indicated within. * * The entirety of this work is licensed under the Apache License, From f4c1ff652d0ca5b537ceda7f91ec93bdf45c6f90 Mon Sep 17 00:00:00 2001 From: Blake Devcich Date: Tue, 28 Feb 2023 16:03:05 -0600 Subject: [PATCH 34/71] Make sure example container profiles contain retryLimit Signed-off-by: Blake Devcich --- config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml b/config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml index 3b53bfa8f..a00a1cf81 100644 --- a/config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml +++ b/config/examples/nnf_v1alpha1_nnfcontainerprofiles.yaml @@ -3,6 +3,7 @@ kind: NnfContainerProfile metadata: name: example-success data: + retryLimit: 6 storages: - name: DW_JOB_foo-local-storage optional: false @@ -23,6 +24,7 @@ kind: NnfContainerProfile metadata: name: example-randomly-fail data: + retryLimit: 6 storages: - name: DW_JOB_foo-local-storage optional: false @@ -48,6 +50,7 @@ kind: NnfContainerProfile metadata: name: example-forever data: + retryLimit: 6 template: spec: containers: From 4b04f52198fdbfe597f0c15deff987c07b5d33a3 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Wed, 1 Mar 2023 11:06:47 -0600 Subject: [PATCH 35/71] NNF Port Manager (#163) NNF Port Manager infrastructure and tests --------- Signed-off-by: Nate Thornton --- PROJECT | 13 + api/v1alpha1/nnf_port_manager_types.go | 136 ++++++++ api/v1alpha1/zz_generated.deepcopy.go | 144 +++++++++ .../nnf.cray.hpe.com_nnfportmanagers.yaml | 222 +++++++++++++ config/crd/kustomization.yaml | 3 + .../cainjection_in_nnfportmanagers.yaml | 7 + .../patches/webhook_in_nnfportmanagers.yaml | 16 + config/manager/port_manager.yaml | 9 + config/rbac/nnfportmanager_editor_role.yaml | 31 ++ config/rbac/nnfportmanager_viewer_role.yaml | 27 ++ config/rbac/role.yaml | 26 ++ .../samples/nnf_v1alpha1_nnfportmanager.yaml | 12 + controllers/nnf_port_manager_controller.go | 296 ++++++++++++++++++ .../nnf_port_manager_controller_test.go | 184 +++++++++++ controllers/suite_test.go | 6 + go.mod | 2 +- go.sum | 4 +- main.go | 15 + ...types.go => system_configuration_types.go} | 7 + .../dws/api/v1alpha1/zz_generated.deepcopy.go | 6 + ...dws.cray.hpe.com_systemconfigurations.yaml | 12 + .../HewlettPackard/dws/utils/ports/ports.go | 148 +++++++++ vendor/modules.txt | 3 +- 23 files changed, 1325 insertions(+), 4 deletions(-) create mode 100644 api/v1alpha1/nnf_port_manager_types.go create mode 100644 config/crd/bases/nnf.cray.hpe.com_nnfportmanagers.yaml create mode 100644 config/crd/patches/cainjection_in_nnfportmanagers.yaml create mode 100644 config/crd/patches/webhook_in_nnfportmanagers.yaml create mode 100644 config/manager/port_manager.yaml create mode 100644 config/rbac/nnfportmanager_editor_role.yaml create mode 100644 config/rbac/nnfportmanager_viewer_role.yaml create mode 100644 config/samples/nnf_v1alpha1_nnfportmanager.yaml create mode 100644 controllers/nnf_port_manager_controller.go create mode 100644 controllers/nnf_port_manager_controller_test.go rename vendor/github.com/HewlettPackard/dws/api/v1alpha1/{systemconfiguration_types.go => system_configuration_types.go} (88%) create mode 100644 vendor/github.com/HewlettPackard/dws/utils/ports/ports.go diff --git a/PROJECT b/PROJECT index 693e8cc79..14402eaf2 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: cray.hpe.com layout: - go.kubebuilder.io/v3 @@ -113,4 +117,13 @@ resources: kind: NnfContainerProfile path: github.com/NearNodeFlash/nnf-sos/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cray.hpe.com + group: nnf + kind: NnfPortManager + path: github.com/NearNodeFlash/nnf-sos/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/nnf_port_manager_types.go b/api/v1alpha1/nnf_port_manager_types.go new file mode 100644 index 000000000..2b987799c --- /dev/null +++ b/api/v1alpha1/nnf_port_manager_types.go @@ -0,0 +1,136 @@ +/* + * Copyright 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1alpha1 + +import ( + "github.com/HewlettPackard/dws/utils/updater" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// NnfPortManagerAllocationSpec defines the desired state for a single port allocation +type NnfPortManagerAllocationSpec struct { + // Requester is an object reference to the requester of a ports. + Requester corev1.ObjectReference `json:"requester"` + + // Count is the number of desired ports the requester needs. The port manager + // will attempt to allocate this many ports. + // +kubebuilder:default:=1 + Count int `json:"count"` +} + +// NnfPortManagerSpec defines the desired state of NnfPortManager +type NnfPortManagerSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // SystemConfiguration is an object reference to the system configuration. The + // Port Manager will use the available ports defined in the system configuration. + SystemConfiguration corev1.ObjectReference `json:"systemConfiguration"` + + // Allocations is a list of allocation requests that the Port Manager will attempt + // to satisfy. To request port resources from the port manager, clients should add + // an entry to the allocations. Entries must be unique. The port manager controller + // will attempt to allocate port resources for each allocation specification in the + // list. To remove an allocation and free up port resources, remove the allocation + // from the list. + Allocations []NnfPortManagerAllocationSpec `json:"allocations"` +} + +// AllocationStatus is the current status of a port requestor. A port that is in use by the respective owner +// will have a status of "InUse". A port that is freed by the owner but not yet reclaimed by the port manager +// will have a status of "Free". Any other status value indicates a failure of the port allocation. +// +kubebuilder:validation:Enum:=InUse;Free;InvalidConfiguration;InsufficientResources +type NnfPortManagerAllocationStatusStatus string + +const ( + NnfPortManagerAllocationStatusInUse NnfPortManagerAllocationStatusStatus = "InUse" + NnfPortManagerAllocationStatusFree NnfPortManagerAllocationStatusStatus = "Free" + NnfPortManagerAllocationStatusInvalidConfiguration NnfPortManagerAllocationStatusStatus = "InvalidConfiguration" + NnfPortManagerAllocationStatusInsufficientResources NnfPortManagerAllocationStatusStatus = "InsufficientResources" + // NOTE: You must ensure any new value is added to the above kubebuilder validation enum +) + +// NnfPortManagerAllocationStatus defines the allocation status of a port for a given requester. +type NnfPortManagerAllocationStatus struct { + // Requester is an object reference to the requester of the port resource, if one exists, or + // empty otherwise. + Requester *corev1.ObjectReference `json:"requester,omitempty"` + + // Ports is list of ports allocated to the owning resource. + Ports []uint16 `json:"ports,omitempty"` + + // Status is the ownership status of the port. + Status NnfPortManagerAllocationStatusStatus `json:"status"` +} + +// PortManagerStatus is the current status of the port manager. +// +kubebuilder:validation:Enum:=Ready;SystemConfigurationNotFound +type NnfPortManagerStatusStatus string + +const ( + NnfPortManagerStatusReady NnfPortManagerStatusStatus = "Ready" + NnfPortManagerStatusSystemConfigurationNotFound NnfPortManagerStatusStatus = "SystemConfigurationNotFound" + // NOTE: You must ensure any new value is added in the above kubebuilder validation enum +) + +// NnfPortManagerStatus defines the observed state of NnfPortManager +type NnfPortManagerStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Allocations is a list of port allocation status'. + Allocations []NnfPortManagerAllocationStatus `json:"allocations,omitempty"` + + // Status is the current status of the port manager. + Status NnfPortManagerStatusStatus `json:"status"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// NnfPortManager is the Schema for the nnfportmanagers API +type NnfPortManager struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NnfPortManagerSpec `json:"spec,omitempty"` + Status NnfPortManagerStatus `json:"status,omitempty"` +} + +func (mgr *NnfPortManager) GetStatus() updater.Status[*NnfPortManagerStatus] { + return &mgr.Status +} + +//+kubebuilder:object:root=true + +// NnfPortManagerList contains a list of NnfPortManager +type NnfPortManagerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NnfPortManager `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NnfPortManager{}, &NnfPortManagerList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 49df07a38..69ff0d662 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -817,6 +818,149 @@ func (in *NnfNodeStorageStatus) DeepCopy() *NnfNodeStorageStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfPortManager) DeepCopyInto(out *NnfPortManager) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfPortManager. +func (in *NnfPortManager) DeepCopy() *NnfPortManager { + if in == nil { + return nil + } + out := new(NnfPortManager) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NnfPortManager) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfPortManagerAllocationSpec) DeepCopyInto(out *NnfPortManagerAllocationSpec) { + *out = *in + out.Requester = in.Requester +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfPortManagerAllocationSpec. +func (in *NnfPortManagerAllocationSpec) DeepCopy() *NnfPortManagerAllocationSpec { + if in == nil { + return nil + } + out := new(NnfPortManagerAllocationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfPortManagerAllocationStatus) DeepCopyInto(out *NnfPortManagerAllocationStatus) { + *out = *in + if in.Requester != nil { + in, out := &in.Requester, &out.Requester + *out = new(corev1.ObjectReference) + **out = **in + } + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]uint16, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfPortManagerAllocationStatus. +func (in *NnfPortManagerAllocationStatus) DeepCopy() *NnfPortManagerAllocationStatus { + if in == nil { + return nil + } + out := new(NnfPortManagerAllocationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfPortManagerList) DeepCopyInto(out *NnfPortManagerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NnfPortManager, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfPortManagerList. +func (in *NnfPortManagerList) DeepCopy() *NnfPortManagerList { + if in == nil { + return nil + } + out := new(NnfPortManagerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NnfPortManagerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfPortManagerSpec) DeepCopyInto(out *NnfPortManagerSpec) { + *out = *in + out.SystemConfiguration = in.SystemConfiguration + if in.Allocations != nil { + in, out := &in.Allocations, &out.Allocations + *out = make([]NnfPortManagerAllocationSpec, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfPortManagerSpec. +func (in *NnfPortManagerSpec) DeepCopy() *NnfPortManagerSpec { + if in == nil { + return nil + } + out := new(NnfPortManagerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NnfPortManagerStatus) DeepCopyInto(out *NnfPortManagerStatus) { + *out = *in + if in.Allocations != nil { + in, out := &in.Allocations, &out.Allocations + *out = make([]NnfPortManagerAllocationStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NnfPortManagerStatus. +func (in *NnfPortManagerStatus) DeepCopy() *NnfPortManagerStatus { + if in == nil { + return nil + } + out := new(NnfPortManagerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NnfResourceStatus) DeepCopyInto(out *NnfResourceStatus) { *out = *in diff --git a/config/crd/bases/nnf.cray.hpe.com_nnfportmanagers.yaml b/config/crd/bases/nnf.cray.hpe.com_nnfportmanagers.yaml new file mode 100644 index 000000000..83f47e65f --- /dev/null +++ b/config/crd/bases/nnf.cray.hpe.com_nnfportmanagers.yaml @@ -0,0 +1,222 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: nnfportmanagers.nnf.cray.hpe.com +spec: + group: nnf.cray.hpe.com + names: + kind: NnfPortManager + listKind: NnfPortManagerList + plural: nnfportmanagers + singular: nnfportmanager + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NnfPortManager is the Schema for the nnfportmanagers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NnfPortManagerSpec defines the desired state of NnfPortManager + properties: + allocations: + description: Allocations is a list of allocation requests that the + Port Manager will attempt to satisfy. To request port resources + from the port manager, clients should add an entry to the allocations. + Entries must be unique. The port manager controller will attempt + to allocate port resources for each allocation specification in + the list. To remove an allocation and free up port resources, remove + the allocation from the list. + items: + description: NnfPortManagerAllocationSpec defines the desired state + for a single port allocation + properties: + count: + default: 1 + description: Count is the number of desired ports the requester + needs. The port manager will attempt to allocate this many + ports. + type: integer + requester: + description: Requester is an object reference to the requester + of a ports. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - count + - requester + type: object + type: array + systemConfiguration: + description: SystemConfiguration is an object reference to the system + configuration. The Port Manager will use the available ports defined + in the system configuration. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - allocations + - systemConfiguration + type: object + status: + description: NnfPortManagerStatus defines the observed state of NnfPortManager + properties: + allocations: + description: Allocations is a list of port allocation status'. + items: + description: NnfPortManagerAllocationStatus defines the allocation + status of a port for a given requester. + properties: + ports: + description: Ports is list of ports allocated to the owning + resource. + items: + type: integer + type: array + requester: + description: Requester is an object reference to the requester + of the port resource, if one exists, or empty otherwise. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + status: + description: Status is the ownership status of the port. + enum: + - InUse + - Free + - InvalidConfiguration + - InsufficientResources + type: string + required: + - status + type: object + type: array + status: + description: Status is the current status of the port manager. + enum: + - Ready + - SystemConfigurationNotFound + type: string + required: + - status + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 56fe837f4..813540a4d 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -10,6 +10,7 @@ resources: - bases/nnf.cray.hpe.com_nnfstorageprofiles.yaml - bases/nnf.cray.hpe.com_nnfnodeecdata.yaml - bases/nnf.cray.hpe.com_nnfcontainerprofiles.yaml +- bases/nnf.cray.hpe.com_nnfportmanagers.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -23,6 +24,7 @@ patchesStrategicMerge: #- patches/webhook_in_nnfstorageprofiles.yaml #- patches/webhook_in_nnfnodeecdata.yaml #- patches/webhook_in_nnfcontainerprofiles.yaml +#- patches/webhook_in_nnfportmanagers.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. @@ -35,6 +37,7 @@ patchesStrategicMerge: #- patches/cainjection_in_nnfstorageprofiles.yaml #- patches/cainjection_in_nnfnodeecdata.yaml #- patches/cainjection_in_nnfcontainerprofiles.yaml +#- patches/cainjection_in_nnfportmanagers.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_nnfportmanagers.yaml b/config/crd/patches/cainjection_in_nnfportmanagers.yaml new file mode 100644 index 000000000..e20d83ef5 --- /dev/null +++ b/config/crd/patches/cainjection_in_nnfportmanagers.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: nnfportmanagers.nnf.cray.hpe.com diff --git a/config/crd/patches/webhook_in_nnfportmanagers.yaml b/config/crd/patches/webhook_in_nnfportmanagers.yaml new file mode 100644 index 000000000..289fc1da8 --- /dev/null +++ b/config/crd/patches/webhook_in_nnfportmanagers.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: nnfportmanagers.nnf.cray.hpe.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/manager/port_manager.yaml b/config/manager/port_manager.yaml new file mode 100644 index 000000000..2b7bcad12 --- /dev/null +++ b/config/manager/port_manager.yaml @@ -0,0 +1,9 @@ +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfPortManager +metadata: + name: port-manager +spec: + systemConfiguration: + name: default + namespace: default + allocations: [] \ No newline at end of file diff --git a/config/rbac/nnfportmanager_editor_role.yaml b/config/rbac/nnfportmanager_editor_role.yaml new file mode 100644 index 000000000..4992622b9 --- /dev/null +++ b/config/rbac/nnfportmanager_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit nnfportmanagers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: nnfportmanager-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: nnf-sos + app.kubernetes.io/part-of: nnf-sos + app.kubernetes.io/managed-by: kustomize + name: nnfportmanager-editor-role +rules: +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfportmanagers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfportmanagers/status + verbs: + - get diff --git a/config/rbac/nnfportmanager_viewer_role.yaml b/config/rbac/nnfportmanager_viewer_role.yaml new file mode 100644 index 000000000..d974ccd87 --- /dev/null +++ b/config/rbac/nnfportmanager_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view nnfportmanagers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: nnfportmanager-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: nnf-sos + app.kubernetes.io/part-of: nnf-sos + app.kubernetes.io/managed-by: kustomize + name: nnfportmanager-viewer-role +rules: +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfportmanagers + verbs: + - get + - list + - watch +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfportmanagers/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ad252ceb2..feaffca75 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -395,6 +395,32 @@ rules: - nnfnodestorages/finalizers verbs: - update +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfportmanagers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfportmanagers/finalizers + verbs: + - update +- apiGroups: + - nnf.cray.hpe.com + resources: + - nnfportmanagers/status + verbs: + - get + - patch + - update - apiGroups: - nnf.cray.hpe.com resources: diff --git a/config/samples/nnf_v1alpha1_nnfportmanager.yaml b/config/samples/nnf_v1alpha1_nnfportmanager.yaml new file mode 100644 index 000000000..8d344dfd0 --- /dev/null +++ b/config/samples/nnf_v1alpha1_nnfportmanager.yaml @@ -0,0 +1,12 @@ +apiVersion: nnf.cray.hpe.com/v1alpha1 +kind: NnfPortManager +metadata: + labels: + app.kubernetes.io/name: nnfportmanager + app.kubernetes.io/instance: nnfportmanager-sample + app.kubernetes.io/part-of: nnf-sos + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: nnf-sos + name: nnfportmanager-sample +spec: + # TODO(user): Add fields here diff --git a/controllers/nnf_port_manager_controller.go b/controllers/nnf_port_manager_controller.go new file mode 100644 index 000000000..7ac1fa925 --- /dev/null +++ b/controllers/nnf_port_manager_controller.go @@ -0,0 +1,296 @@ +/* + * Copyright 2022 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + portsutil "github.com/HewlettPackard/dws/utils/ports" + "github.com/HewlettPackard/dws/utils/updater" + "github.com/go-logr/logr" + + dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" + nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" +) + +// NnfPortManagerReconciler reconciles a NnfPortManager object +type NnfPortManagerReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// type aliases for name shortening +type AllocationSpec = nnfv1alpha1.NnfPortManagerAllocationSpec +type AllocationStatus = nnfv1alpha1.NnfPortManagerAllocationStatus + +//+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfportmanagers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfportmanagers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=nnf.cray.hpe.com,resources=nnfportmanagers/finalizers,verbs=update + +// System Configuration provides the +//+kubebuilder:rbac:groups=dws.cray.hpe.com,resources=systemconfigurations,verbs=get;list;watch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the NnfPortManager object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.14.1/pkg/reconcile +func (r *NnfPortManagerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) { + log := log.FromContext(ctx) + + mgr := &nnfv1alpha1.NnfPortManager{} + if err := r.Get(ctx, req.NamespacedName, mgr); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Create a resource status updater to ensure the status subresource is updated. + statusUpdater := updater.NewStatusUpdater[*nnfv1alpha1.NnfPortManagerStatus](mgr) + defer func() { err = statusUpdater.CloseWithStatusUpdate(ctx, r, err) }() + + // Read in the system configuration which contains the available ports. + config := &dwsv1alpha1.SystemConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: mgr.Spec.SystemConfiguration.Name, + Namespace: mgr.Spec.SystemConfiguration.Namespace, + }, + } + + mgr.Status.Status = nnfv1alpha1.NnfPortManagerStatusReady + if err := r.Get(ctx, client.ObjectKeyFromObject(config), config); err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + log.Info("System Configuration not found", "config", client.ObjectKeyFromObject(config).String()) + mgr.Status.Status = nnfv1alpha1.NnfPortManagerStatusSystemConfigurationNotFound + res = ctrl.Result{Requeue: true} // Force a requeue - we want the manager to go ready even if there are zero allocations + } + + // Free any unused allocations + r.cleanupUnusedAllocations(log, mgr) + + // For each "requester" in the mgr.Spec.Allocations, try to satisfy the request by + // allocating the desired ports. + for _, spec := range mgr.Spec.Allocations { + + // If the specification is already included in the allocations, continue + if r.isAllocated(mgr, spec) { + continue + } + + var ports []uint16 + var status nnfv1alpha1.NnfPortManagerAllocationStatusStatus + if mgr.Status.Status != nnfv1alpha1.NnfPortManagerStatusReady { + ports, status = nil, nnfv1alpha1.NnfPortManagerAllocationStatusInvalidConfiguration + } else { + ports, status = r.findFreePorts(log, mgr, config, spec) + } + + log.Info("Allocation", "requester", spec.Requester, "count", spec.Count, "ports", ports, "status", status) + allocationStatus := AllocationStatus{ + Requester: &corev1.ObjectReference{}, + Ports: ports, + Status: status, + } + + spec.Requester.DeepCopyInto(allocationStatus.Requester) + + if mgr.Status.Allocations == nil { + mgr.Status.Allocations = make([]nnfv1alpha1.NnfPortManagerAllocationStatus, 0) + } + + mgr.Status.Allocations = append(mgr.Status.Allocations, allocationStatus) + } + + return res, nil +} + +// isAllocationNeeded returns true if the provided Port Allocation Status has a matching value +// requester in the specification, and false otherwise. +func (r *NnfPortManagerReconciler) isAllocationNeeded(mgr *nnfv1alpha1.NnfPortManager, status *AllocationStatus) bool { + if status.Status != nnfv1alpha1.NnfPortManagerAllocationStatusInUse { + return false + } + + if status.Requester == nil { + return false + } + + for _, desired := range mgr.Spec.Allocations { + if *status.Requester == desired.Requester { + return true + } + } + + return false +} + +func (r *NnfPortManagerReconciler) cleanupUnusedAllocations(log logr.Logger, mgr *nnfv1alpha1.NnfPortManager) { + + // Free unused allocations. This will check if the Status.Allocations exist in + // the list of desired allocations in the Spec field and mark any unused allocations + // as freed. + failedIndices := make([]int, 0) + for idx := range mgr.Status.Allocations { + status := &mgr.Status.Allocations[idx] + + if !r.isAllocationNeeded(mgr, status) { + log.Info("Allocation unused", "requester", status.Requester, "status", status.Status) + if status.Status == nnfv1alpha1.NnfPortManagerAllocationStatusInUse { + status.Requester = nil + status.Status = nnfv1alpha1.NnfPortManagerAllocationStatusFree + } else if status.Status != nnfv1alpha1.NnfPortManagerAllocationStatusFree { + failedIndices = append(failedIndices, idx) + } + } + } + + for idx := range failedIndices { + failedIdx := failedIndices[len(failedIndices)-1-idx] // remove in reverse order + mgr.Status.Allocations = append(mgr.Status.Allocations[:failedIdx], mgr.Status.Allocations[failedIdx+1:]...) + } +} + +func (r *NnfPortManagerReconciler) findAllocationStatus(mgr *nnfv1alpha1.NnfPortManager, spec AllocationSpec) *AllocationStatus { + for idx := range mgr.Status.Allocations { + status := &mgr.Status.Allocations[idx] + if status.Requester == nil { + continue + } + + if *status.Requester == spec.Requester { + return status + } + } + + return nil +} + +// isAllocated returns true if the provided specification is in the Port Manager's allocation +// status', and false otherwise. +func (r *NnfPortManagerReconciler) isAllocated(mgr *nnfv1alpha1.NnfPortManager, spec AllocationSpec) bool { + return r.findAllocationStatus(mgr, spec) != nil +} + +// Find free ports to satisfy the provided specification. +func (r *NnfPortManagerReconciler) findFreePorts(log logr.Logger, mgr *nnfv1alpha1.NnfPortManager, config *dwsv1alpha1.SystemConfiguration, spec AllocationSpec) ([]uint16, nnfv1alpha1.NnfPortManagerAllocationStatusStatus) { + + portsInUse := make([]uint16, 0) + for _, status := range mgr.Status.Allocations { + if status.Status == nnfv1alpha1.NnfPortManagerAllocationStatusInUse { + portsInUse = append(portsInUse, status.Ports...) + } + } + + isPortInUse := func(port uint16) bool { + for _, p := range portsInUse { + if p == port { + return true + } + } + + return false + } + + count := spec.Count + ports := make([]uint16, 0) + + // We first turn to the system configuration and try to allocate free ports from there. + itr := portsutil.NewPortIterator(config.Spec.Ports) + for port := itr.Next(); port != portsutil.InvalidPort; port = itr.Next() { + + if isPortInUse(port) { + continue + } + + ports = append(ports, port) + + if len(ports) >= count { + break + } + } + + if len(ports) >= count { + log.Info("Ports claimed from system configuration", "ports", ports) + return ports[:count], nnfv1alpha1.NnfPortManagerAllocationStatusInUse + } + + // If we still haven't found a sufficient number of free ports, free up unused allocations + // in the status' until we have the desired number of ports. + + const ( + exhausted = iota + more + ) + + // Search for a free allocation and claim ports from that allocation. Returns 'more' if there + // are potentially more allocations available with free ports, and 'exhausted' otherwise. + claimPortsFromFreeAllocation := func() int { + for idx := range mgr.Status.Allocations { + status := &mgr.Status.Allocations[idx] + + if status.Status == nnfv1alpha1.NnfPortManagerAllocationStatusFree { + log.Info("Ports claimed from free list", "ports", status.Ports) + + // Append this values ports to the returned ports. We could over-allocate here, but + // that is okay because we'll truncate the ports on return. + ports = append(ports, status.Ports...) + + // Remove the allocation from the list. + mgr.Status.Allocations = append(mgr.Status.Allocations[:idx], mgr.Status.Allocations[idx+1:]...) + + return more // because we just modified the list in place, must break from the for loop and force an additional search + } + } + + return exhausted + } + + // Loop until we have all (or more) of the requested port count, or we've exhausted the search + for len(ports) < count { + switch claimPortsFromFreeAllocation() { + case exhausted: + return []uint16{}, nnfv1alpha1.NnfPortManagerAllocationStatusInsufficientResources + case more: + // loop again if needed + } + } + + return ports[:count], nnfv1alpha1.NnfPortManagerAllocationStatusInUse +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NnfPortManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nnfv1alpha1.NnfPortManager{}). + Complete(r) +} diff --git a/controllers/nnf_port_manager_controller_test.go b/controllers/nnf_port_manager_controller_test.go new file mode 100644 index 000000000..e990cf321 --- /dev/null +++ b/controllers/nnf_port_manager_controller_test.go @@ -0,0 +1,184 @@ +/* + * Copyright 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import ( + "fmt" + "reflect" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" + nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" +) + +var _ = Context("NNF Port Manager Controller Setup", Ordered, func() { + + var cfg *dwsv1alpha1.SystemConfiguration + const portStart = 20 + const portEnd = 29 + + BeforeAll(func() { + cfg = &dwsv1alpha1.SystemConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "port-manager-system-config", + Namespace: corev1.NamespaceDefault, + }, + Spec: dwsv1alpha1.SystemConfigurationSpec{ + Ports: []intstr.IntOrString{ + intstr.FromString(fmt.Sprintf("%d-%d", portStart, portEnd)), + }, + }, + } + + Expect(k8sClient.Create(ctx, cfg)).To(Succeed()) + DeferCleanup(func() { Expect(k8sClient.Delete(ctx, cfg)).To(Succeed()) }) + }) + + Describe("NNF Port Manager Controller Test", func() { + + var mgr *nnfv1alpha1.NnfPortManager + var r = &NnfPortManagerReconciler{} // use this to access private reconciler methods + + BeforeEach(func() { + mgr = &nnfv1alpha1.NnfPortManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nnf-port-manager", + Namespace: corev1.NamespaceDefault, + }, + Spec: nnfv1alpha1.NnfPortManagerSpec{ + SystemConfiguration: corev1.ObjectReference{ + Name: cfg.Name, + Namespace: cfg.Namespace, + Kind: reflect.TypeOf(*cfg).Name(), + }, + Allocations: make([]nnfv1alpha1.NnfPortManagerAllocationSpec, 0), + }, + } + }) + + JustBeforeEach(func() { + Expect(k8sClient.Create(ctx, mgr)).To(Succeed()) + DeferCleanup(func() { Expect(k8sClient.Delete(ctx, mgr)).To(Succeed()) }) + }) + + reservePorts := func(mgr *nnfv1alpha1.NnfPortManager, name string, count int) { + By(fmt.Sprintf("Reserving %d ports for '%s'", count, name)) + + allocation := nnfv1alpha1.NnfPortManagerAllocationSpec{ + Requester: corev1.ObjectReference{Name: name}, + Count: count, + } + + Eventually(func(g Gomega) error { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mgr), mgr)).To(Succeed()) + + mgr.Spec.Allocations = append(mgr.Spec.Allocations, allocation) + + return k8sClient.Update(ctx, mgr) + }).Should(Succeed()) + + Eventually(func() bool { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mgr), mgr)).To(Succeed()) + return r.isAllocated(mgr, allocation) + }).Should(BeTrue()) + + status := r.findAllocationStatus(mgr, allocation) + Expect(status).ToNot(BeNil()) + Expect(status.Ports).To(HaveLen(allocation.Count)) + Expect(status.Status).To(Equal(nnfv1alpha1.NnfPortManagerAllocationStatusInUse)) + } + + releasePorts := func(mgr *nnfv1alpha1.NnfPortManager, name string) { + By(fmt.Sprintf("Releasing ports for '%s'", name)) + + requester := corev1.ObjectReference{Name: name} + + Eventually(func() error { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mgr), mgr)).To(Succeed()) + + for idx, allocation := range mgr.Spec.Allocations { + if allocation.Requester == requester { + mgr.Spec.Allocations = append(mgr.Spec.Allocations[:idx], mgr.Spec.Allocations[idx+1:]...) + } + } + + return k8sClient.Update(ctx, mgr) + }).Should(Succeed()) + } + + It("Reserves & removes a single port", func() { + const name = "single" + reservePorts(mgr, name, 1) + releasePorts(mgr, name) + }) + + It("Reserves & removes all ports", func() { + const name = "all" + reservePorts(mgr, name, portEnd-portStart+1) + releasePorts(mgr, name) + }) + + It("Reserves from free list", func() { + const single = "single" + reservePorts(mgr, single, 1) + + const remaining = "remaining" + reservePorts(mgr, remaining, portEnd-portStart) + + releasePorts(mgr, single) + + reservePorts(mgr, "free", 1) + }) + + It("Fails with insufficient resources", func() { + const name = "all" + reservePorts(mgr, name, portEnd-portStart+1) + + allocation := nnfv1alpha1.NnfPortManagerAllocationSpec{ + Requester: corev1.ObjectReference{Name: "insufficient-resources"}, + Count: 1, + } + + Eventually(func() error { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mgr), mgr)).To(Succeed()) + mgr.Spec.Allocations = append(mgr.Spec.Allocations, allocation) + return k8sClient.Update(ctx, mgr) + }).Should(Succeed()) + + Eventually(func() bool { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mgr), mgr)).To(Succeed()) + return r.isAllocated(mgr, allocation) + }).Should(BeTrue()) + + status := r.findAllocationStatus(mgr, allocation) + Expect(status).ToNot(BeNil()) + Expect(status.Ports).To(BeEmpty()) + Expect(status.Status).To(Equal(nnfv1alpha1.NnfPortManagerAllocationStatusInsufficientResources)) + }) + }) + +}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index b38b13deb..0beccd9f8 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -246,6 +246,12 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + err = (&NnfPortManagerReconciler{ + Client: k8sManager.GetClient(), + Scheme: testEnv.Scheme, + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + err = (&DWSStorageReconciler{ Client: k8sManager.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("Storage"), diff --git a/go.mod b/go.mod index 11aad4e11..6ec20a420 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/NearNodeFlash/nnf-sos go 1.19 require ( - github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 + github.com/HewlettPackard/dws v0.0.0-20230224185520-92a8916979a7 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index cb4a73fdb..e0f01f045 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 h1:847PiLepVQ10SoPQEc3OKdxWg6LXt70WXcCQj7ih3Bc= -github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075/go.mod h1:OxTK/qLMVBktON5wHI6zGuKEsoT8iLpPtUX+y02/m10= +github.com/HewlettPackard/dws v0.0.0-20230224185520-92a8916979a7 h1:wh2jAzQHY2WaT0eVYDtIo6hCbotQ5yUvIwe3s90Wob0= +github.com/HewlettPackard/dws v0.0.0-20230224185520-92a8916979a7/go.mod h1:ut/okqv7uImSeheKFGZQe07U1d6J9lluvw2eecRqiWU= github.com/HewlettPackard/structex v1.0.2 h1:p2EH/p6zvUd5fSa0onudAUvLWdKsvQSRP0jGKeblA5E= github.com/HewlettPackard/structex v1.0.2/go.mod h1:3frC4RY/cPsP/4+N8rkxsNAGlQwHV+zDC7qvrN+N+rE= github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= diff --git a/main.go b/main.go index e87f60af8..2750085dd 100644 --- a/main.go +++ b/main.go @@ -126,6 +126,13 @@ func main() { os.Exit(1) } + if err = (&controllers.NnfPortManagerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NnfPortManager") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { @@ -222,6 +229,14 @@ func (c *storageController) SetupReconcilers(mgr manager.Manager, opts *nnf.Opti return err } + if err := (&controllers.NnfPortManagerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + // Note: Log is built into controller-runtime + }).SetupWithManager(mgr); err != nil { + return err + } + if err := (&controllers.NnfWorkflowReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("NnfWorkflow"), diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/systemconfiguration_types.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/system_configuration_types.go similarity index 88% rename from vendor/github.com/HewlettPackard/dws/api/v1alpha1/systemconfiguration_types.go rename to vendor/github.com/HewlettPackard/dws/api/v1alpha1/system_configuration_types.go index 09e7c53f0..2605665dc 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/systemconfiguration_types.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/system_configuration_types.go @@ -21,6 +21,7 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // SystemConfigurationComputeNode describes a compute node in the system @@ -59,6 +60,12 @@ type SystemConfigurationSpec struct { // StorageNodes is the list of storage nodes on the system StorageNodes []SystemConfigurationStorageNode `json:"storageNodes,omitempty"` + + // Ports is the list of ports available for communication between nodes in the system. + // Valid values are single integers, or a range of values of the form "START-END" where + // START is an integer value that represents the start of a port range and END is an + // integer value that represents the end of the port range (inclusive). + Ports []intstr.IntOrString `json:"ports,omitempty"` } // SystemConfigurationStatus defines the status of SystemConfiguration diff --git a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go index a91fa9351..3055cd1c1 100644 --- a/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/github.com/HewlettPackard/dws/api/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,7 @@ import ( "github.com/HewlettPackard/dws/utils/dwdparse" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -1202,6 +1203,11 @@ func (in *SystemConfigurationSpec) DeepCopyInto(out *SystemConfigurationSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]intstr.IntOrString, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SystemConfigurationSpec. diff --git a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml index cfc5b2395..8d176cb09 100644 --- a/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml +++ b/vendor/github.com/HewlettPackard/dws/config/crd/bases/dws.cray.hpe.com_systemconfigurations.yaml @@ -59,6 +59,18 @@ spec: - name type: object type: array + ports: + description: Ports is the list of ports available for communication + between nodes in the system. Valid values are single integers, or + a range of values of the form "START-END" where START is an integer + value that represents the start of a port range and END is an integer + value that represents the end of the port range (inclusive). + items: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: array storageNodes: description: StorageNodes is the list of storage nodes on the system items: diff --git a/vendor/github.com/HewlettPackard/dws/utils/ports/ports.go b/vendor/github.com/HewlettPackard/dws/utils/ports/ports.go new file mode 100644 index 000000000..bca8e6493 --- /dev/null +++ b/vendor/github.com/HewlettPackard/dws/utils/ports/ports.go @@ -0,0 +1,148 @@ +/* + * Copyright 2023 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ports + +import ( + "fmt" + "math" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/util/intstr" +) + +const maxPort = math.MaxUint16 + +// Validate will validate the provided list of ports. +func Validate(ports []intstr.IntOrString) error { + + for _, port := range ports { + + switch port.Type { + case intstr.Int: + if port.IntVal <= 0 { + return fmt.Errorf("port value '%d' must be a positive value", port.IntVal) + } + if port.IntVal > maxPort { + return fmt.Errorf("port value '%d' exceeds maximum port size %d", port.IntVal, maxPort) + } + case intstr.String: + + // Expect the string value of the form "START-END" + parsed := strings.SplitN(port.StrVal, "-", 2) + if len(parsed) != 2 { + return fmt.Errorf("port range '%s' invalid", port.StrVal) + } + + start, err := strconv.Atoi(parsed[0]) + if err != nil { + return fmt.Errorf("port range '%s' starting value '%s' failed to parse", port.StrVal, parsed[0]) + } + + if start <= 0 { + return fmt.Errorf("port range '%s' starting value '%d' must be a positive value", port.StrVal, start) + } + + if start > maxPort { + return fmt.Errorf("port range '%s' starting value '%d' exceeds maximum port size %d", port.StrVal, start, maxPort) + } + + end, err := strconv.Atoi(parsed[1]) + if err != nil { + return fmt.Errorf("port range '%s' ending value '%s' failed to parse", port.StrVal, parsed[1]) + } + + if end <= 0 { + return fmt.Errorf("port range '%s' ending value '%d' must be a positive value", port.StrVal, end) + } + + if end > maxPort { + return fmt.Errorf("port range '%s' ending value '%d' exceeds maximum port size %d", port.StrVal, end, maxPort) + } + + if start >= end { + return fmt.Errorf("port range '%s' invalid", port.StrVal) + } + } + } + + // TODO: Do we wish to validate the port order? Check for overlaps? + + return nil +} + +const ( + InvalidPort = uint16(0) +) + +// NewPortIterator will return a new iterator for enumerating ports in the system +// configuration. +func NewPortIterator(ports []intstr.IntOrString) *portIterator { + return &portIterator{ + ports: ports, + index: 0, + subindex: 0, + } +} + +type portIterator struct { + ports []intstr.IntOrString + index int // index into ports array + subindex int // if port is string type, index into ranged port value "FIRST-LAST" +} + +// Next returns the next port from the iterator, or InvalidPort when done or an error has occurred. +func (itr *portIterator) Next() uint16 { + if itr.index >= len(itr.ports) { + return InvalidPort + } + + current := itr.ports[itr.index] + if current.Type == intstr.Int { + itr.index++ + itr.subindex = 0 + return uint16(current.IntVal) + } + + parsed := strings.SplitN(current.StrVal, "-", 2) + if len(parsed) != 2 { + return InvalidPort + } + + first, err := strconv.ParseUint(parsed[0], 10, 16) + if err != nil { + return InvalidPort + } + + last, err := strconv.ParseUint(parsed[1], 10, 16) + if err != nil { + return InvalidPort + } + + port := first + uint64(itr.subindex) + itr.subindex++ + + if port == last { + itr.index++ + itr.subindex = 0 + } + + return uint16(port) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 64d47ec8a..93fcdb835 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.6.0 ## explicit; go 1.12 github.com/Azure/go-autorest/tracing -# github.com/HewlettPackard/dws v0.0.0-20230207214424-016335e15075 +# github.com/HewlettPackard/dws v0.0.0-20230224185520-92a8916979a7 ## explicit; go 1.19 github.com/HewlettPackard/dws/api/v1alpha1 github.com/HewlettPackard/dws/config/crd/bases @@ -28,6 +28,7 @@ github.com/HewlettPackard/dws/config/webhook github.com/HewlettPackard/dws/controllers github.com/HewlettPackard/dws/controllers/metrics github.com/HewlettPackard/dws/utils/dwdparse +github.com/HewlettPackard/dws/utils/ports github.com/HewlettPackard/dws/utils/updater # github.com/HewlettPackard/structex v1.0.2 ## explicit; go 1.14 From b2c4363218b951fed54f21b64e871ad1d6559156 Mon Sep 17 00:00:00 2001 From: Nate Thornton Date: Wed, 1 Mar 2023 13:28:20 -0600 Subject: [PATCH 36/71] Nnf ec enhanced logging (#166) * NNF-EC logger * upgrade to nnf-ec master (47eb7a) * expose zap options --------- Signed-off-by: Nate Thornton --- controllers/nnf_node_ec_data_controller.go | 18 +- controllers/suite_test.go | 1 + go.mod | 71 +- go.sum | 581 +-- main.go | 14 +- .../go/compute/internal/version.go | 18 + .../go/compute/metadata/CHANGES.md | 19 + .../go/compute/metadata/LICENSE | 202 + .../go/compute/metadata/README.md | 27 + .../go/compute/metadata/metadata.go | 3 +- .../go/compute/metadata/tidyfix.go | 23 + .../HewlettPackard/structex/README.md | 13 +- .../HewlettPackard/structex/decoder.go | 86 +- .../HewlettPackard/structex/encoder.go | 73 +- .../HewlettPackard/structex/size.go | 16 + .../HewlettPackard/structex/tags.go | 19 +- .../HewlettPackard/structex/transcoder.go | 11 +- .../internal/switchtec/pkg/nvme/nvme.go | 55 +- .../internal/switchtec/pkg/nvme/switchtec.go | 25 +- .../switchtec/pkg/switchtec/switchtec.go | 24 +- .../NearNodeFlash/nnf-ec/pkg/controller.go | 10 +- .../NearNodeFlash/nnf-ec/pkg/ec/ec.go | 71 +- .../nnf-ec/pkg/manager-event/router.go | 2 +- .../nnf-ec/pkg/manager-fabric/config.go | 2 - .../pkg/manager-fabric/config_default.yaml | 1 - .../pkg/manager-fabric/config_dp1a.yaml | 1 - .../nnf-ec/pkg/manager-fabric/manager.go | 163 +- .../nnf-ec/pkg/manager-fabric/router.go | 4 +- .../pkg/manager-message-registry/router.go | 2 +- .../nnf-ec/pkg/manager-nnf/aer.go | 4 +- .../nnf-ec/pkg/manager-nnf/api.go | 5 +- .../nnf-ec/pkg/manager-nnf/config.go | 2 - .../nnf-ec/pkg/manager-nnf/config.yaml | 1 - .../nnf-ec/pkg/manager-nnf/manager.go | 250 +- .../nnf-ec/pkg/manager-nnf/persistent.go | 15 +- .../nnf-ec/pkg/manager-nnf/router.go | 4 +- .../nnf-ec/pkg/manager-nnf/servicer.go | 13 +- .../nnf-ec/pkg/manager-nnf/storage_pool.go | 15 +- .../nnf-ec/pkg/manager-nvme/config.go | 1 - .../nnf-ec/pkg/manager-nvme/config.yaml | 1 - .../nnf-ec/pkg/manager-nvme/manager.go | 225 +- .../nnf-ec/pkg/manager-nvme/router.go | 4 +- .../nnf-ec/pkg/manager-telemetry/router.go | 2 +- vendor/github.com/cespare/xxhash/v2/README.md | 31 +- .../github.com/cespare/xxhash/v2/testall.sh | 10 + vendor/github.com/cespare/xxhash/v2/xxhash.go | 47 +- .../cespare/xxhash/v2/xxhash_amd64.s | 336 +- .../cespare/xxhash/v2/xxhash_arm64.s | 183 + .../v2/{xxhash_amd64.go => xxhash_asm.go} | 2 + .../cespare/xxhash/v2/xxhash_other.go | 22 +- .../cespare/xxhash/v2/xxhash_safe.go | 1 + .../cespare/xxhash/v2/xxhash_unsafe.go | 3 +- .../dgraph-io/badger/v3/.go-version | 1 + .../dgraph-io/badger/v3/.golangci.yml | 22 +- .../dgraph-io/badger/v3/.travis.yml | 48 - .../dgraph-io/badger/v3/CHANGELOG.md | 20 + .../github.com/dgraph-io/badger/v3/Makefile | 60 + .../github.com/dgraph-io/badger/v3/README.md | 57 +- vendor/github.com/dgraph-io/badger/v3/db.go | 34 +- .../github.com/dgraph-io/badger/v3/levels.go | 18 +- .../dgraph-io/badger/v3/manifest.go | 10 +- .../dgraph-io/badger/v3/publisher.go | 21 +- .../dgraph-io/badger/v3/table/builder.go | 6 +- vendor/github.com/dgraph-io/badger/v3/test.sh | 87 +- .../dgraph-io/ristretto/.go-version | 1 + .../dgraph-io/ristretto/.golangci.yml | 23 + .../dgraph-io/ristretto/CHANGELOG.md | 19 +- .../github.com/dgraph-io/ristretto/README.md | 10 +- .../github.com/dgraph-io/ristretto/sketch.go | 3 +- vendor/github.com/dgraph-io/ristretto/test.sh | 20 - .../dgraph-io/ristretto/z/allocator.go | 2 +- .../dgraph-io/ristretto/z/mmap_linux.go | 30 - .../dgraph-io/ristretto/z/mremap_linux.go | 56 + .../ristretto/z/mremap_linux_arm64.go | 52 + .../dgraph-io/ristretto/z/simd/search.go | 2 +- .../github.com/dustin/go-humanize/.travis.yml | 16 +- .../dustin/go-humanize/README.markdown | 2 +- .../github.com/dustin/go-humanize/bigbytes.go | 20 +- .../github.com/dustin/go-humanize/commaf.go | 1 + vendor/github.com/dustin/go-humanize/ftoa.go | 3 + .../github.com/dustin/go-humanize/number.go | 2 +- vendor/github.com/dustin/go-humanize/si.go | 4 + vendor/github.com/go-logr/logr/funcr/funcr.go | 787 ++++ .../go-task/slim-sprig/.editorconfig | 14 + .../go-task/slim-sprig/.gitattributes | 1 + .../github.com/go-task/slim-sprig/.gitignore | 2 + .../go-task/slim-sprig/CHANGELOG.md | 364 ++ .../slim-sprig/LICENSE.txt} | 12 +- .../github.com/go-task/slim-sprig/README.md | 73 + .../go-task/slim-sprig/Taskfile.yml | 12 + .../github.com/go-task/slim-sprig/crypto.go | 24 + vendor/github.com/go-task/slim-sprig/date.go | 152 + .../github.com/go-task/slim-sprig/defaults.go | 163 + vendor/github.com/go-task/slim-sprig/dict.go | 118 + vendor/github.com/go-task/slim-sprig/doc.go | 19 + .../go-task/slim-sprig/functions.go | 317 ++ vendor/github.com/go-task/slim-sprig/list.go | 464 ++ .../github.com/go-task/slim-sprig/network.go | 12 + .../github.com/go-task/slim-sprig/numeric.go | 228 + .../github.com/go-task/slim-sprig/reflect.go | 28 + vendor/github.com/go-task/slim-sprig/regex.go | 83 + .../github.com/go-task/slim-sprig/strings.go | 189 + vendor/github.com/go-task/slim-sprig/url.go | 66 + .../google/flatbuffers/go/builder.go | 21 + .../github.com/google/flatbuffers/go/lib.go | 5 + vendor/github.com/google/pprof/AUTHORS | 7 + vendor/github.com/google/pprof/CONTRIBUTORS | 16 + vendor/github.com/google/pprof/LICENSE | 202 + .../github.com/google/pprof/profile/encode.go | 588 +++ .../github.com/google/pprof/profile/filter.go | 274 ++ .../github.com/google/pprof/profile/index.go | 64 + .../pprof/profile/legacy_java_profile.go | 315 ++ .../google/pprof/profile/legacy_profile.go | 1228 +++++ .../github.com/google/pprof/profile/merge.go | 568 +++ .../google/pprof/profile/profile.go | 813 ++++ .../github.com/google/pprof/profile/proto.go | 367 ++ .../github.com/google/pprof/profile/prune.go | 194 + .../github.com/klauspost/compress/.gitignore | 7 + .../klauspost/compress/.goreleaser.yml | 4 + vendor/github.com/klauspost/compress/LICENSE | 12 + .../github.com/klauspost/compress/README.md | 194 +- .../klauspost/compress/fse/compress.go | 31 +- .../klauspost/compress/huff0/bitreader.go | 130 +- .../klauspost/compress/huff0/bitwriter.go | 115 - .../klauspost/compress/huff0/bytereader.go | 10 - .../klauspost/compress/huff0/compress.go | 124 +- .../klauspost/compress/huff0/decompress.go | 1017 ++-- .../compress/huff0/decompress_amd64.go | 226 + .../compress/huff0/decompress_amd64.s | 830 ++++ .../compress/huff0/decompress_generic.go | 299 ++ .../klauspost/compress/huff0/huff0.go | 2 + .../compress/internal/cpuinfo/cpuinfo.go | 34 + .../internal/cpuinfo/cpuinfo_amd64.go | 11 + .../compress/internal/cpuinfo/cpuinfo_amd64.s | 36 + .../compress/internal/snapref/encode_other.go | 28 +- .../klauspost/compress/zstd/README.md | 172 +- .../klauspost/compress/zstd/bitreader.go | 12 +- .../klauspost/compress/zstd/bitwriter.go | 98 +- .../klauspost/compress/zstd/blockdec.go | 520 +-- .../klauspost/compress/zstd/blockenc.go | 108 +- .../klauspost/compress/zstd/bytebuf.go | 23 +- .../klauspost/compress/zstd/bytereader.go | 6 - .../klauspost/compress/zstd/decodeheader.go | 93 +- .../klauspost/compress/zstd/decoder.go | 690 ++- .../compress/zstd/decoder_options.go | 103 +- .../klauspost/compress/zstd/dict.go | 51 +- .../klauspost/compress/zstd/enc_base.go | 16 +- .../klauspost/compress/zstd/enc_best.go | 64 +- .../klauspost/compress/zstd/enc_better.go | 43 +- .../klauspost/compress/zstd/enc_dfast.go | 33 +- .../klauspost/compress/zstd/enc_fast.go | 163 +- .../klauspost/compress/zstd/encoder.go | 107 +- .../compress/zstd/encoder_options.go | 49 +- .../klauspost/compress/zstd/framedec.go | 339 +- .../klauspost/compress/zstd/fse_decoder.go | 128 +- .../compress/zstd/fse_decoder_amd64.go | 65 + .../compress/zstd/fse_decoder_amd64.s | 126 + .../compress/zstd/fse_decoder_generic.go | 72 + .../klauspost/compress/zstd/fse_encoder.go | 28 +- .../klauspost/compress/zstd/hash.go | 6 - .../klauspost/compress/zstd/history.go | 67 +- .../compress/zstd/internal/xxhash/README.md | 49 +- .../compress/zstd/internal/xxhash/xxhash.go | 48 +- .../zstd/internal/xxhash/xxhash_amd64.go | 12 - .../zstd/internal/xxhash/xxhash_amd64.s | 337 +- .../zstd/internal/xxhash/xxhash_arm64.s | 184 + .../zstd/internal/xxhash/xxhash_asm.go | 16 + .../zstd/internal/xxhash/xxhash_other.go | 23 +- .../klauspost/compress/zstd/seqdec.go | 327 +- .../klauspost/compress/zstd/seqdec_amd64.go | 379 ++ .../klauspost/compress/zstd/seqdec_amd64.s | 4079 +++++++++++++++++ .../klauspost/compress/zstd/seqdec_generic.go | 237 + .../github.com/klauspost/compress/zstd/zip.go | 69 +- .../klauspost/compress/zstd/zstd.go | 51 +- .../github.com/mattn/go-isatty/isatty_bsd.go | 4 +- vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md | 259 ++ vendor/github.com/onsi/ginkgo/v2/README.md | 58 +- vendor/github.com/onsi/ginkgo/v2/RELEASING.md | 8 +- vendor/github.com/onsi/ginkgo/v2/core_dsl.go | 165 +- .../onsi/ginkgo/v2/decorator_dsl.go | 49 +- .../onsi/ginkgo/v2/formatter/formatter.go | 61 +- .../ginkgo/v2/ginkgo/build/build_command.go | 63 + .../onsi/ginkgo/v2/ginkgo/command/abort.go | 61 + .../onsi/ginkgo/v2/ginkgo/command/command.go | 50 + .../onsi/ginkgo/v2/ginkgo/command/program.go | 182 + .../ginkgo/generators/boostrap_templates.go | 48 + .../v2/ginkgo/generators/bootstrap_command.go | 133 + .../v2/ginkgo/generators/generate_command.go | 259 ++ .../ginkgo/generators/generate_templates.go | 41 + .../v2/ginkgo/generators/generators_common.go | 64 + .../onsi/ginkgo/v2/ginkgo/internal/compile.go | 152 + .../ginkgo/internal/profiles_and_reports.go | 237 + .../onsi/ginkgo/v2/ginkgo/internal/run.go | 355 ++ .../ginkgo/v2/ginkgo/internal/test_suite.go | 283 ++ .../onsi/ginkgo/v2/ginkgo/internal/utils.go | 86 + .../v2/ginkgo/internal/verify_version.go | 54 + .../ginkgo/v2/ginkgo/labels/labels_command.go | 123 + .../github.com/onsi/ginkgo/v2/ginkgo/main.go | 58 + .../onsi/ginkgo/v2/ginkgo/outline/ginkgo.go | 302 ++ .../onsi/ginkgo/v2/ginkgo/outline/import.go | 65 + .../onsi/ginkgo/v2/ginkgo/outline/outline.go | 110 + .../v2/ginkgo/outline/outline_command.go | 98 + .../onsi/ginkgo/v2/ginkgo/run/run_command.go | 232 + .../v2/ginkgo/unfocus/unfocus_command.go | 186 + .../onsi/ginkgo/v2/ginkgo/watch/delta.go | 22 + .../ginkgo/v2/ginkgo/watch/delta_tracker.go | 75 + .../ginkgo/v2/ginkgo/watch/dependencies.go | 92 + .../ginkgo/v2/ginkgo/watch/package_hash.go | 108 + .../ginkgo/v2/ginkgo/watch/package_hashes.go | 85 + .../onsi/ginkgo/v2/ginkgo/watch/suite.go | 87 + .../ginkgo/v2/ginkgo/watch/watch_command.go | 192 + .../onsi/ginkgo/v2/ginkgo_cli_dependencies.go | 8 + .../github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go | 2 +- .../onsi/ginkgo/v2/internal/group.go | 315 +- .../interrupt_handler/interrupt_handler.go | 164 +- .../onsi/ginkgo/v2/internal/node.go | 386 +- .../onsi/ginkgo/v2/internal/ordering.go | 77 +- .../parallel_support/client_server.go | 2 + .../internal/parallel_support/http_client.go | 13 + .../internal/parallel_support/http_server.go | 29 +- .../internal/parallel_support/rpc_client.go | 13 + .../parallel_support/server_handler.go | 55 +- .../ginkgo/v2/internal/progress_report.go | 28 +- .../onsi/ginkgo/v2/internal/report_entry.go | 21 +- .../onsi/ginkgo/v2/internal/spec.go | 16 + .../onsi/ginkgo/v2/internal/spec_context.go | 90 + .../onsi/ginkgo/v2/internal/suite.go | 469 +- .../internal/testingtproxy/testing_t_proxy.go | 2 +- .../onsi/ginkgo/v2/internal/writer.go | 51 +- .../ginkgo/v2/reporters/default_reporter.go | 630 ++- .../v2/reporters/deprecated_reporter.go | 2 +- .../onsi/ginkgo/v2/reporters/junit_report.go | 119 +- .../onsi/ginkgo/v2/reporters/reporter.go | 18 +- .../ginkgo/v2/reporters/teamcity_report.go | 12 +- .../onsi/ginkgo/v2/reporting_dsl.go | 28 +- vendor/github.com/onsi/ginkgo/v2/table_dsl.go | 96 +- .../onsi/ginkgo/v2/types/code_location.go | 77 +- .../github.com/onsi/ginkgo/v2/types/config.go | 42 +- .../ginkgo/v2/types/deprecation_support.go | 7 + .../github.com/onsi/ginkgo/v2/types/errors.go | 103 +- .../onsi/ginkgo/v2/types/label_filter.go | 11 + .../onsi/ginkgo/v2/types/report_entry.go | 14 +- .../github.com/onsi/ginkgo/v2/types/types.go | 364 +- .../onsi/ginkgo/v2/types/version.go | 2 +- vendor/github.com/onsi/gomega/.gitignore | 2 + vendor/github.com/onsi/gomega/CHANGELOG.md | 151 + vendor/github.com/onsi/gomega/RELEASING.md | 8 +- .../github.com/onsi/gomega/format/format.go | 78 +- vendor/github.com/onsi/gomega/gomega_dsl.go | 198 +- .../onsi/gomega/internal/assertion.go | 8 +- .../onsi/gomega/internal/async_assertion.go | 566 ++- .../onsi/gomega/internal/duration_bundle.go | 16 +- .../github.com/onsi/gomega/internal/gomega.go | 67 +- .../gomega/internal/polling_signal_error.go | 106 + vendor/github.com/onsi/gomega/matchers.go | 534 ++- .../onsi/gomega/matchers/be_key_of_matcher.go | 45 + .../gomega/matchers/have_exact_elements.go | 75 + .../gomega/matchers/match_error_matcher.go | 12 +- .../onsi/gomega/matchers/succeed_matcher.go | 9 + vendor/github.com/onsi/gomega/types/types.go | 20 +- vendor/github.com/rs/cors/.travis.yml | 7 - vendor/github.com/rs/cors/README.md | 3 +- vendor/github.com/rs/cors/cors.go | 33 +- vendor/github.com/sigurn/crc8/crc8.go | 21 +- vendor/github.com/sigurn/utils/utils.go | 25 - .../common/runtime/goroutine/goroutine_id.go | 2 +- vendor/go.opencensus.io/Makefile | 8 +- vendor/go.opencensus.io/opencensus.go | 2 +- vendor/go.opencensus.io/trace/doc.go | 13 +- vendor/go.opencensus.io/trace/lrumap.go | 2 +- vendor/go.opencensus.io/trace/trace_go11.go | 1 + .../go.opencensus.io/trace/trace_nongo11.go | 1 + vendor/go.uber.org/multierr/CHANGELOG.md | 8 + vendor/go.uber.org/multierr/error.go | 345 +- vendor/go.uber.org/zap/CHANGELOG.md | 13 + vendor/go.uber.org/zap/array_go118.go | 8 +- vendor/go.uber.org/zap/logger.go | 7 + vendor/go.uber.org/zap/options.go | 3 +- vendor/go.uber.org/zap/sink.go | 100 +- vendor/go.uber.org/zap/stacktrace.go | 2 +- vendor/go.uber.org/zap/sugar.go | 31 +- vendor/go.uber.org/zap/writer.go | 2 +- vendor/go.uber.org/zap/zapcore/entry.go | 3 +- vendor/golang.org/x/crypto/pkcs12/crypto.go | 2 +- .../x/crypto/pkcs12/internal/rc2/rc2.go | 53 +- vendor/golang.org/x/net/html/parse.go | 4 +- vendor/golang.org/x/net/html/token.go | 57 +- vendor/golang.org/x/net/http2/flow.go | 88 +- vendor/golang.org/x/net/http2/frame.go | 11 +- vendor/golang.org/x/net/http2/headermap.go | 18 + vendor/golang.org/x/net/http2/hpack/encode.go | 5 + vendor/golang.org/x/net/http2/hpack/hpack.go | 81 +- .../x/net/http2/hpack/static_table.go | 188 + vendor/golang.org/x/net/http2/hpack/tables.go | 78 +- vendor/golang.org/x/net/http2/server.go | 335 +- vendor/golang.org/x/net/http2/transport.go | 287 +- vendor/golang.org/x/net/trace/histogram.go | 2 +- vendor/golang.org/x/net/trace/trace.go | 2 +- vendor/golang.org/x/oauth2/google/doc.go | 12 +- .../google/internal/externalaccount/aws.go | 105 +- .../externalaccount/basecredentials.go | 6 + vendor/golang.org/x/sync/errgroup/errgroup.go | 4 +- vendor/golang.org/x/sys/plan9/mkerrors.sh | 4 +- vendor/golang.org/x/sys/unix/asm_bsd_ppc64.s | 31 + vendor/golang.org/x/sys/unix/dirent.go | 4 +- vendor/golang.org/x/sys/unix/gccgo.go | 4 +- vendor/golang.org/x/sys/unix/gccgo_c.c | 4 +- vendor/golang.org/x/sys/unix/ioctl.go | 4 +- vendor/golang.org/x/sys/unix/mkall.sh | 22 +- vendor/golang.org/x/sys/unix/mkerrors.sh | 4 +- vendor/golang.org/x/sys/unix/sockcmsg_unix.go | 14 + .../golang.org/x/sys/unix/syscall_darwin.go | 1 + .../x/sys/unix/syscall_dragonfly.go | 1 + .../golang.org/x/sys/unix/syscall_freebsd.go | 1 + .../x/sys/unix/syscall_freebsd_386.go | 9 +- .../x/sys/unix/syscall_freebsd_amd64.go | 9 +- .../x/sys/unix/syscall_freebsd_arm.go | 9 +- .../x/sys/unix/syscall_freebsd_arm64.go | 9 +- .../x/sys/unix/syscall_freebsd_riscv64.go | 9 +- vendor/golang.org/x/sys/unix/syscall_hurd.go | 22 + .../golang.org/x/sys/unix/syscall_hurd_386.go | 29 + .../golang.org/x/sys/unix/syscall_illumos.go | 106 - vendor/golang.org/x/sys/unix/syscall_linux.go | 52 +- .../golang.org/x/sys/unix/syscall_netbsd.go | 15 + .../golang.org/x/sys/unix/syscall_openbsd.go | 1 + .../x/sys/unix/syscall_openbsd_libc.go | 4 +- .../x/sys/unix/syscall_openbsd_ppc64.go | 42 + .../x/sys/unix/syscall_openbsd_riscv64.go | 42 + .../golang.org/x/sys/unix/syscall_solaris.go | 105 + vendor/golang.org/x/sys/unix/syscall_unix.go | 57 +- .../golang.org/x/sys/unix/syscall_unix_gc.go | 6 +- .../x/sys/unix/syscall_zos_s390x.go | 173 +- vendor/golang.org/x/sys/unix/timestruct.go | 2 +- vendor/golang.org/x/sys/unix/xattr_bsd.go | 9 +- vendor/golang.org/x/sys/unix/zerrors_linux.go | 30 +- .../x/sys/unix/zerrors_linux_386.go | 1 + .../x/sys/unix/zerrors_linux_amd64.go | 1 + .../x/sys/unix/zerrors_linux_arm.go | 1 + .../x/sys/unix/zerrors_linux_arm64.go | 1 + .../x/sys/unix/zerrors_linux_loong64.go | 1 + .../x/sys/unix/zerrors_linux_mips.go | 1 + .../x/sys/unix/zerrors_linux_mips64.go | 1 + .../x/sys/unix/zerrors_linux_mips64le.go | 1 + .../x/sys/unix/zerrors_linux_mipsle.go | 1 + .../x/sys/unix/zerrors_linux_ppc.go | 1 + .../x/sys/unix/zerrors_linux_ppc64.go | 1 + .../x/sys/unix/zerrors_linux_ppc64le.go | 1 + .../x/sys/unix/zerrors_linux_riscv64.go | 1 + .../x/sys/unix/zerrors_linux_s390x.go | 1 + .../x/sys/unix/zerrors_linux_sparc64.go | 1 + .../x/sys/unix/zerrors_openbsd_386.go | 356 +- .../x/sys/unix/zerrors_openbsd_amd64.go | 189 +- .../x/sys/unix/zerrors_openbsd_arm.go | 348 +- .../x/sys/unix/zerrors_openbsd_arm64.go | 160 +- .../x/sys/unix/zerrors_openbsd_mips64.go | 95 +- .../x/sys/unix/zerrors_openbsd_ppc64.go | 1905 ++++++++ .../x/sys/unix/zerrors_openbsd_riscv64.go | 1904 ++++++++ .../x/sys/unix/zsyscall_dragonfly_amd64.go | 10 + .../x/sys/unix/zsyscall_freebsd_386.go | 10 + .../x/sys/unix/zsyscall_freebsd_amd64.go | 10 + .../x/sys/unix/zsyscall_freebsd_arm.go | 10 + .../x/sys/unix/zsyscall_freebsd_arm64.go | 10 + .../x/sys/unix/zsyscall_freebsd_riscv64.go | 10 + .../x/sys/unix/zsyscall_illumos_amd64.go | 28 +- .../golang.org/x/sys/unix/zsyscall_linux.go | 11 + .../x/sys/unix/zsyscall_netbsd_386.go | 10 + .../x/sys/unix/zsyscall_netbsd_amd64.go | 10 + .../x/sys/unix/zsyscall_netbsd_arm.go | 10 + .../x/sys/unix/zsyscall_netbsd_arm64.go | 10 + .../x/sys/unix/zsyscall_openbsd_386.go | 14 + .../x/sys/unix/zsyscall_openbsd_386.s | 137 +- .../x/sys/unix/zsyscall_openbsd_amd64.go | 14 + .../x/sys/unix/zsyscall_openbsd_amd64.s | 137 +- .../x/sys/unix/zsyscall_openbsd_arm.go | 14 + .../x/sys/unix/zsyscall_openbsd_arm.s | 137 +- .../x/sys/unix/zsyscall_openbsd_arm64.go | 14 + .../x/sys/unix/zsyscall_openbsd_arm64.s | 137 +- .../x/sys/unix/zsyscall_openbsd_mips64.go | 812 +++- .../x/sys/unix/zsyscall_openbsd_mips64.s | 669 +++ .../x/sys/unix/zsyscall_openbsd_ppc64.go | 2235 +++++++++ .../x/sys/unix/zsyscall_openbsd_ppc64.s | 802 ++++ .../x/sys/unix/zsyscall_openbsd_riscv64.go | 2235 +++++++++ .../x/sys/unix/zsyscall_openbsd_riscv64.s | 669 +++ .../x/sys/unix/zsyscall_solaris_amd64.go | 41 +- .../x/sys/unix/zsysctl_openbsd_386.go | 51 +- .../x/sys/unix/zsysctl_openbsd_amd64.go | 17 +- .../x/sys/unix/zsysctl_openbsd_arm.go | 51 +- .../x/sys/unix/zsysctl_openbsd_arm64.go | 11 +- .../x/sys/unix/zsysctl_openbsd_mips64.go | 3 +- .../x/sys/unix/zsysctl_openbsd_ppc64.go | 281 ++ .../x/sys/unix/zsysctl_openbsd_riscv64.go | 282 ++ .../x/sys/unix/zsysnum_openbsd_mips64.go | 1 + .../x/sys/unix/zsysnum_openbsd_ppc64.go | 218 + .../x/sys/unix/zsysnum_openbsd_riscv64.go | 219 + .../x/sys/unix/ztypes_illumos_amd64.go | 42 - vendor/golang.org/x/sys/unix/ztypes_linux.go | 217 +- .../x/sys/unix/ztypes_netbsd_386.go | 84 + .../x/sys/unix/ztypes_netbsd_amd64.go | 84 + .../x/sys/unix/ztypes_netbsd_arm.go | 84 + .../x/sys/unix/ztypes_netbsd_arm64.go | 84 + .../x/sys/unix/ztypes_openbsd_386.go | 97 +- .../x/sys/unix/ztypes_openbsd_amd64.go | 33 +- .../x/sys/unix/ztypes_openbsd_arm.go | 9 +- .../x/sys/unix/ztypes_openbsd_arm64.go | 9 +- .../x/sys/unix/ztypes_openbsd_mips64.go | 9 +- .../x/sys/unix/ztypes_openbsd_ppc64.go | 571 +++ .../x/sys/unix/ztypes_openbsd_riscv64.go | 571 +++ .../x/sys/unix/ztypes_solaris_amd64.go | 35 + .../golang.org/x/sys/unix/ztypes_zos_s390x.go | 11 +- .../x/sys/windows/syscall_windows.go | 19 +- .../x/sys/windows/zsyscall_windows.go | 7 + vendor/golang.org/x/term/terminal.go | 1 - .../x/text/encoding/htmlindex/tables.go | 9 + .../text/encoding/internal/identifier/mib.go | 8 + .../x/text/encoding/simplifiedchinese/gbk.go | 6 +- .../internal/language/compact/language.go | 2 +- .../text/internal/language/compact/tables.go | 6 +- .../x/text/internal/language/language.go | 2 +- .../x/text/internal/language/lookup.go | 4 +- .../x/text/internal/language/parse.go | 24 +- .../x/text/internal/language/tables.go | 868 ++-- .../internal/utf8internal/utf8internal.go | 2 +- vendor/golang.org/x/text/language/doc.go | 44 +- vendor/golang.org/x/text/language/go1_1.go | 39 - vendor/golang.org/x/text/language/go1_2.go | 12 - vendor/golang.org/x/text/language/language.go | 2 +- vendor/golang.org/x/text/language/match.go | 2 +- vendor/golang.org/x/text/language/parse.go | 8 +- vendor/golang.org/x/text/language/tables.go | 18 +- vendor/golang.org/x/text/runes/runes.go | 2 +- vendor/golang.org/x/text/unicode/bidi/core.go | 26 +- .../golang.org/x/text/unicode/bidi/trieval.go | 12 - .../x/text/unicode/norm/forminfo.go | 9 +- .../x/text/unicode/norm/normalize.go | 11 +- .../x/text/unicode/norm/tables13.0.0.go | 4 +- vendor/golang.org/x/time/rate/rate.go | 20 +- vendor/golang.org/x/time/rate/sometimes.go | 67 + vendor/golang.org/x/tools/LICENSE | 27 + vendor/golang.org/x/tools/PATENTS | 22 + .../x/tools/go/ast/inspector/inspector.go | 218 + .../x/tools/go/ast/inspector/typeof.go | 229 + .../x/tools/internal/typeparams/common.go | 179 + .../x/tools/internal/typeparams/coretype.go | 122 + .../internal/typeparams/enabled_go117.go | 12 + .../internal/typeparams/enabled_go118.go | 15 + .../x/tools/internal/typeparams/normalize.go | 218 + .../x/tools/internal/typeparams/termlist.go | 163 + .../internal/typeparams/typeparams_go117.go | 197 + .../internal/typeparams/typeparams_go118.go | 151 + .../x/tools/internal/typeparams/typeterm.go | 170 + .../google.golang.org/appengine/.travis.yml | 18 - .../appengine/CONTRIBUTING.md | 6 +- vendor/google.golang.org/appengine/README.md | 4 +- .../google.golang.org/appengine/appengine.go | 21 +- .../appengine/appengine_vm.go | 12 +- .../appengine/internal/api.go | 296 +- .../appengine/internal/api_classic.go | 5 +- .../appengine/internal/api_common.go | 20 +- .../appengine/internal/identity_classic.go | 1 + .../appengine/internal/identity_flex.go | 1 + .../appengine/internal/identity_vm.go | 3 +- .../appengine/internal/main.go | 1 + .../appengine/internal/main_vm.go | 3 +- .../appengine/travis_install.sh | 18 - .../appengine/travis_test.sh | 12 - .../appengine/urlfetch/urlfetch.go | 7 +- .../apimachinery/pkg/api/meta/errors.go | 35 +- .../apimachinery/pkg/apis/meta/v1/helpers.go | 2 +- .../pkg/apis/meta/v1/micro_time_proto.go | 10 +- .../apimachinery/pkg/labels/selector.go | 10 +- .../k8s.io/apimachinery/pkg/util/sets/byte.go | 148 +- .../k8s.io/apimachinery/pkg/util/sets/doc.go | 7 +- .../apimachinery/pkg/util/sets/empty.go | 4 +- .../k8s.io/apimachinery/pkg/util/sets/int.go | 148 +- .../apimachinery/pkg/util/sets/int32.go | 148 +- .../apimachinery/pkg/util/sets/int64.go | 148 +- .../apimachinery/pkg/util/sets/ordered.go | 53 + .../k8s.io/apimachinery/pkg/util/sets/set.go | 227 + .../apimachinery/pkg/util/sets/string.go | 148 +- .../k8s.io/klog/v2/internal/buffer/buffer.go | 40 +- .../klog/v2/internal/serialize/keyvalues.go | 179 +- vendor/k8s.io/klog/v2/k8s_references.go | 78 +- vendor/k8s.io/klog/v2/klog.go | 38 +- vendor/k8s.io/klog/v2/klogr.go | 12 +- .../k8s.io/kube-openapi/pkg/internal/flags.go | 20 + .../go-json-experiment/json}/AUTHORS | 2 +- .../go-json-experiment/json}/CONTRIBUTORS | 2 +- .../go-json-experiment/json/LICENSE | 27 + .../go-json-experiment/json/README.md | 321 ++ .../go-json-experiment/json/arshal.go | 506 ++ .../go-json-experiment/json/arshal_any.go | 219 + .../go-json-experiment/json/arshal_default.go | 1446 ++++++ .../go-json-experiment/json/arshal_funcs.go | 387 ++ .../go-json-experiment/json/arshal_inlined.go | 186 + .../go-json-experiment/json/arshal_methods.go | 229 + .../go-json-experiment/json/arshal_time.go | 196 + .../go-json-experiment/json/decode.go | 1655 +++++++ .../go-json-experiment/json/doc.go | 185 + .../go-json-experiment/json/encode.go | 1146 +++++ .../go-json-experiment/json/errors.go | 183 + .../go-json-experiment/json/fields.go | 509 ++ .../go-json-experiment/json/fold.go | 56 + .../go-json-experiment/json/intern.go | 86 + .../go-json-experiment/json/pools.go | 150 + .../go-json-experiment/json/state.go | 747 +++ .../go-json-experiment/json/token.go | 522 +++ .../go-json-experiment/json/value.go | 375 ++ .../kube-openapi/pkg/validation/spec/fuzz.go | 502 ++ .../pkg/validation/spec/gnostic.go | 6 +- .../pkg/validation/spec/header.go | 30 + .../kube-openapi/pkg/validation/spec/info.go | 48 + .../kube-openapi/pkg/validation/spec/items.go | 28 + .../pkg/validation/spec/operation.go | 24 + .../pkg/validation/spec/parameter.go | 67 +- .../pkg/validation/spec/path_item.go | 31 + .../kube-openapi/pkg/validation/spec/paths.go | 59 + .../pkg/validation/spec/response.go | 37 +- .../pkg/validation/spec/responses.go | 67 + .../pkg/validation/spec/schema.go | 41 + .../pkg/validation/spec/security_scheme.go | 18 + .../pkg/validation/spec/swagger.go | 107 + .../kube-openapi/pkg/validation/spec/tag.go | 23 + vendor/k8s.io/mount-utils/mount.go | 10 +- vendor/k8s.io/mount-utils/mount_linux.go | 5 +- .../k8s.io/mount-utils/mount_unsupported.go | 2 +- vendor/k8s.io/mount-utils/mount_windows.go | 8 +- vendor/k8s.io/utils/net/ipfamily.go | 181 + vendor/k8s.io/utils/net/net.go | 126 +- vendor/k8s.io/utils/net/port.go | 18 +- vendor/k8s.io/utils/pointer/pointer.go | 20 + vendor/k8s.io/utils/trace/trace.go | 30 +- vendor/modules.txt | 114 +- .../internal/golang/encoding/json/decode.go | 5 +- .../internal/golang/encoding/json/encode.go | 37 +- .../internal/golang/encoding/json/fold.go | 5 +- .../internal/golang/encoding/json/scanner.go | 2 + .../internal/golang/encoding/json/stream.go | 1 - vendor/sigs.k8s.io/json/json.go | 28 +- 538 files changed, 58412 insertions(+), 9250 deletions(-) create mode 100644 vendor/cloud.google.com/go/compute/internal/version.go create mode 100644 vendor/cloud.google.com/go/compute/metadata/CHANGES.md create mode 100644 vendor/cloud.google.com/go/compute/metadata/LICENSE create mode 100644 vendor/cloud.google.com/go/compute/metadata/README.md create mode 100644 vendor/cloud.google.com/go/compute/metadata/tidyfix.go create mode 100644 vendor/github.com/cespare/xxhash/v2/testall.sh create mode 100644 vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s rename vendor/github.com/cespare/xxhash/v2/{xxhash_amd64.go => xxhash_asm.go} (73%) create mode 100644 vendor/github.com/dgraph-io/badger/v3/.go-version delete mode 100644 vendor/github.com/dgraph-io/badger/v3/.travis.yml create mode 100644 vendor/github.com/dgraph-io/badger/v3/Makefile create mode 100644 vendor/github.com/dgraph-io/ristretto/.go-version create mode 100644 vendor/github.com/dgraph-io/ristretto/.golangci.yml delete mode 100644 vendor/github.com/dgraph-io/ristretto/test.sh create mode 100644 vendor/github.com/dgraph-io/ristretto/z/mremap_linux.go create mode 100644 vendor/github.com/dgraph-io/ristretto/z/mremap_linux_arm64.go create mode 100644 vendor/github.com/go-logr/logr/funcr/funcr.go create mode 100644 vendor/github.com/go-task/slim-sprig/.editorconfig create mode 100644 vendor/github.com/go-task/slim-sprig/.gitattributes create mode 100644 vendor/github.com/go-task/slim-sprig/.gitignore create mode 100644 vendor/github.com/go-task/slim-sprig/CHANGELOG.md rename vendor/github.com/{sigurn/utils/LICENSE.md => go-task/slim-sprig/LICENSE.txt} (87%) create mode 100644 vendor/github.com/go-task/slim-sprig/README.md create mode 100644 vendor/github.com/go-task/slim-sprig/Taskfile.yml create mode 100644 vendor/github.com/go-task/slim-sprig/crypto.go create mode 100644 vendor/github.com/go-task/slim-sprig/date.go create mode 100644 vendor/github.com/go-task/slim-sprig/defaults.go create mode 100644 vendor/github.com/go-task/slim-sprig/dict.go create mode 100644 vendor/github.com/go-task/slim-sprig/doc.go create mode 100644 vendor/github.com/go-task/slim-sprig/functions.go create mode 100644 vendor/github.com/go-task/slim-sprig/list.go create mode 100644 vendor/github.com/go-task/slim-sprig/network.go create mode 100644 vendor/github.com/go-task/slim-sprig/numeric.go create mode 100644 vendor/github.com/go-task/slim-sprig/reflect.go create mode 100644 vendor/github.com/go-task/slim-sprig/regex.go create mode 100644 vendor/github.com/go-task/slim-sprig/strings.go create mode 100644 vendor/github.com/go-task/slim-sprig/url.go create mode 100644 vendor/github.com/google/pprof/AUTHORS create mode 100644 vendor/github.com/google/pprof/CONTRIBUTORS create mode 100644 vendor/github.com/google/pprof/LICENSE create mode 100644 vendor/github.com/google/pprof/profile/encode.go create mode 100644 vendor/github.com/google/pprof/profile/filter.go create mode 100644 vendor/github.com/google/pprof/profile/index.go create mode 100644 vendor/github.com/google/pprof/profile/legacy_java_profile.go create mode 100644 vendor/github.com/google/pprof/profile/legacy_profile.go create mode 100644 vendor/github.com/google/pprof/profile/merge.go create mode 100644 vendor/github.com/google/pprof/profile/profile.go create mode 100644 vendor/github.com/google/pprof/profile/proto.go create mode 100644 vendor/github.com/google/pprof/profile/prune.go create mode 100644 vendor/github.com/klauspost/compress/huff0/decompress_amd64.go create mode 100644 vendor/github.com/klauspost/compress/huff0/decompress_amd64.s create mode 100644 vendor/github.com/klauspost/compress/huff0/decompress_generic.go create mode 100644 vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go create mode 100644 vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go create mode 100644 vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s create mode 100644 vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go create mode 100644 vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s create mode 100644 vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go create mode 100644 vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go create mode 100644 vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s create mode 100644 vendor/github.com/klauspost/compress/zstd/seqdec_generic.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/build/build_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/command/abort.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/command/command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/command/program.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/boostrap_templates.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/bootstrap_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_templates.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generators_common.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/compile.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/profiles_and_reports.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/run.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/test_suite.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/utils.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/verify_version.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/labels/labels_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/main.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/ginkgo.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/import.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/run/run_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/unfocus/unfocus_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta_tracker.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/dependencies.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hash.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hashes.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/suite.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/watch_command.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/ginkgo_cli_dependencies.go create mode 100644 vendor/github.com/onsi/ginkgo/v2/internal/spec_context.go create mode 100644 vendor/github.com/onsi/gomega/internal/polling_signal_error.go create mode 100644 vendor/github.com/onsi/gomega/matchers/be_key_of_matcher.go create mode 100644 vendor/github.com/onsi/gomega/matchers/have_exact_elements.go delete mode 100644 vendor/github.com/rs/cors/.travis.yml delete mode 100644 vendor/github.com/sigurn/utils/utils.go create mode 100644 vendor/golang.org/x/net/http2/hpack/static_table.go create mode 100644 vendor/golang.org/x/sys/unix/asm_bsd_ppc64.s create mode 100644 vendor/golang.org/x/sys/unix/syscall_hurd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_hurd_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd_riscv64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_openbsd_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_openbsd_riscv64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s create mode 100644 vendor/golang.org/x/sys/unix/zsysctl_openbsd_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysctl_openbsd_riscv64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_openbsd_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_openbsd_riscv64.go delete mode 100644 vendor/golang.org/x/sys/unix/ztypes_illumos_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_openbsd_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_openbsd_riscv64.go delete mode 100644 vendor/golang.org/x/text/language/go1_1.go delete mode 100644 vendor/golang.org/x/text/language/go1_2.go create mode 100644 vendor/golang.org/x/time/rate/sometimes.go create mode 100644 vendor/golang.org/x/tools/LICENSE create mode 100644 vendor/golang.org/x/tools/PATENTS create mode 100644 vendor/golang.org/x/tools/go/ast/inspector/inspector.go create mode 100644 vendor/golang.org/x/tools/go/ast/inspector/typeof.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/common.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/coretype.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/enabled_go117.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/enabled_go118.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/normalize.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/termlist.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/typeparams_go117.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/typeparams_go118.go create mode 100644 vendor/golang.org/x/tools/internal/typeparams/typeterm.go delete mode 100644 vendor/google.golang.org/appengine/.travis.yml delete mode 100644 vendor/google.golang.org/appengine/travis_install.sh delete mode 100644 vendor/google.golang.org/appengine/travis_test.sh create mode 100644 vendor/k8s.io/apimachinery/pkg/util/sets/ordered.go create mode 100644 vendor/k8s.io/apimachinery/pkg/util/sets/set.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/flags.go rename vendor/{golang.org/x/text => k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json}/AUTHORS (74%) rename vendor/{golang.org/x/text => k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json}/CONTRIBUTORS (70%) create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/LICENSE create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/README.md create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/arshal.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/arshal_any.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/arshal_default.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/arshal_funcs.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/arshal_inlined.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/arshal_methods.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/arshal_time.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/decode.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/doc.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/encode.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/errors.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/fields.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/fold.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/intern.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/pools.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/state.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/token.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json/value.go create mode 100644 vendor/k8s.io/kube-openapi/pkg/validation/spec/fuzz.go create mode 100644 vendor/k8s.io/utils/net/ipfamily.go diff --git a/controllers/nnf_node_ec_data_controller.go b/controllers/nnf_node_ec_data_controller.go index 5dec1b73c..ec965287f 100644 --- a/controllers/nnf_node_ec_data_controller.go +++ b/controllers/nnf_node_ec_data_controller.go @@ -36,6 +36,7 @@ import ( "github.com/NearNodeFlash/nnf-ec/pkg/persistent" nnfv1alpha1 "github.com/NearNodeFlash/nnf-sos/api/v1alpha1" "github.com/NearNodeFlash/nnf-sos/controllers/metrics" + "github.com/go-logr/logr" ) const ( @@ -48,6 +49,8 @@ type NnfNodeECDataReconciler struct { Scheme *runtime.Scheme Options *nnfec.Options types.NamespacedName + + RawLog logr.Logger // RawLog, as opposed to Log, is the un-edited controller logger } // Start implements manager.Runnable @@ -83,8 +86,21 @@ func (r *NnfNodeECDataReconciler) Start(ctx context.Context) error { persistent.StorageProvider = r } + // NNF Element Controller logger provides 4 verbosity levels. By default, nnf-ec + // will use the raw logger provided. If you are using zap defaults, this will + // only include the INFO and DEBUG levels. You can increase the defaults, by + // manually setting the zap.Options.Level, or using the -zap-log-level flag. + // + // You can decrease the log level of nnf-ec by adding verbosity levels to the + // logger. For example, `logger = logger.V(1)` would reduce the logging level + // by 1 in all nnf-ec logs. Unfortunately, the inverse is not true; one cannot + // increase the log level of nnf-ec as the `logger.V` method does not support + // negative values. + logger := r.RawLog.WithName("ec") + // Start the NNF Element Controller - c := nnfec.NewController(r.Options) + c := nnfec.NewController(r.Options).WithLogger(logger) + if err := c.Init(&ec.Options{Http: !testing, Port: nnfec.Port}); err != nil { return err } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 0beccd9f8..82d7afb1c 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -222,6 +222,7 @@ var _ = BeforeSuite(func() { Client: k8sManager.GetClient(), Scheme: testEnv.Scheme, Options: nnf.NewMockOptions(false), + RawLog: ctrl.Log, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/go.mod b/go.mod index 6ec20a420..d9e7bf990 100644 --- a/go.mod +++ b/go.mod @@ -5,38 +5,39 @@ go 1.19 require ( github.com/HewlettPackard/dws v0.0.0-20230224185520-92a8916979a7 github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 - github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f + github.com/NearNodeFlash/nnf-ec v0.0.0-20230228220541-47eb7a4dbe07 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/google/uuid v1.3.0 - github.com/onsi/ginkgo/v2 v2.2.0 - github.com/onsi/gomega v1.20.2 + github.com/onsi/ginkgo/v2 v2.8.4 + github.com/onsi/gomega v1.27.1 github.com/prometheus/client_golang v1.13.0 - go.uber.org/zap v1.23.0 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 + go.uber.org/zap v1.24.0 + golang.org/x/sync v0.1.0 k8s.io/api v0.25.2 - k8s.io/apimachinery v0.25.2 + k8s.io/apimachinery v0.26.1 k8s.io/client-go v0.25.2 - k8s.io/mount-utils v0.25.2 + k8s.io/mount-utils v0.26.1 sigs.k8s.io/controller-runtime v0.13.0 ) require ( - cloud.google.com/go/compute v1.10.0 // indirect + cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/HewlettPackard/structex v1.0.2 // indirect + github.com/HewlettPackard/structex v1.0.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/badger/v3 v3.2103.1 // indirect - github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dgraph-io/badger/v3 v3.2103.5 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -45,23 +46,25 @@ require ( github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v2.0.0+incompatible // indirect + github.com/google/flatbuffers v23.1.21+incompatible // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.13.5 // indirect + github.com/klauspost/compress v1.16.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -72,35 +75,35 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/rs/cors v1.8.0 // indirect + github.com/rs/cors v1.8.3 // indirect github.com/senseyeio/duration v0.0.0-20180430131211-7c2a214ada46 // indirect - github.com/sigurn/crc8 v0.0.0-20160107002456-e55481d6f45c // indirect - github.com/sigurn/utils v0.0.0-20190728110027-e1fefb11a144 // indirect + github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - go.chromium.org/luci v0.0.0-20210915061045-7722a2154c29 // indirect - go.opencensus.io v0.23.0 // indirect + go.chromium.org/luci v0.0.0-20230227223707-c4460eb434d8 // indirect + go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect - golang.org/x/net v0.0.0-20220927171203-f486391704dc // indirect - golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect - golang.org/x/term v0.0.0-20220919170432-7a66f970e087 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.5.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/oauth2 v0.4.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.6.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/appengine v1.6.8-0.20221117013220-504804fb50de // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.25.2 // indirect k8s.io/component-base v0.25.2 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20220928191237-829ce0c27909 // indirect - k8s.io/utils v0.0.0-20220922133306-665eaaec4324 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + k8s.io/klog/v2 v2.90.0 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index e0f01f045..1e921cea3 100644 --- a/go.sum +++ b/go.sum @@ -13,59 +13,27 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.92.1/go.mod h1:cMc7asehN84LBi1JGTHo4n8wuaGuNAZ7lR7b1YNJBrE= -cloud.google.com/go v0.92.2/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.92.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.22.0/go.mod h1:8c8KM5gUHw1doiZR4FGIwTBEmXC/xyM3vqmcvTXpUgc= -cloud.google.com/go/bigtable v1.10.1/go.mod h1:cyHeKlx6dcZCO0oSQucYdauseD8kIENGuDOJPKMCVg8= -cloud.google.com/go/cloudtasks v0.1.0/go.mod h1:jw28nXttHWt/e2wWiTCvA+q76y0rFb8boKyHUu33/+M= -cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4= -cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/datacatalog v0.1.0/go.mod h1:MI16U99JCHsfQJtEA4kIsGlWiaTljiRinWYu78at7ks= +cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.5.0/go.mod h1:RGUNM0FFAVkYA94BLTxoXBgfIyY1Riq67TwaBXH0lwc= -cloud.google.com/go/errorreporting v0.1.0/go.mod h1:cZSiBMvrnl0X13pD9DwKf9sQ8Eqy3EzHqkyKBZxiIrM= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/kms v0.1.0/go.mod h1:8Qp8PCAypHg4FdmlyW1QRAv09BGQ9Uzh7JnmIZxPk+c= -cloud.google.com/go/kms v0.2.0/go.mod h1:CBIzEY1pzeYnGSN3fwc7OHerywU5LlL0hoHLtGIxW1A= -cloud.google.com/go/logging v1.4.2/go.mod h1:jco9QZSx8HiVVqLJReq7z7bVdj0P1Jb9PDFs63T+axo= -cloud.google.com/go/monitoring v0.2.0/go.mod h1:K/JoZWY3xszHf38AMkzZGx1n5eT1/57ilElGMpESsEE= -cloud.google.com/go/profiler v0.1.0/go.mod h1:D7S7LV/zKbRWkOzYL1b5xytpqt8Ikd/v/yvf1/Tx2pQ= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.17.0/go.mod h1:bBIeYx9ftf/hr7eoSUim6cRaOYZE/hHuigwdwLLByi8= -cloud.google.com/go/secretmanager v0.1.0/go.mod h1:3nGKHvnzDUVit7U0S9KAKJ4aOsO1xtwRG+7ey5LK1bM= -cloud.google.com/go/spanner v1.25.0/go.mod h1:kQUft3x355hzzaeFbObjsvkzZDgpDkesp3v75WBnI8w= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.16.1/go.mod h1:LaNorbty3ehnU3rEjXSNV/NRgQA0O8Y+uh6bPe5UOk4= -cloud.google.com/go/trace v0.1.0/go.mod h1:wxEwsoeRVPbeSkt7ZC9nWCgmoKQRAoySN7XHW2AmI7g= -contrib.go.opencensus.io/exporter/stackdriver v0.13.8/go.mod h1:huNtlWx75MwO7qMs0KrMxPZXzNNWebav1Sq/pm02JdQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -85,96 +53,67 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/HewlettPackard/dws v0.0.0-20230224185520-92a8916979a7 h1:wh2jAzQHY2WaT0eVYDtIo6hCbotQ5yUvIwe3s90Wob0= github.com/HewlettPackard/dws v0.0.0-20230224185520-92a8916979a7/go.mod h1:ut/okqv7uImSeheKFGZQe07U1d6J9lluvw2eecRqiWU= -github.com/HewlettPackard/structex v1.0.2 h1:p2EH/p6zvUd5fSa0onudAUvLWdKsvQSRP0jGKeblA5E= -github.com/HewlettPackard/structex v1.0.2/go.mod h1:3frC4RY/cPsP/4+N8rkxsNAGlQwHV+zDC7qvrN+N+rE= -github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/HewlettPackard/structex v1.0.4 h1:RVTdN5FWhDWr1IkjllU8wxuLjISo4gr6u5ryZpzyHcA= +github.com/HewlettPackard/structex v1.0.4/go.mod h1:3frC4RY/cPsP/4+N8rkxsNAGlQwHV+zDC7qvrN+N+rE= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0 h1:upOQcpaEDmgLxoK7SZeqzGOQs8raxd9/paJYmBS/3/M= github.com/NearNodeFlash/lustre-fs-operator v0.0.0-20220929204230-5dcfe552c9e0/go.mod h1:1SLPG4mFeuISXS/GoPtHV541AqR6e9n1Fd9KEnuTDIU= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f h1:Dr/JRcUA0N8ZKbUnXx/J6pHIPfVT7rrzFFFKyelRe8E= -github.com/NearNodeFlash/nnf-ec v0.0.0-20230221180225-1dce5b1f270f/go.mod h1:s6Gp5d88rhMFcqrkF/Ds8D+n6GMAv9iw004YhAL3Neo= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230228220541-47eb7a4dbe07 h1:e2HfGFA3rqAxhLbvfB6A5PeaLUYAsVEV5GyufvmPVys= +github.com/NearNodeFlash/nnf-ec v0.0.0-20230228220541-47eb7a4dbe07/go.mod h1:ZeU6daZ/sVfzcN1DPmMNmiMwnVJy69bWs14l0u1Va4o= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE= -github.com/alecthomas/kong v0.2.17/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.40.42/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/bazelbuild/buildtools v0.0.0-20210911013817-37179d5767a1/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= -github.com/bazelbuild/remote-apis v0.0.0-20210718193713-0ecef08215cf/go.mod h1:ry8Y6CkQqCVcYsjPOlLXDX2iRVjOnjogdNwhvHmRcz8= -github.com/bazelbuild/remote-apis v0.0.0-20210812183132-3e816456ee28/go.mod h1:ry8Y6CkQqCVcYsjPOlLXDX2iRVjOnjogdNwhvHmRcz8= -github.com/bazelbuild/remote-apis-sdks v0.0.0-20210909182119-af1232ee0d79/go.mod h1:50ZasLDlgIh6V79vGr/vS7K8LXZC9xjJFJX8QCrxNhE= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/danjacques/gofslock v0.0.0-20200623023034-5d0bd0fa6ef0/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v3 v3.2103.1 h1:zaX53IRg7ycxVlkd5pYdCeFp1FynD6qBGQoQql3R3Hk= -github.com/dgraph-io/badger/v3 v3.2103.1/go.mod h1:dULbq6ehJ5K0cGW/1TQ9iSfUk0gbSiToDWmWmTsJ53E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -182,17 +121,12 @@ github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCv github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -200,12 +134,10 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= @@ -220,14 +152,10 @@ github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXym github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -237,7 +165,6 @@ github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -250,8 +177,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -267,20 +192,16 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= -github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.1.21+incompatible h1:bUqzx/MXCDxuS0hRJL2EfjyZL3uQrPbMocUa8zGqsTA= +github.com/google/flatbuffers v23.1.21+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -290,11 +211,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -302,8 +221,6 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -311,157 +228,63 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210804190019-f964ff605595/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210827144239-02619b876842/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY= +github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/tink/go v1.6.1/go.mod h1:IGW53kTgag+st5yPhKKwJ6u2l+SSp5/v9XF7spovjlY= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20210901121439-eee08aaf2717/go.mod h1:0RnbP5ioI0nqRf3R9iK3iQaUJgsn0htlZEHCMn8FSfw= -github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= -github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= -github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= -github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= -github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/luci/gtreap v0.0.0-20161228054646-35df89791e8f/go.mod h1:OjKOY0UvVOOH5nWXSIWTbQWESn8dDiGlaEZx6IAsWhU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/maruel/subcommands v1.1.0/go.mod h1:b25AG9Eho2Rs1NUPAPAYBFy1B5y63QMxw/2WmLGO8m8= -github.com/maruel/ut v1.0.2/go.mod h1:RV8PwPD9dd2KFlnlCc/DB2JVvkXmyaalfc5xvmSrRSs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/sys/mountinfo v0.6.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -471,42 +294,17 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mostynb/zstdpool-syncpool v0.0.7/go.mod h1:YpzqIpN8xvRZZvemem7CMLPWkjuaKR37MnkQruSj6aw= -github.com/mostynb/zstdpool-syncpool v0.0.10/go.mod h1:BmhpjzZxG8KCduFi0N/Do6j9w+JYt6vR2cM8J9AwujI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= -github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI= -github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= -github.com/onsi/gomega v1.20.2/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/onsi/ginkgo/v2 v2.8.4 h1:gf5mIQ8cLFieruNLAdgijHF1PYfLphKm2dxxcUtcqK0= +github.com/onsi/ginkgo/v2 v2.8.4/go.mod h1:427dEDQZkDKsBvCjc2A/ZPefhKxsTTrsQegMlayL730= +github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754= +github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -515,9 +313,7 @@ github.com/pkg/term v1.1.0 h1:xIAAdCMh3QIAy+5FrE8Ad8XoDhEU4ufwbaSozViP9kk= github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -529,8 +325,6 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -538,139 +332,94 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/protocolbuffers/txtpbfmt v0.0.0-20210910155415-af4447816e4d/go.mod h1:KbKfKPy2I6ecOIGA9apfheFv14+P3RSmmQvshofQyMY= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= -github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= +github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= +github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/senseyeio/duration v0.0.0-20180430131211-7c2a214ada46 h1:Dz0HrI1AtNSGCE8LXLLqoZU4iuOJXPWndenCsZfstA8= github.com/senseyeio/duration v0.0.0-20180430131211-7c2a214ada46/go.mod h1:is8FVkzSi7PYLWEXT5MgWhglFsyyiW8ffxAoJqfuFZo= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sigurn/crc8 v0.0.0-20160107002456-e55481d6f45c h1:hk0Jigjfq59yDMgd6bzi22Das5tyxU0CtOkh7a9io84= -github.com/sigurn/crc8 v0.0.0-20160107002456-e55481d6f45c/go.mod h1:cyrWuItcOVIGX6fBZ/G00z4ykprWM7hH58fSavNkjRg= -github.com/sigurn/utils v0.0.0-20190728110027-e1fefb11a144 h1:ccb8W1+mYuZvlpn/mJUMAbsFHTMCpcJBS78AsBQxNcY= -github.com/sigurn/utils v0.0.0-20190728110027-e1fefb11a144/go.mod h1:VRI4lXkrUH5Cygl6mbG1BRUfMMoT2o8BkrtBDUAm+GU= +github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f h1:1R9KdKjCNSd7F8iGTxIpoID9prlYH8nuNYKt0XvweHA= +github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f/go.mod h1:vQhwQ4meQEDfahT5kd61wLAF5AAeh5ZPLVI4JJ/tYo8= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/xtgo/set v1.0.0/go.mod h1:d3NHzGzSa0NmB2NhFyECA+QdRp29oEn2xbT+TpeFoM8= -github.com/yosuke-furukawa/json5 v0.1.1/go.mod h1:sw49aWDqNdRJ6DYUtIQiaA3xyj2IL9tjeNYmX2ixwcU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.chromium.org/luci v0.0.0-20210915061045-7722a2154c29 h1:TgvWIQO0I0E7pKVLq/ARDhT2p2kHGKLtZhAN0EtUxiI= -go.chromium.org/luci v0.0.0-20210915061045-7722a2154c29/go.mod h1:gk1qa1CBZAdya1PqugRJjkwk8LIymAoB5wf5UA7NrFk= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.chromium.org/luci v0.0.0-20230227223707-c4460eb434d8 h1:Rcq1XvY7FYkQoDT+MvdePfC2aNIaTxGSYM+wa034w48= +go.chromium.org/luci v0.0.0-20230227223707-c4460eb434d8/go.mod h1:vTpW7gzqLQ9mhMO3qOY76wkndTysy4FStdksL0l4JsQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= -go.starlark.net v0.0.0-20210901212718-87f333178d59/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -693,8 +442,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -703,18 +450,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -736,52 +474,30 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= -golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -792,22 +508,14 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -815,18 +523,11 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -841,77 +542,49 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210507014357-30e306a8bba5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= -golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -922,7 +595,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -949,21 +621,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -986,32 +646,16 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.37.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8-0.20221117013220-504804fb50de h1:MvEeYmzkzk0Rsw+ceqy28aIJN7Mum+4aYqBwCMqYNug= +google.golang.org/appengine v1.6.8-0.20221117013220-504804fb50de/go.mod h1:BbwiCY3WCmCUKOJTrX5NwgQzew1c32w3kxa6Sxvs0cQ= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1040,47 +684,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210601144548-a796c710e9b6/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210824181836-a4879c3d0e89/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210825212027-de86158e7fda/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210827211047-25e5f791fe06/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1090,20 +697,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1120,24 +717,14 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1164,31 +751,27 @@ k8s.io/api v0.25.2 h1:v6G8RyFcwf0HR5jQGIAYlvtRNrxMJQG1xJzaSeVnIS8= k8s.io/api v0.25.2/go.mod h1:qP1Rn4sCVFwx/xIhe+we2cwBLTXNcheRyYXwajonhy0= k8s.io/apiextensions-apiserver v0.25.2 h1:8uOQX17RE7XL02ngtnh3TgifY7EhekpK+/piwzQNnBo= k8s.io/apiextensions-apiserver v0.25.2/go.mod h1:iRwwRDlWPfaHhuBfQ0WMa5skdQfrE18QXJaJvIDLvE8= -k8s.io/apimachinery v0.25.2 h1:WbxfAjCx+AeN8Ilp9joWnyJ6xu9OMeS/fsfjK/5zaQs= -k8s.io/apimachinery v0.25.2/go.mod h1:hqqA1X0bsgsxI6dXsJ4HnNTBOmJNxyPp8dw3u2fSHwA= +k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= +k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= k8s.io/client-go v0.25.2 h1:SUPp9p5CwM0yXGQrwYurw9LWz+YtMwhWd0GqOsSiefo= k8s.io/client-go v0.25.2/go.mod h1:i7cNU7N+yGQmJkewcRD2+Vuj4iz7b30kI8OcL3horQ4= k8s.io/component-base v0.25.2 h1:Nve/ZyHLUBHz1rqwkjXm/Re6IniNa5k7KgzxZpTfSQY= k8s.io/component-base v0.25.2/go.mod h1:90W21YMr+Yjg7MX+DohmZLzjsBtaxQDDwaX4YxDkl60= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20220928191237-829ce0c27909 h1:q/70bz7C1/LGuQu/JBX7Fpi55CwcCts/wbvlehe0RRo= -k8s.io/kube-openapi v0.0.0-20220928191237-829ce0c27909/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/mount-utils v0.25.0/go.mod h1:WTYq8Ev/JrnkqK2h1jFUnC8qWGuqzMb9XDC+Lu3WNU0= -k8s.io/mount-utils v0.25.2 h1:ZNIHSaPaaPqFqtZzaTbgH3oHKoWZHoNBnTKafQRj6A4= -k8s.io/mount-utils v0.25.2/go.mod h1:WTYq8Ev/JrnkqK2h1jFUnC8qWGuqzMb9XDC+Lu3WNU0= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220922133306-665eaaec4324 h1:i+xdFemcSNuJvIfBlaYuXgRondKxK4z4prVPKzEaelI= -k8s.io/utils v0.0.0-20220922133306-665eaaec4324/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M= +k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/mount-utils v0.26.1 h1:deN1IBPyi5UFEAgQYXBEDUejzQUNzRC1ML7BUMWljzA= +k8s.io/mount-utils v0.26.1/go.mod h1:au99w4FWU5ZWelLb3Yx6kJc8RZ387IyWVM9tN65Yhxo= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.13.0 h1:iqa5RNciy7ADWnIc8QxCbOX5FEKVR3uxVxKHRMc2WIQ= sigs.k8s.io/controller-runtime v0.13.0/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/main.go b/main.go index 2750085dd..96cc91226 100644 --- a/main.go +++ b/main.go @@ -27,10 +27,9 @@ import ( // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. + "go.uber.org/zap/zapcore" _ "k8s.io/client-go/plugin/pkg/client/auth" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" kruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -38,7 +37,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" - zapcr "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" dwsv1alpha1 "github.com/HewlettPackard/dws/api/v1alpha1" @@ -86,15 +85,15 @@ func main() { nnfopts := nnf.BindFlags(flag.CommandLine) - opts := zapcr.Options{ + opts := zap.Options{ Development: true, + TimeEncoder: zapcore.ISO8601TimeEncoder, } opts.BindFlags(flag.CommandLine) + flag.Parse() - encoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) - zaplogger := zapcr.New(zapcr.WriteTo(os.Stdout), zapcr.Encoder(encoder)) - ctrl.SetLogger(zaplogger) + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog.Info("GOMAXPROCS", "value", runtime.GOMAXPROCS(0)) @@ -195,6 +194,7 @@ func (c *nodeLocalController) SetupReconcilers(mgr manager.Manager, opts *nnf.Op Scheme: mgr.GetScheme(), NamespacedName: types.NamespacedName{Name: controllers.NnfNodeECDataResourceName, Namespace: os.Getenv("NNF_NODE_NAME")}, Options: opts, + RawLog: ctrl.Log, }).SetupWithManager(mgr); err != nil { return err } diff --git a/vendor/cloud.google.com/go/compute/internal/version.go b/vendor/cloud.google.com/go/compute/internal/version.go new file mode 100644 index 000000000..7dd6a0aa8 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/internal/version.go @@ -0,0 +1,18 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +// Version is the current tagged release of the library. +const Version = "1.15.1" diff --git a/vendor/cloud.google.com/go/compute/metadata/CHANGES.md b/vendor/cloud.google.com/go/compute/metadata/CHANGES.md new file mode 100644 index 000000000..06b957349 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/CHANGES.md @@ -0,0 +1,19 @@ +# Changes + +## [0.2.3](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.2...compute/metadata/v0.2.3) (2022-12-15) + + +### Bug Fixes + +* **compute/metadata:** Switch DNS lookup to an absolute lookup ([119b410](https://github.com/googleapis/google-cloud-go/commit/119b41060c7895e45e48aee5621ad35607c4d021)), refs [#7165](https://github.com/googleapis/google-cloud-go/issues/7165) + +## [0.2.2](https://github.com/googleapis/google-cloud-go/compare/compute/metadata/v0.2.1...compute/metadata/v0.2.2) (2022-12-01) + + +### Bug Fixes + +* **compute/metadata:** Set IdleConnTimeout for http.Client ([#7084](https://github.com/googleapis/google-cloud-go/issues/7084)) ([766516a](https://github.com/googleapis/google-cloud-go/commit/766516aaf3816bfb3159efeea65aa3d1d205a3e2)), refs [#5430](https://github.com/googleapis/google-cloud-go/issues/5430) + +## [0.1.0] (2022-10-26) + +Initial release of metadata being it's own module. diff --git a/vendor/cloud.google.com/go/compute/metadata/LICENSE b/vendor/cloud.google.com/go/compute/metadata/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/cloud.google.com/go/compute/metadata/README.md b/vendor/cloud.google.com/go/compute/metadata/README.md new file mode 100644 index 000000000..f940fb2c8 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/README.md @@ -0,0 +1,27 @@ +# Compute API + +[![Go Reference](https://pkg.go.dev/badge/cloud.google.com/go/compute.svg)](https://pkg.go.dev/cloud.google.com/go/compute/metadata) + +This is a utility library for communicating with Google Cloud metadata service +on Google Cloud. + +## Install + +```bash +go get cloud.google.com/go/compute/metadata +``` + +## Go Version Support + +See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) +section in the root directory's README. + +## Contributing + +Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) +document for details. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. See +[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. diff --git a/vendor/cloud.google.com/go/compute/metadata/metadata.go b/vendor/cloud.google.com/go/compute/metadata/metadata.go index 50538b1d3..c17faa142 100644 --- a/vendor/cloud.google.com/go/compute/metadata/metadata.go +++ b/vendor/cloud.google.com/go/compute/metadata/metadata.go @@ -70,6 +70,7 @@ func newDefaultHTTPClient() *http.Client { Timeout: 2 * time.Second, KeepAlive: 30 * time.Second, }).Dial, + IdleConnTimeout: 60 * time.Second, }, Timeout: 5 * time.Second, } @@ -146,7 +147,7 @@ func testOnGCE() bool { go func() { resolver := &net.Resolver{} - addrs, err := resolver.LookupHost(ctx, "metadata.google.internal") + addrs, err := resolver.LookupHost(ctx, "metadata.google.internal.") if err != nil || len(addrs) == 0 { resc <- false return diff --git a/vendor/cloud.google.com/go/compute/metadata/tidyfix.go b/vendor/cloud.google.com/go/compute/metadata/tidyfix.go new file mode 100644 index 000000000..4cef48500 --- /dev/null +++ b/vendor/cloud.google.com/go/compute/metadata/tidyfix.go @@ -0,0 +1,23 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file, and the {{.RootMod}} import, won't actually become part of +// the resultant binary. +//go:build modhack +// +build modhack + +package metadata + +// Necessary for safely adding multi-module repo. See: https://github.com/golang/go/wiki/Modules#is-it-possible-to-add-a-module-to-a-multi-module-repository +import _ "cloud.google.com/go/compute/internal" diff --git a/vendor/github.com/HewlettPackard/structex/README.md b/vendor/github.com/HewlettPackard/structex/README.md index 18a9d7154..66faac255 100644 --- a/vendor/github.com/HewlettPackard/structex/README.md +++ b/vendor/github.com/HewlettPackard/structex/README.md @@ -44,6 +44,13 @@ func main() { ``` ## Annotation Format +### Endianness + +Little-endian (default) or big-endian tags support for integer and unsigned integer types +`little:""` + +`big:""` + ### Bitfield Bitfields define a structure field with an explicit size in bits. They are analogous to bit fields in the C specification. @@ -88,7 +95,7 @@ buffer. If truncate is not specified and the provided buffer is of smaller size `truncate:""` -### Alignment (Not Yet Implemented) +### Alignment Annotations can specified the byte-alignment requirement for structure fields. Analogous to the alignas specifier in C. Can only be applied to non-bitfield structure fields. @@ -104,6 +111,8 @@ The tags documented above are abbreviated for ease of use; if desired, the full Examples ``` +`structex:"little"` +`structex:"big"` `structex:"bitfield='3'"` `structex:"bitfield='3,reserved'"` `structex:"countOf='D'"` @@ -114,7 +123,7 @@ Examples ``` ## Performance -`structex` places code and structure definition readability ahead of any pack/unpack performance. In brief, performance is abysmal. +`structex` places code readability ahead of any pack/unpack performance. Reflection is used to analyze the structure definitions, which itself carries heavy overhead. Parsing the tags also takes time. For those looking for more performant code, consider [code generation](https://github.com/golang/text/blob/master/internal/gen/bitfield/bitfield.go) or a [cache based](https://github.com/lunixbochs/struc) solutions diff --git a/vendor/github.com/HewlettPackard/structex/decoder.go b/vendor/github.com/HewlettPackard/structex/decoder.go index d8be4d5db..17f2424a9 100644 --- a/vendor/github.com/HewlettPackard/structex/decoder.go +++ b/vendor/github.com/HewlettPackard/structex/decoder.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "math" + "math/bits" "reflect" ) @@ -40,45 +41,50 @@ type decoder struct { func (d *decoder) read(nbits uint64) (uint64, error) { if nbits == 0 { - return 0, fmt.Errorf("Unsupported zero bit operation") + return 0, fmt.Errorf("unsupported zero bit operation") } - if nbits < 8 { - if d.bitOffset == 0 { - b, err := d.reader.ReadByte() - if err != nil { - return 0, err - } - d.currentByte = b - } + if nbits > 64 { + return 0, fmt.Errorf("bitfield exceeds 64-bit limitation") + } - if nbits > uint64(8-d.bitOffset) { - return 0, fmt.Errorf("Insufficient bit count for reading") - } + var value uint64 = 0 + var offset uint64 = 0 - mask := uint(math.Pow(2, float64(nbits)) - 1) - value := uint(d.currentByte>>d.bitOffset) & mask + // Check for carry-over bits from previous bitfields + if d.bitOffset != 0 { + mask := uint8(math.Pow(2, float64(nbits)) - 1) + value = uint64((d.currentByte >> d.bitOffset) & mask) - d.bitOffset += nbits - if d.bitOffset >= 8 { + if d.bitOffset+nbits < 8 { + d.bitOffset += nbits + return value, nil + } else { + offset = 8 - d.bitOffset + nbits = nbits - (8 - d.bitOffset) d.bitOffset = 0 } - - return uint64(value), nil - } - - if nbits%8 != 0 { - return 0, fmt.Errorf("Unsupported bit span of %d bits", nbits) } - var value uint64 = 0 - for i := uint64(0); i < nbits; i += 8 { + for nbits != 0 { b, err := d.reader.ReadByte() if err != nil { return 0, err } - value |= uint64(b) << i + d.currentByte = b + d.byteOffset += 1 + + if nbits < 8 { + mask := uint8(math.Pow(2, float64(nbits)) - 1) + value |= uint64(d.currentByte&mask) << offset + d.bitOffset += nbits + return value, nil + } else { + value |= uint64(d.currentByte) << offset + offset += 8 + nbits -= 8 + } } return value, nil @@ -92,7 +98,8 @@ func (d *decoder) readValue(value reflect.Value, tags *tags) (uint64, error) { } nbits := uint64(0) - if value.Kind() == reflect.Bool { + kind := value.Kind() + if kind == reflect.Bool { nbits = 1 } else { nbits = uint64(value.Type().Bits()) @@ -113,6 +120,17 @@ func (d *decoder) readValue(value reflect.Value, tags *tags) (uint64, error) { return 0, err } + if tags != nil && tags.endian == big { + switch kind { + case reflect.Uint16, reflect.Int16: + v = uint64(bits.ReverseBytes16(uint16(v))) + case reflect.Uint32, reflect.Int32, reflect.Uint, reflect.Int: + v = uint64(bits.ReverseBytes32(uint32(v))) + case reflect.Uint64, reflect.Int64: + v = bits.ReverseBytes64(v) + } + } + switch value.Kind() { case reflect.Bool: value.SetBool(v == 1) @@ -127,6 +145,22 @@ func (d *decoder) readValue(value reflect.Value, tags *tags) (uint64, error) { return v, nil } +func (d *decoder) align(val alignment) error { + if d.bitOffset != 0 { + if _, err := d.read(8 - d.bitOffset); err != nil { + return err + } + } + + for d.byteOffset%uint64(val) != 0 { + if _, err := d.read(8); err != nil { + return err + } + } + + return nil +} + func (d *decoder) field(val reflect.Value, tags *tags) error { _, err := d.readValue(val, tags) return err diff --git a/vendor/github.com/HewlettPackard/structex/encoder.go b/vendor/github.com/HewlettPackard/structex/encoder.go index f9dd591fd..2d3dc6a58 100644 --- a/vendor/github.com/HewlettPackard/structex/encoder.go +++ b/vendor/github.com/HewlettPackard/structex/encoder.go @@ -26,6 +26,7 @@ import ( "fmt" "io" "math" + "math/bits" "reflect" ) @@ -38,39 +39,38 @@ type encoder struct { func (e *encoder) write(value uint64, nbits uint64) error { - if nbits > 1 && value > uint64(math.Pow(2, float64(nbits)))-1 { - return fmt.Errorf("Value %d will overflow bitfield of %d bits", value, nbits) + if nbits > 1 && value > math.MaxUint64 { + return fmt.Errorf("Value %d (%#x) will overflow bitfield of %d bits", value, value, nbits) } - if nbits < 8 { - shift := e.bitOffset - if shift+nbits > 8 { - return fmt.Errorf("Bitfield not allowed to overflow single byte") - } + // Write any bits that might be part of previous bitfield definitions + if e.bitOffset != 0 { + e.currentByte |= uint8(value << e.bitOffset) - e.currentByte |= uint8(value << shift) - e.bitOffset += nbits + remainingBits := 8 - e.bitOffset + if nbits < remainingBits { + e.bitOffset += nbits + return nil + } else { + value = value >> remainingBits + nbits -= remainingBits - if e.bitOffset == 8 { e.writeByte(e.currentByte) + e.currentByte = uint8(value) e.bitOffset = 0 - e.currentByte = 0 } - - return nil - } - - if e.bitOffset != 0 { - return fmt.Errorf("Cannot span multi-byte bitfield") } - if nbits%8 != 0 { - return fmt.Errorf("Can only write 8-bit byte multiples") - } + for nbits != 0 { + e.currentByte = uint8(value) + if nbits < 8 { + e.bitOffset += nbits + return nil + } else { + e.writeByte(e.currentByte) - for b := uint64(0); b < nbits/8; b++ { - if err := e.writeByte(uint8(value >> (b * 8))); err != nil { - return err + value = value >> 8 + nbits -= 8 } } @@ -86,12 +86,39 @@ func (e *encoder) writeByte(value uint8) error { return nil } +func (e *encoder) align(val alignment) error { + if e.bitOffset != 0 { + if err := e.write(0, 8-e.bitOffset); err != nil { + return err + } + } + + for e.byteOffset%uint64(val) != 0 { + if err := e.write(0, 8); err != nil { + return err + } + } + + return nil +} + func (e *encoder) field(val reflect.Value, tags *tags) error { v := getValue(val) if tags == nil { return e.write(v, uint64(val.Type().Bits())) } + if tags.endian == big { + switch val.Kind() { + case reflect.Uint16, reflect.Int16: + v = uint64(bits.ReverseBytes16(uint16(v))) + case reflect.Uint32, reflect.Int32, reflect.Uint, reflect.Int: + v = uint64(bits.ReverseBytes32(uint32(v))) + case reflect.Uint64, reflect.Int64: + v = bits.ReverseBytes64(v) + } + } + return e.write(v, tags.bitfield.nbits) } diff --git a/vendor/github.com/HewlettPackard/structex/size.go b/vendor/github.com/HewlettPackard/structex/size.go index 7f97e61f4..9c64aecb2 100644 --- a/vendor/github.com/HewlettPackard/structex/size.go +++ b/vendor/github.com/HewlettPackard/structex/size.go @@ -47,6 +47,22 @@ func (s *sizer) addBits(nbits uint64) error { return nil } +func (s *sizer) align(val alignment) error { + if s.nbits != 0 { + if err := s.addBits(8 - s.nbits); err != nil { + return err + } + } + + for s.nbytes % uint64(val) != 0 { + if err := s.addBits(8); err != nil { + return err + } + } + + return nil +} + func (s *sizer) field(val reflect.Value, tags *tags) error { if tags == nil { return s.addBits(uint64(val.Type().Bits())) diff --git a/vendor/github.com/HewlettPackard/structex/tags.go b/vendor/github.com/HewlettPackard/structex/tags.go index e7deb58e7..32c66b375 100644 --- a/vendor/github.com/HewlettPackard/structex/tags.go +++ b/vendor/github.com/HewlettPackard/structex/tags.go @@ -29,6 +29,13 @@ import ( "strings" ) +type endian int + +const ( + little endian = 0 + big endian = 1 +) + type bitfield struct { nbits uint64 reserved bool @@ -47,9 +54,10 @@ type layout struct { value uint64 } -type alignment int64 +type alignment uint64 type tags struct { + endian endian bitfield bitfield layout layout alignment alignment @@ -74,9 +82,10 @@ structures defined by the the structure extension values. */ func parseFieldTags(sf reflect.StructField) tags { t := tags{ + endian: little, bitfield: bitfield{0, false}, layout: layout{none, "", false, 0}, - alignment: -1, + alignment: 0, truncate: false, } @@ -170,6 +179,12 @@ func (t *tags) parseString(sf reflect.StructField, tagString string, opts parseO func (t *tags) add(sf reflect.StructField, key string, val string) { switch strings.ToLower(key) { + case "little": + t.endian = little + + case "big": + t.endian = big + case "bitfield": if nbs := strings.Split(val, ",")[0]; len(nbs) != 0 { nbits, err := strconv.ParseInt(nbs, 0, int(sf.Type.Bits())) diff --git a/vendor/github.com/HewlettPackard/structex/transcoder.go b/vendor/github.com/HewlettPackard/structex/transcoder.go index 47950076e..c35fb26e9 100644 --- a/vendor/github.com/HewlettPackard/structex/transcoder.go +++ b/vendor/github.com/HewlettPackard/structex/transcoder.go @@ -38,6 +38,7 @@ type stack struct { } type handler interface { + align(a alignment) error field(val reflect.Value, tags *tags) error layout(val reflect.Value, ref *tagReference) error array(t *transcoder, arr reflect.Value, tags *tags, ref *tagReference) error @@ -86,6 +87,12 @@ func (t *transcoder) transcode(val reflect.Value) error { tags := parseFieldTags(fieldTyp) + if tags.alignment != 0 { + if err := t.handler.align(tags.alignment); err != nil { + return err + } + } + switch fieldTyp.Type.Kind() { case reflect.Struct: @@ -111,11 +118,11 @@ func (t *transcoder) transcode(val reflect.Value) error { found := t.fieldByName(tags.layout.name) if !found.IsValid() || found.Type() == reflect.PtrTo(reflect.TypeOf(reflect.Invalid)) { - return fmt.Errorf("Cannot locate field name '%s'", tags.layout.name) + return fmt.Errorf("cannot locate referenced field '%s'", tags.layout.name) } if found.Kind() != reflect.Slice && found.Kind() != reflect.Array { - return fmt.Errorf("Referenced layout must be of type slice or array; Is of type %s", found.Kind().String()) + return fmt.Errorf("referenced layout must be of type slice or array; is of type %s", found.Kind().String()) } ref := &tagReference{ diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go index da64c61a8..d39a58ec6 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/nvme.go @@ -41,6 +41,8 @@ type Device struct { ops ops } +func (dev *Device) String() string { return dev.Path } + // UserIoCmd represents an NVMe User I/O Request // This is a copy of the C definition in '/include/linux/nvme_ioctl.h' struct nvme_user_io{} type UserIoCmd struct { @@ -84,6 +86,10 @@ type PassthruCmd struct { // AdminCmd aliases the PassthruCmd type AdminCmd = PassthruCmd +func (cmd *AdminCmd) String() string { + return fmt.Sprintf("OpCode: %s (%#02x)", AdminCommandOpCode(cmd.Opcode), cmd.Opcode) +} + // AdminCommandOpCode sizes the opcodes listed below type AdminCommandOpCode uint8 @@ -101,6 +107,33 @@ const ( FormatNvmOpCode AdminCommandOpCode = 0x80 ) +func (code AdminCommandOpCode) String() string { + switch code { + case GetLogPage: + return "Get Log Page" + case IdentifyOpCode: + return "Identify" + case SetFeatures: + return "Set Features" + case GetFeatures: + return "Get Features" + case NamespaceManagementOpCode: + return "Namespace Management" + case NamespaceAttachOpCode: + return "Namespace Attach/Detach" + case VirtualMgmtOpCode: + return "Virtualization Management" + case NvmeMiSend: + return "Management Interface Send" + case NvmeMiRecv: + return "Management Interface Recv" + case FormatNvmOpCode: + return "Format" + } + + return "UNKNOWN" +} + // StatusCode sizes the status codes listed below type StatusCode uint32 @@ -119,6 +152,17 @@ const ( NamespaceNotAttached StatusCode = 0x11a ) +func (sc StatusCode) String() string { + switch sc { + case NamespaceAlreadyAttached: + return "Namespace Already Attached" + case NamespaceNotAttached: + return "Namespace Not Attached" + } + + return "UNKNOWN" +} + // CommandError captures an error and details about handling type CommandError struct { StatusCode StatusCode @@ -128,11 +172,11 @@ type CommandError struct { } func (e *CommandError) Error() string { - return fmt.Sprintf("NVMe Status: %#03x CRD: %d More: %t DNR: %t", e.StatusCode, e.CommandRetryDelay, e.More, e.DoNotRetry) + return fmt.Sprintf("NVMe Status: %s (%#03x) CRD: %d More: %t DNR: %t", e.StatusCode, uint32(e.StatusCode), e.CommandRetryDelay, e.More, e.DoNotRetry) } // NewCommandError generates the default CommandError for use -func NewCommandError(status uint32) error { +func newCommandError(status uint32) error { return &CommandError{ StatusCode: StatusCode(status & StatusCodeMask), CommandRetryDelay: uint8((status & CommandRetryDelayMask) >> CommandRetryDelayShift), @@ -195,7 +239,12 @@ func Connect(dev *switchtec.Device, pdfid uint16) (*Device, error) { return nil, err } - return &Device{ops: &ops}, nil + device := &Device{ + Path: fmt.Sprintf("%#x@%s", pdfid, dev.Path), + ops: &ops, + } + + return device, nil } func connectDevice(dev *switchtec.Device, ops *switchOps) (err error) { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/switchtec.go b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/switchtec.go index dcd82e7a2..d61cb0a15 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/switchtec.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme/switchtec.go @@ -22,6 +22,7 @@ package nvme import ( "bytes" "encoding/binary" + "fmt" "unsafe" "github.com/HewlettPackard/structex" @@ -37,6 +38,26 @@ type switchOps struct { tunnelStatus switchtec.EpTunnelStatus } +type SwitchtecNvmeCommandError struct { + dev *Device + cmd *AdminCmd + err error +} + +func newSwitchtecNvmeCommandError(dev *Device, cmd *AdminCmd, err error) *SwitchtecNvmeCommandError { + return &SwitchtecNvmeCommandError{ + dev: dev, cmd: cmd, err: err, + } +} + +func (e *SwitchtecNvmeCommandError) Error() string { + return fmt.Sprintf("Device %s: Failed NVMe Command: %s: Error: %s", e.dev, e.cmd, e.err) +} + +func (e *SwitchtecNvmeCommandError) Unwrap() error { + return e.err +} + func (ops *switchOps) close() error { if ops.dev != nil { return ops.dev.Close() @@ -104,7 +125,7 @@ func (ops *switchOps) submitAdminPassthru(dev *Device, cmd *AdminCmd, data []byt rspLen := int(unsafe.Sizeof(rsp.Cqe)) + int(cmd.DataLen) rspData, err := ops.dev.NvmeAdminPassthru(ops.pdfid, reqData, rspLen) if err != nil { - return err + return newSwitchtecNvmeCommandError(dev, cmd, err) } if err := structex.DecodeByteBuffer(bytes.NewBuffer(rspData), rsp); err != nil { @@ -113,7 +134,7 @@ func (ops *switchOps) submitAdminPassthru(dev *Device, cmd *AdminCmd, data []byt status := (0xfffe0000 & rsp.Cqe[3]) >> 17 if status != 0 { - return NewCommandError(status) + return newSwitchtecNvmeCommandError(dev, cmd, newCommandError(status)) } cmd.Result = rsp.Cqe[0] diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/switchtec.go b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/switchtec.go index d15e82d04..06cfd5347 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/switchtec.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/switchtec/switchtec.go @@ -33,6 +33,8 @@ import ( // Device describing the switchtec device type Device struct { + Path string + file *os.File name string paxID int32 @@ -48,6 +50,23 @@ const ( PAXIdShift = 18 ) +type CommandError struct { + cmd Command + err error +} + +func newCommandError(cmd Command, err error) *CommandError { + return &CommandError{cmd: cmd, err: err} +} + +func (e *CommandError) Error() string { + return fmt.Sprintf("Switchtec Command: %s (%#x) Error: %s", e.cmd, e.cmd, e.err) +} + +func (e *CommandError) Unwrap() error { + return e.err +} + // Open returns the device such that it is accessible to other // API functions. func Open(path string) (*Device, error) { @@ -65,6 +84,7 @@ func Open(path string) (*Device, error) { syscall.SetNonblock(int(f.Fd()), false) var dev = new(Device) + dev.Path = path dev.file = f dev.name = path dev.ops = &charOps{} @@ -118,13 +138,13 @@ func (dev *Device) RunCommand(cmd Command, payload interface{}, response interfa defer dev.Unlock() if err := dev.ops.submitCommand(dev, cmd, payloadBuf.Bytes()); err != nil { - return err + return newCommandError(cmd, err) } responseBuf := structex.NewBuffer(response) if err := dev.ops.readResponse(dev, responseBuf.Bytes()); err != nil { - return err + return newCommandError(cmd, err) } if response == nil { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/controller.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/controller.go index 7d019fc7f..ea8f60049 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/controller.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/controller.go @@ -33,6 +33,8 @@ import ( "flag" "os" + log "github.com/sirupsen/logrus" + ec "github.com/NearNodeFlash/nnf-ec/pkg/ec" event "github.com/NearNodeFlash/nnf-ec/pkg/manager-event" fabric "github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric" @@ -41,7 +43,6 @@ import ( nvme "github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme" telemetry "github.com/NearNodeFlash/nnf-ec/pkg/manager-telemetry" "github.com/NearNodeFlash/nnf-ec/pkg/persistent" - log "github.com/sirupsen/logrus" ) const ( @@ -115,12 +116,7 @@ func NewController(opts *Options) *ec.Controller { persistent.StorageProvider = persistent.NewJsonFilePersistentStorageProvider(opts.json) } - return &ec.Controller{ - Name: Name, - Port: Port, - Version: Version, - Routers: NewDefaultApiRouters(switchCtrl, nvmeCtrl, nnfCtrl), - } + return ec.NewController(Name, Port, Version, NewDefaultApiRouters(switchCtrl, nvmeCtrl, nnfCtrl)) } // NewDefaultApiRouters - diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/ec.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/ec.go index 041b6b4f1..4c947f720 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/ec.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/ec/ec.go @@ -35,7 +35,7 @@ import ( "github.com/gorilla/mux" "github.com/rs/cors" - log "github.com/sirupsen/logrus" + logr "github.com/go-logr/logr" ) var ( @@ -62,7 +62,7 @@ type Router interface { Routes() Routes Name() string - Init() error + Init(Logger) error Start() error Close() error } @@ -70,18 +70,36 @@ type Router interface { // Routers - type Routers []Router +type Logger = logr.Logger + // Controller - type Controller struct { Name string Port int Version string Routers Routers + Log Logger options Options router *mux.Router processor ControllerProcessor } +func NewController(name string, port int, version string, routers Routers) *Controller { + return &Controller{ + Name: name, + Port: port, + Version: version, + Routers: routers, + Log: logr.Discard(), + } +} + +func (ctrl *Controller) WithLogger(log Logger) *Controller { + ctrl.Log = log + return ctrl +} + // Options - type Options struct { Http bool @@ -144,7 +162,7 @@ func (c *Controller) initialize(opts *Options) error { c.router = mux.NewRouter().StrictSlash(true) for _, api := range c.Routers { - if err := api.Init(); err != nil { + if err := api.Init(c.Log); err != nil { return err } } @@ -178,19 +196,20 @@ type HttpControllerProcessor struct { } func (p *HttpControllerProcessor) Run(c *Controller, options Options) error { + log := c.Log if options.Log { c.router.Use(func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var rlog = log.WithFields(log.Fields{ - "Method": r.Method, - "URL": r.RequestURI, - }) + var log = log.WithValues( + "method", r.Method, + "url", r.RequestURI, + ) if options.Verbose && r.Method == POST_METHOD { body, _ := ioutil.ReadAll(r.Body) r.Body = ioutil.NopCloser(bytes.NewBuffer(body)) - rlog.WithField("Request", string(body)).Infof("Http Request: %s %s", r.Method, r.URL) + log.WithValues("request", string(body)).Info("Http Request") } start := time.Now() @@ -204,13 +223,14 @@ func (p *HttpControllerProcessor) Run(c *Controller, options Options) error { w.Write(recorder.Body.Bytes()) if options.Verbose { - rlog = rlog.WithField("Response", recorder.Body.String()) + log = log.WithValues("response", recorder.Body.String()) } - rlog.WithFields(log.Fields{ - "Status": status, - "ElapsedTime": time.Since(start).String(), - }).Infof("Http Response: %d (%s)", status, http.StatusText(status)) + log.WithValues( + "status", status, + "statusText", http.StatusText(status), + "elapsedTime", time.Since(start).String(), + ).Info("Http Response") }) }) } @@ -225,9 +245,9 @@ func (p *HttpControllerProcessor) Run(c *Controller, options Options) error { Handler: crs.Handler(c.router), } - log.Infof("Starting HTTP Server at %s", p.server.Addr) + log.Info("Starting HTTP Server", "address", p.server.Addr) if err := p.server.ListenAndServe(); err != http.ErrServerClosed { - log.Fatalf("ListenAndServer Failed: Error: %v", err) + log.Error(err, "ListenAndServer Failed") return err } @@ -303,15 +323,18 @@ func (c *Controller) Init(opts *Options) error { // Run - Run a controller with standard behavior - that is with GRPC server and // request handling that operates by unpacking the GRPC request and // forwardining it to the element controller's handlers. -func (c *Controller) Run() { +func (c *Controller) Run() error { if c.processor == nil { - log.Fatalf("Controller %s must call Init() prior to run", c.Name) + return fmt.Errorf("controller processor uninitialized") } + c.Attach(c.router, nil) if err := c.processor.Run(c, c.options); err != nil { - log.WithError(err).Fatalf("%s failed to run", c.Name) + return err } + + return nil } // Send a request to the element controller @@ -348,8 +371,7 @@ func (c *Controller) Close() { } // EncodeResponse - -func EncodeResponse(s interface{}, err error, w http.ResponseWriter) { - +func EncodeResponse(s interface{}, err error, w http.ResponseWriter) error { if err != nil { // If the supplied error is of an Element Controller Controller Error type, // encode the response to a new error response packet. @@ -366,14 +388,13 @@ func EncodeResponse(s interface{}, err error, w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") response, err := json.Marshal(s) if err != nil { - log.WithError(err).Error("Failed to marshal json response") - w.WriteHeader(http.StatusInternalServerError) + return err } _, err = w.Write(response) if err != nil { - log.WithError(err).Error("Failed to write json response") - w.WriteHeader(http.StatusInternalServerError) + return err } - } + + return nil } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-event/router.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-event/router.go index bdbe2e94c..a1027678c 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-event/router.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-event/router.go @@ -36,7 +36,7 @@ func (*DefaultApiRouter) Name() string { return "Event Manager" } -func (r *DefaultApiRouter) Init() error { +func (r *DefaultApiRouter) Init(log ec.Logger) error { return r.servicer.Initialize() } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go index 220f4adb2..9bb6d4a4f 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config.go @@ -42,8 +42,6 @@ type ConfigFile struct { Name string } - DebugLevel string `yaml:"debugLevel"` - ManagementConfig ManagementConfig `yaml:"managementConfig"` Switches []SwitchConfig diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml index b68f1d258..a8deb63d7 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_default.yaml @@ -1,7 +1,6 @@ version: v1 metadata: name: Rabbit -debugLevel: info switches: - id: 0 metadata: diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_dp1a.yaml b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_dp1a.yaml index d3de3c825..320ccc4aa 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_dp1a.yaml +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/config_dp1a.yaml @@ -1,7 +1,6 @@ version: v1 metadata: name: DP1a -debugLevel: info managementConfig: primaryDevice: 0 secondaryDevice: 1 diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go index 99cc4b1c0..192c3e7d9 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/manager.go @@ -24,8 +24,6 @@ import ( "os" "strconv" - log "github.com/sirupsen/logrus" - "github.com/NearNodeFlash/nnf-ec/pkg/api" ec "github.com/NearNodeFlash/nnf-ec/pkg/ec" event "github.com/NearNodeFlash/nnf-ec/pkg/manager-event" @@ -41,6 +39,13 @@ const ( FabricId = "Rabbit" ) +const ( + switchIdKey = "switchId" + portIdKey = "portId" + endpointIdKey = "endpointId" + slotKey = "slot" +) + type Fabric struct { ctrl SwitchtecControllerInterface @@ -57,6 +62,8 @@ type Fabric struct { managementEndpointCount int upstreamEndpointCount int downstreamEndpointCount int + + log ec.Logger } type Switch struct { @@ -79,6 +86,8 @@ type Switch struct { firmwareVersion string DEBUG_NEXT_SWITCH_LOG_PORT_ID int + + log ec.Logger } type Port struct { @@ -94,6 +103,8 @@ type Port struct { config *PortConfig endpoints []*Endpoint + + log ec.Logger } type portStatus struct { @@ -159,7 +170,13 @@ type Connection struct { // odataid string // } -var manager Fabric +var manager = Fabric{ + id: FabricId, + status: sf.ResourceStatus{ + State: sf.UNAVAILABLE_OFFLINE_RST, + Health: sf.CRITICAL_RH, + }, +} // Used to provide access to exported functions while protecting the singleton (i.e. manager) var FabricController *Fabric @@ -352,39 +369,43 @@ func (s *Switch) setDown() { s.path = "" } func (s *Switch) identify() error { f := s.fabric + log := s.log + for i := 0; i < len(f.switches); i++ { path := fmt.Sprintf("/dev/switchtec%d", i) - log.Debugf("Identify Switch %s: Opening %s", s.id, path) + log.V(2).Info("Opening path", "path", path) + dev, err := f.ctrl.Open(path) if os.IsNotExist(err) { - log.WithError(err).Debugf("path %s", path) continue } else if err != nil { - log.WithError(err).Warnf("Identify Switch %s: Open Error", s.id) + log.Error(err, "Error opening path") return err } paxId, err := dev.Identify() if err != nil { - log.WithError(err).Warnf("Identify Switch %s: Identify Error", s.id) + log.Error(err, "Identify error") return err } - log.Infof("Identify Switch %s: Device ID: %d", s.id, paxId) + log.V(2).Info("Identified switch device", "pax", paxId) if id := strconv.Itoa(int(paxId)); id == s.id { s.dev = dev s.path = path s.paxId = paxId - log.Infof("Identify Switch %s: Loading Mfg Info", s.id) - s.model = s.getModel() s.manufacturer = s.getManufacturer() s.serialNumber = s.getSerialNumber() s.firmwareVersion = s.getFirmwareVersion() + log.Info("Identified switch", "path", path, + "model", s.model, "manufacturer", s.manufacturer, + "serialNumber", s.serialNumber, "firmwareVersion", s.firmwareVersion) + return nil } } @@ -393,7 +414,7 @@ func (s *Switch) identify() error { // if we're using management routing, which enables forwarding of commands to // the secondary PAX through the primary PAX if f.config.IsManagementRoutingEnabled() { - log.Warningf("Switch %d is acting as primary device for switch %d", f.config.ManagementConfig.PrimaryDevice, f.config.ManagementConfig.SecondaryDevice) + log.Info("Switch routing enabled") for _, sw := range f.switches { if sw.dev.Device().ID() == f.config.ManagementConfig.PrimaryDevice { @@ -512,7 +533,7 @@ func (s *Switch) getDeviceStringByFunc(f func(dev SwitchtecDeviceInterface) (str if s.dev != nil { ret, err := f(s.dev) if err != nil { - log.WithError(err).Warnf("Failed to retrieve device string") + s.log.Error(err, "Failed to retrieve device string") } return ret @@ -602,7 +623,8 @@ func (p *Port) getResourceState() sf.ResourceState { } func (p *Port) Initialize() error { - log.Infof("Initialize Port %s: Name: %s Physical Port: %d", p.id, p.config.Name, p.config.Port) + log := p.log + log.V(2).Info("Initialize port") switch p.portType { case sf.DOWNSTREAM_PORT_PV130PT: @@ -628,7 +650,7 @@ func (p *Port) Initialize() error { f := port.swtch.fabric if len(epPort.Ep.Functions) < 1 /*PF*/ +f.managementEndpointCount+f.upstreamEndpointCount { - log.Warnf("Port %s: Insufficient function count %d", port.id, len(epPort.Ep.Functions)) + return fmt.Errorf("Port %s: Insufficient function count: Expected: %d Actual: %d", port.id, 1 /*PF*/ +f.managementEndpointCount+f.upstreamEndpointCount, epPort.Ep.Functions) } for idx, f := range epPort.Ep.Functions { @@ -651,25 +673,28 @@ func (p *Port) Initialize() error { } } - log.Infof("Initialize Port: Switch %s enumerting DSP %d", p.swtch.id, p.config.Port) + log.V(2).Info("Initialize downstream port") if err := p.swtch.dev.EnumerateEndpoint(uint8(p.config.Port), processPort(p)); err != nil { - log.WithError(err).Warnf("Initialize Port: Port Enumeration Failed: Physical Port %d", p.config.Port) + log.Error(err, "Port initialization failed") return err } } + log.Info("Port initialized") + return nil } func (p *Port) bind() error { f := p.swtch.fabric + log := p.log if p.portStatus.linkStatus != sf.LINK_UP_PV130LS { - log.Warnf("Port %+v: Port not up, skipping bind operation", p) + log.Info("Port not up. Skipping bind operation") return nil } - log.Debugf("Port %s: Bind Operation Starting: Switch: %s, PAX: %d Physical Port: %d Type: %s", p.id, p.swtch.id, p.swtch.paxId, p.config.Port, p.portType) + log.V(2).Info("Starting port bind operation") if p.portType != sf.DOWNSTREAM_PORT_PV130PT { panic(fmt.Sprintf("Port %s: Bind operation not allowed for port type %s", p.id, p.portType)) @@ -685,6 +710,8 @@ func (p *Port) bind() error { for _, ep := range p.endpoints[1:] { // Skip PF + log := log.WithValues(endpointIdKey, ep.id) + for _, epg := range f.endpointGroups { initiator := *(epg.initiator) @@ -720,6 +747,8 @@ func (p *Port) bind() error { for _, initiatorPort := range initiator.ports { s := initiatorPort.swtch + log := log.WithValues("initiatorPort", initiatorPort.config.Port, "logicalPortId", logicalPortId, "pdfid", endpoint.pdfid) + if initiator.endpointType == sf.PROCESSOR_EV150ET { if !f.config.IsManagementRoutingEnabled() { if p.swtch.id != s.id { @@ -738,25 +767,29 @@ func (p *Port) bind() error { } if endpoint.pdfid == 0 { - log.Errorf("Endpoint has no PDFID, skipping bind to initiator port %d (%s)", initiatorPort.config.Port, initiatorPort.config.Name) + log.Info("Endpoint has no PDFID, skipping bind to initiator port") continue } if endpoint.bound { - logFunc := log.Debugf + + log.Info("Endpoint already bound", + "paxId", s.paxId, "boundPaxId", endpoint.boundPaxId, + "phyPortId", initiatorPort.config.Port, "boundPhyPortId", endpoint.boundHvdPhyId, + "logPortId", logicalPortId, "boundLogPortId", endpoint.boundHvdLogId) + if endpoint.boundPaxId != uint8(s.paxId) || endpoint.boundHvdPhyId != uint8(initiatorPort.config.Port) || endpoint.boundHvdLogId != uint8(logicalPortId) { - logFunc = log.Errorf + panic(fmt.Sprintf("Already Bound: Misconfigured Port: PAX: %d, Physical Port: %d, Logical Port: %d, PDFID: %#04x", endpoint.boundPaxId, endpoint.boundHvdPhyId, endpoint.boundHvdLogId, endpoint.pdfid)) } - logFunc("Already Bound: PAX: %d, Physical Port: %d, Logical Port: %d, PDFID: %#04x", endpoint.boundPaxId, endpoint.boundHvdPhyId, endpoint.boundHvdLogId, endpoint.pdfid) break } - log.Infof("Bind: Switch: %s, PAX: %d, Physical Port: %d, Logical Port: %d, PDFID %#04x", s.id, s.paxId, initiatorPort.config.Port, logicalPortId, endpoint.pdfid) + log.Info("Binding Port") if err := s.dev.Bind(uint8(initiatorPort.config.Port), uint8(logicalPortId), endpoint.pdfid); err != nil { - log.WithError(err).Errorf("Bind Failed: Switch %s: PAX: %d Port: %d, Logical Port: %d, PDFID: %#04x", s.id, s.paxId, initiatorPort.config.Port, logicalPortId, endpoint.pdfid) + log.Error(err, "Bind Failed") } break @@ -832,80 +865,66 @@ func (c *Connection) fmt(format string, a ...interface{}) string { } // Initialize -func Initialize(ctrl SwitchtecControllerInterface) error { +func Initialize(log ec.Logger, ctrl SwitchtecControllerInterface) error { + log.Info("Initialize Fabric Manager") - manager = Fabric{ - id: FabricId, - ctrl: ctrl, - status: sf.ResourceStatus{ - State: sf.UNAVAILABLE_OFFLINE_RST, - Health: sf.CRITICAL_RH, - }, - } + const name = "fabric" + log.V(2).Info("Creating logger", "name", name) + log = log.WithName(name) - m := &manager + manager.log = log + manager.ctrl = ctrl - log.SetLevel(log.DebugLevel) - log.Infof("Fabric Manager %s Initializing", m.id) + m := &manager conf, err := loadConfig() if err != nil { - log.WithError(err).Errorf("Fabric Manager %s failed to load configuration", m.id) + log.Error(err, "Failed to load configuration") return err } m.config = conf - log.Debugf("Fabric Configuration '%s' Loaded...", conf.Metadata.Name) - log.Debugf(" Debug Level: %s", conf.DebugLevel) - log.Debugf(" Management Ports: %d", conf.ManagementPortCount) - log.Debugf(" Upstream Ports: %d", conf.UpstreamPortCount) - log.Debugf(" Downstream Ports: %d", conf.DownstreamPortCount) - for _, switchConf := range conf.Switches { - log.Debugf(" Switch %s Configuration: %s", switchConf.Id, switchConf.Metadata.Name) - log.Debugf(" Management Ports: %d", switchConf.ManagementPortCount) - log.Debugf(" Upstream Ports: %d", switchConf.UpstreamPortCount) - log.Debugf(" Downstream Ports: %d", switchConf.DownstreamPortCount) - } - - level, err := log.ParseLevel(conf.DebugLevel) - if err != nil { - log.WithError(err).Errorf("Failed to parse debug level: %s", conf.DebugLevel) - return err - } - - log.SetLevel(level) + log.V(1).Info("Loaded configuration", + "managementPortCount", conf.ManagementPortCount, + "upstreamPortCount", conf.UpstreamPortCount, + "downstreamPortCount", conf.DownstreamPortCount) m.switches = make([]Switch, len(conf.Switches)) var fabricPortId = 0 for switchIdx := range conf.Switches { switchConf := &conf.Switches[switchIdx] - log.Infof("Initialize switch %s", switchConf.Id) + log := log.WithName(switchConf.Id).WithValues(switchIdKey, switchConf.Id) + m.switches[switchIdx] = Switch{ id: switchConf.Id, idx: switchIdx, fabric: m, config: switchConf, ports: make([]Port, len(switchConf.Ports)), + log: log, } s := &m.switches[switchIdx] // TODO: This should probably move to Start() routine, although // if we can't find the switch Start() won't really do anything anyways. - log.Infof("identify switch %s", switchConf.Id) + + log.Info("Identify switch") if err := s.identify(); err != nil { - log.WithError(err).Warnf("Failed to identify switch %s", s.id) + log.Error(err, "Failed to identify switch") } - log.Infof("Switch %s identified: PAX %d", switchConf.Id, s.paxId) + log.Info("Switch identified", "pax", s.paxId) for portIdx, portConf := range switchConf.Ports { portType := portConf.getPortType() + id := strconv.Itoa(portIdx) + slot := int64(portConf.Slot) s.ports[portIdx] = Port{ - id: strconv.Itoa(portIdx), + id: id, fabricId: strconv.Itoa(fabricPortId), - slot: int64(portConf.Slot), + slot: slot, portType: portType, portStatus: portStatus{ cfgLinkWidth: 0, @@ -918,6 +937,7 @@ func Initialize(ctrl SwitchtecControllerInterface) error { swtch: &m.switches[switchIdx], config: &switchConf.Ports[portIdx], endpoints: []*Endpoint{}, + log: log.WithName(id).WithValues(portIdKey, id, slotKey, slot), } fabricPortId++ @@ -948,10 +968,7 @@ func Initialize(ctrl SwitchtecControllerInterface) error { m.downstreamEndpointCount = (1 + // PF mangementAndUpstreamEndpointCount) * m.config.DownstreamPortCount - log.Debugf("Creating Endpoints:") - log.Debugf(" Management Endpoints: % 3d", m.managementEndpointCount) - log.Debugf(" Upstream Endpoints: % 3d", m.upstreamEndpointCount) - log.Debugf(" Downstream Endpoints: % 3d", m.downstreamEndpointCount) + log.V(2).Info("Creating endpoints") m.endpoints = make([]Endpoint, mangementAndUpstreamEndpointCount+m.downstreamEndpointCount) @@ -1057,14 +1074,14 @@ func Initialize(ctrl SwitchtecControllerInterface) error { // act accordingly. event.EventManager.Subscribe(m) - log.Infof("Fabric Manager %s Initialization Finished", m.id) + log.Info("Fabric Manager Initialized") return nil } // Start - func Start() error { m := &manager - log.Infof("Starting Fabric Manager %s", m.id) + m.log.V(1).Info("Starting manager") m.status.State = sf.STARTING_RST @@ -1074,14 +1091,18 @@ func Start() error { s := &m.switches[switchIdx] if !s.isReady() { - log.Errorf("Failed to start switch %s: Switch is Down", s.id) + s.log.Info("Failed to start switch") continue } for portIdx := range s.ports { p := &s.ports[portIdx] + if err := p.Initialize(); err != nil { - log.WithError(err).Errorf("Switch %s Port %s failed to initialize", s.id, p.id) + m.log.Error(err, "Port initialization failed") + + // Port initialization is not fatal - the manager can continue + // to operate with down ports. } } @@ -1111,7 +1132,7 @@ func (m *Fabric) EventHandler(e event.Event) error { if p.portType == sf.DOWNSTREAM_PORT_PV130PT { if err := p.bind(); err != nil { - log.WithError(err).Errorf("Port %s (switch: %s port: %d) failed to bind", p.id, p.swtch.id, p.config.Port) + m.log.Error(err, "Port bind failed") } } } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/router.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/router.go index 5131f268b..e78e591f3 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/router.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric/router.go @@ -48,8 +48,8 @@ func (*DefaultApiRouter) Name() string { } // Init - -func (r *DefaultApiRouter) Init() error { - return Initialize(r.controller) +func (r *DefaultApiRouter) Init(log ec.Logger) error { + return Initialize(log, r.controller) } // Start - diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-message-registry/router.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-message-registry/router.go index 8142f7717..848981459 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-message-registry/router.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-message-registry/router.go @@ -35,7 +35,7 @@ func (*DefaultApiRouter) Name() string { return "Message Registries" } -func (*DefaultApiRouter) Init() error { +func (*DefaultApiRouter) Init(log ec.Logger) error { return MessageRegistryManager.Initialize() } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/aer.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/aer.go index d23618f12..c78a7aad7 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/aer.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/aer.go @@ -63,8 +63,8 @@ func (aer *AerService) c(err error) error { return err } -func (aer *AerService) Initialize(ctrl NnfControllerInterface) error { - return aer.c(aer.s.Initialize(ctrl)) +func (aer *AerService) Initialize(log ec.Logger, ctrl NnfControllerInterface) error { + return aer.c(aer.s.Initialize(log, ctrl)) } func (aer *AerService) Close() error { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/api.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/api.go index 8238d003c..a642d83c3 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/api.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/api.go @@ -22,6 +22,7 @@ package nnf import ( "net/http" + ec "github.com/NearNodeFlash/nnf-ec/pkg/ec" sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models" ) @@ -30,7 +31,7 @@ import ( // corresponding Handler method (see below). Each method should respond to the request by // updating the http.ResponesWriter. type Api interface { - Initialize(NnfControllerInterface) error + Initialize(ec.Logger, NnfControllerInterface) error Close() error Id() string @@ -76,7 +77,7 @@ type Api interface { // Storage Service API defines the interface for the above API methods to call. Each API method must have // an equivalent method. Methods take request paramters and a Redfish / Swordfish model to populate. type StorageServiceApi interface { - Initialize(NnfControllerInterface) error + Initialize(ec.Logger, NnfControllerInterface) error Close() error Id() string diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.go index 60fac73b1..69859371b 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.go @@ -36,8 +36,6 @@ type ConfigFile struct { Id string - DebugLevel string `yaml:"debugLevel"` - AllocationConfig AllocationConfig `yaml:"allocationConfig"` RemoteConfig RemoteConfig `yaml:"remoteConfig"` } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.yaml b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.yaml index 85d0963a0..aaccd04a7 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.yaml +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/config.yaml @@ -2,7 +2,6 @@ version: v1 metadata: name: Rabbit Storage Manager id: NNF -debugLevel: info allocationConfig: policy: spares standard: strict diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go index 6661a3f16..4316b8250 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/manager.go @@ -25,7 +25,6 @@ import ( "strings" "github.com/google/uuid" - log "github.com/sirupsen/logrus" ec "github.com/NearNodeFlash/nnf-ec/pkg/ec" event "github.com/NearNodeFlash/nnf-ec/pkg/manager-event" @@ -38,6 +37,18 @@ import ( sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models" ) +// Logging Keys used for named arguments. Should be kept in lowerCamelCase per k8s standard. +// https://github.com/kubernetes/community/blob/HEAD/contributors/devel/sig-instrumentation/migration-to-structured-logging.md#name-arguments +const ( + modelIdKey = "id" + storagePoolIdKey = "storagePoolId" + storageGroupIdKey = "storageGroupId" + fileSystemIdKey = "fileSystemId" + fileShareIdKey = "fileShareId" + endpointIdKey = "endpointId" + odataIdKey = "odataId" +) + var storageService = StorageService{ id: DefaultStorageServiceId, state: sf.DISABLED_RST, @@ -67,6 +78,8 @@ type StorageService struct { // That is, given a Storage Service resource OdataId field, ResourceIndex will correspond to the // index within the OdataId splity by "/" i.e. strings.split(OdataId, "/")[ResourceIndex] resourceIndex int + + log ec.Logger } func (s *StorageService) OdataId() string { @@ -386,49 +399,37 @@ func (s *StorageService) cleanupVolumes() { // Storage Service must complete initialization without error prior any // access to the Storage Service. Failure to initialize will cause the // storage service to misbehave. -func (*StorageService) Initialize(ctrl NnfControllerInterface) error { +func (s *StorageService) Initialize(log ec.Logger, ctrl NnfControllerInterface) error { - storageService = StorageService{ - id: DefaultStorageServiceId, - state: sf.STARTING_RST, - health: sf.CRITICAL_RH, + log.Info("Initialize NNF Manager") - // Dynamic controllers for managing objects. These are typically programmed via command line arguments - // the the NNF Controller Interface is created - serverControllerProvider: ctrl.ServerControllerProvider(), - persistentController: ctrl.PersistentControllerProvider(), + storageService.state = sf.STARTING_RST - // Reserve space for the most common allocation types. 32 is the current - // limit for the number of supported namespaces. - pools: make([]StoragePool, 0, 32), - groups: make([]StorageGroup, 0, 32), - fileSystems: make([]FileSystem, 0, 32), - } + // Dynamic controllers for managing objects. These are typically programmed via command line arguments + // the the NNF Controller Interface is created + storageService.serverControllerProvider = ctrl.ServerControllerProvider() + storageService.persistentController = ctrl.PersistentControllerProvider() - s := &storageService + // Reserve space for the most common allocation types. 32 is the current + // limit for the number of supported namespaces. + storageService.pools = make([]StoragePool, 0, 32) + storageService.groups = make([]StorageGroup, 0, 32) + storageService.fileSystems = make([]FileSystem, 0, 32) + + const name = "nnf" + log.V(2).Info("Creating logger", "name", name) + log = log.WithName(name) + storageService.log = log conf, err := loadConfig() if err != nil { - log.WithError(err).Errorf("Failed to load %s configuration", s.id) + log.Error(err, "failed to load configuration", "id", s.id) return err } s.id = conf.Id s.config = conf - log.Debugf("NNF Storage Service '%s' Loaded...", conf.Metadata.Name) - log.Debugf(" Debug Level: %s", conf.DebugLevel) - log.Debugf(" Remote Config : %+v", conf.RemoteConfig) - log.Debugf(" Allocation Config: %+v", conf.AllocationConfig) - - level, err := log.ParseLevel(conf.DebugLevel) - if err != nil { - log.WithError(err).Errorf("Failed to parse debug level: %s", conf.DebugLevel) - return err - } - - log.SetLevel(level) - s.endpoints = make([]Endpoint, len(conf.RemoteConfig.Servers)) for endpointIdx := range s.endpoints { s.endpoints[endpointIdx] = Endpoint{ @@ -462,7 +463,7 @@ func (*StorageService) Initialize(ctrl NnfControllerInterface) error { // Initialize the Server Manager - considered internal to // the NNF Manager if err := server.Initialize(); err != nil { - log.WithError(err).Errorf("Failed to Initialize Server Manager") + log.Error(err, "Failed to Initialize Server Manager") return err } @@ -477,13 +478,14 @@ func (s *StorageService) Close() error { } func (s *StorageService) EventHandler(e event.Event) error { + log := s.log.WithValues("eventId", e.Id, "eventMessage", e.Message, "eventArgs", e.MessageArgs) // Upstream link events linkEstablished := e.Is(msgreg.UpstreamLinkEstablishedFabric("", "")) || e.Is(msgreg.DegradedUpstreamLinkEstablishedFabric("", "")) linkDropped := e.Is(msgreg.UpstreamLinkDroppedFabric("", "")) if linkEstablished || linkDropped { - log.Infof("Storage Service: Event Received %+v", e) + log.V(2).Info("Link event") var switchId, portId string if err := e.Args(&switchId, &portId); err != nil { @@ -503,7 +505,7 @@ func (s *StorageService) EventHandler(e event.Event) error { endpoint.fabricId = fabric.FabricId if linkEstablished { - + log.V(2).Info("Link established") opts := server.ServerControllerOptions{ Local: ep.Type() == sf.PROCESSOR_EV150ET, Address: endpoint.config.Address, @@ -518,7 +520,7 @@ func (s *StorageService) EventHandler(e event.Event) error { } } else if linkDropped { - + log.Info("Link dropped") endpoint.serverCtrl = server.NewDisabledServerController() endpoint.state = sf.UNAVAILABLE_OFFLINE_RST } @@ -529,15 +531,15 @@ func (s *StorageService) EventHandler(e event.Event) error { // Check if the fabric is ready; that is all devices are enumerated and discovery // is complete. We if e.Is(msgreg.FabricReadyNnf("")) { - log.Infof("Storage Service: Event Received %+v", e) + log.V(1).Info("Fabric ready") if err := s.store.Replay(); err != nil { - log.WithError(err).Errorf("Failed to replay storage database") + log.Error(err, "Failed to replay storage database") return err } // Remove any namespaces that are not part of a Storage Pool - log.Infof("Storage Service: Removing Volumes that are not allocated as part of a Storage Pool") + log.V(2).Info("Cleanup obsolete volumes") s.cleanupVolumes() s.state = sf.ENABLED_RST @@ -557,7 +559,7 @@ func (s *StorageService) EventHandler(e event.Event) error { s.health = f.Status.Health } - log.Infof("Storage Service Enabled: Health %s", s.health) + log.Info("Storage Service Enabled", "health", s.health) } return nil @@ -659,36 +661,42 @@ func (*StorageService) StorageServiceIdStoragePoolsGet(storageServiceId string, } // StorageServiceIdStoragePoolsPost - -func (*StorageService) StorageServiceIdStoragePoolsPost(storageServiceId string, model *sf.StoragePoolV150StoragePool) error { +func (*StorageService) StorageServiceIdStoragePoolsPost(storageServiceId string, model *sf.StoragePoolV150StoragePool) (err error) { s := findStorageService(storageServiceId) if s == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(StorageServiceOdataType, storageServiceId)) } // TODO: Check the model for valid RAID configurations + log := s.log.WithValues(modelIdKey, model.Id) + log.V(2).Info("Creating storage pool") + defer func() { + if err != nil { + log.Error(err, "Create storage pool failed") + } + }() policy := NewAllocationPolicy(s.config.AllocationConfig, model.Oem) if policy == nil { - log.Errorf("Failed to allocate storage policy. Config: %+v Oem: %+v", s.config.AllocationConfig, model.Oem) return ec.NewErrNotAcceptable().WithEvent(msgreg.PropertyValueTypeErrorBase("Oem", fmt.Sprintf("%+v", model.Oem))) } - capacityBytes := model.CapacityBytes - if capacityBytes == 0 { - capacityBytes = model.Capacity.Data.AllocatedBytes + capacityInBytes := model.CapacityBytes + if capacityInBytes == 0 { + capacityInBytes = model.Capacity.Data.AllocatedBytes } - if capacityBytes == 0 { + if capacityInBytes == 0 { return ec.NewErrNotAcceptable().WithEvent(msgreg.CreateFailedMissingReqPropertiesBase("CapacityBytes")) } - if err := policy.Initialize(uint64(capacityBytes)); err != nil { - log.WithError(err).Errorf("Failed to initialize storage policy") + if err := policy.Initialize(uint64(capacityInBytes)); err != nil { + log.Error(err, "Failed to initialize storage policy", "capacityInBytes", capacityInBytes) return ec.NewErrInternalServerError().WithResourceType(StorageServiceOdataType).WithError(err).WithCause("Failed to initialize storage policy") } if err := policy.CheckCapacity(); err != nil { - log.WithError(err).Warnf("Storage Policy does not provide sufficient capacity to support requested %d bytes", capacityBytes) + log.Error(err, "Storage policy cannot support capacity", "capacityInBytes", capacityInBytes) return ec.NewErrNotAcceptable().WithResourceType(StorageServiceOdataType).WithError(err).WithCause("Insufficient capacity available") } @@ -709,15 +717,14 @@ func (*StorageService) StorageServiceIdStoragePoolsPost(storageServiceId string, } if err := s.persistentController.CreatePersistentObject(p, updateFunc, storagePoolStorageCreateStartLogEntryType, storagePoolStorageCreateCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to create volume from storage pool %s", p.id) - s.deleteStoragePool(p) - return ec.NewErrInternalServerError().WithResourceType(StorageServiceOdataType).WithError(err).WithCause("Failed to allocate storage volumes") } event.EventManager.PublishResourceEvent(msgreg.ResourceCreatedResourceEvent(), p) + log.Info("Created storage pool", storagePoolIdKey, p.id, "volumes", len(p.providingVolumes), "capacityInBytes", p.allocatedVolume.capacityBytes) + return s.StorageServiceIdStoragePoolIdGet(storageServiceId, p.id, model) } @@ -784,20 +791,28 @@ func (*StorageService) StorageServiceIdStoragePoolIdGet(storageServiceId, storag } // StorageServiceIdStoragePoolIdDelete - -func (*StorageService) StorageServiceIdStoragePoolIdDelete(storageServiceId, storagePoolId string) error { +func (*StorageService) StorageServiceIdStoragePoolIdDelete(storageServiceId, storagePoolId string) (err error) { s, p := findStoragePool(storageServiceId, storagePoolId) if p == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(StoragePoolOdataType, storagePoolId)) } + log := s.log.WithValues(storagePoolIdKey, p.id) + log.V(2).Info("Deleting storage pool") + defer func() { + if err != nil { + log.Error(err, "Delete storage pool failed") + } + }() + if p.fileSystemId != "" { if err := s.StorageServiceIdFileSystemIdDelete(s.id, p.fileSystemId); err != nil { return ec.NewErrInternalServerError().WithResourceType(StoragePoolOdataType).WithError(err).WithCause(fmt.Sprintf("Failed to delete file system '%s'", p.fileSystemId)) } - if p.fileSystemId != "" { - panic("File system not deleted") + if len(p.fileSystemId) != 0 { + return ec.NewErrInternalServerError().WithResourceType(StoragePoolOdataType).WithCause(fmt.Sprintf("File system '%s' not removed from storage pool", p.fileSystemId)) } } @@ -813,7 +828,7 @@ func (*StorageService) StorageServiceIdStoragePoolIdDelete(storageServiceId, sto } if len(p.storageGroupIds) != 0 { - panic("Storage groups not deleted") + return ec.NewErrInternalServerError().WithResourceType(StoragePoolOdataType).WithCause(fmt.Sprintf("Storage groups not removed from storage pool")) } deleteFunc := func() error { @@ -821,14 +836,15 @@ func (*StorageService) StorageServiceIdStoragePoolIdDelete(storageServiceId, sto } if err := s.persistentController.DeletePersistentObject(p, deleteFunc, storagePoolStorageDeleteStartLogEntryType, storagePoolStorageDeleteCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to delete volume from storage pool %s", p.id) - return ec.NewErrInternalServerError().WithResourceType(StoragePoolOdataType).WithError(err).WithCause(fmt.Sprintf("Failed to delete volume")) + return ec.NewErrInternalServerError().WithResourceType(StoragePoolOdataType).WithError(err).WithCause(fmt.Sprintf("Failed to delete storage pool")) } event.EventManager.PublishResourceEvent(msgreg.ResourceRemovedResourceEvent(), p) s.deleteStoragePool(p) + log.Info("Deleted storage pool") + return nil } @@ -946,12 +962,20 @@ func (*StorageService) StorageServiceIdStorageGroupsGet(storageServiceId string, } // StorageServiceIdStorageGroupPost - -func (*StorageService) StorageServiceIdStorageGroupPost(storageServiceId string, model *sf.StorageGroupV150StorageGroup) error { +func (*StorageService) StorageServiceIdStorageGroupPost(storageServiceId string, model *sf.StorageGroupV150StorageGroup) (err error) { s := findStorageService(storageServiceId) if s == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(StorageServiceOdataType, storageServiceId)) } + log := s.log.WithValues(modelIdKey, model.Id) + log.V(2).Info("Creating storage group") + defer func() { + if err != nil { + log.Error(err, "Create storage group failed") + } + }() + fields := strings.Split(model.Links.StoragePool.OdataId, "/") if len(fields) != s.resourceIndex+1 { return ec.NewErrNotAcceptable().WithResourceType(StoragePoolOdataType).WithEvent(msgreg.InvalidURIBase(model.Links.StoragePool.OdataId)) @@ -994,7 +1018,7 @@ func (*StorageService) StorageServiceIdStorageGroupPost(storageServiceId string, } if err := volume.AttachController(sg.endpoint.controllerId); err != nil { - return err + return ec.NewErrInternalServerError().WithResourceType(StorageGroupOdataType).WithError(err).WithCause(fmt.Sprintf("Storage group '%s' attach volume '%s' failed", sg.id, pv.VolumeId)) } } @@ -1002,15 +1026,14 @@ func (*StorageService) StorageServiceIdStorageGroupPost(storageServiceId string, } if err := s.persistentController.CreatePersistentObject(sg, updateFunc, storageGroupCreateStartLogEntryType, storageGroupCreateCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to create storage group %s", sg.id) - s.deleteStorageGroup(sg) - return ec.NewErrInternalServerError().WithResourceType(StorageGroupOdataType).WithError(err).WithCause("failed to create storage group") } event.EventManager.PublishResourceEvent(msgreg.ResourceCreatedResourceEvent(), sg) + log.Info("Created storage group", storageGroupIdKey, sg.id) + return s.StorageServiceIdStorageGroupIdGet(storageServiceId, sg.id, model) } @@ -1067,14 +1090,23 @@ func (*StorageService) StorageServiceIdStorageGroupIdGet(storageServiceId, stora } // StorageServiceIdStorageGroupIdDelete - -func (*StorageService) StorageServiceIdStorageGroupIdDelete(storageServiceId, storageGroupId string) error { +func (*StorageService) StorageServiceIdStorageGroupIdDelete(storageServiceId, storageGroupId string) (err error) { s, sg := findStorageGroup(storageServiceId, storageGroupId) if sg == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(StorageGroupOdataType, storageGroupId)) } + log := s.log.WithValues(storageGroupIdKey, sg.id) + log.V(2).Info("Deleting storage group") + defer func() { + if err != nil { + log.Error(err, "Delete storage group failed") + } + }() + if sg.fileShareId != "" { + log.WithValues(fileShareIdKey, sg.fileShareId).Info("cannot delete storage group with existing file share") return ec.NewErrNotAcceptable().WithResourceType(StorageGroupOdataType).WithEvent(msgreg.ResourceCannotBeDeletedBase()).WithCause(fmt.Sprintf("Storage group '%s' file share present", storageGroupId)) } @@ -1084,7 +1116,6 @@ func (*StorageService) StorageServiceIdStorageGroupIdDelete(storageServiceId, st } deleteFunc := func() error { - // Detach the endpoint from the NVMe namespaces for _, pv := range sp.providingVolumes { volume := pv.Storage.FindVolume(pv.VolumeId) @@ -1106,8 +1137,6 @@ func (*StorageService) StorageServiceIdStorageGroupIdDelete(storageServiceId, st } if err := s.persistentController.DeletePersistentObject(sg, deleteFunc, storageGroupDeleteStartLogEntryType, storageGroupDeleteCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to delete storage group %s", sg.id) - return ec.NewErrInternalServerError().WithResourceType(StorageGroupOdataType).WithError(err).WithCause("Failed to delete storage group") } @@ -1115,6 +1144,8 @@ func (*StorageService) StorageServiceIdStorageGroupIdDelete(storageServiceId, st s.deleteStorageGroup(sg) + log.Info("Deleted storage group") + return nil } @@ -1174,12 +1205,20 @@ func (*StorageService) StorageServiceIdFileSystemsGet(storageServiceId string, m } // StorageServiceIdFileSystemsPost - -func (*StorageService) StorageServiceIdFileSystemsPost(storageServiceId string, model *sf.FileSystemV122FileSystem) error { +func (*StorageService) StorageServiceIdFileSystemsPost(storageServiceId string, model *sf.FileSystemV122FileSystem) (err error) { s := findStorageService(storageServiceId) if s == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(StorageServiceOdataType, storageServiceId)) } + log := s.log.WithValues(modelIdKey, model.Id) + log.V(2).Info("Create file system") + defer func() { + if err != nil { + log.Error(err, "Create file system failed") + } + }() + // Extract the StoragePoolId from the POST model fields := strings.Split(model.Links.StoragePool.OdataId, "/") if len(fields) != s.resourceIndex+1 { @@ -1213,15 +1252,14 @@ func (*StorageService) StorageServiceIdFileSystemsPost(storageServiceId string, fs := s.createFileSystem(model.Id, sp, fsApi, oem) if err := s.persistentController.CreatePersistentObject(fs, func() error { return nil }, fileSystemCreateStartLogEntryType, fileSystemCreateCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to create file system %s", fs.id) - s.deleteFileSystem(fs) - return ec.NewErrInternalServerError().WithResourceType(FileSystemOdataType).WithError(err).WithCause(fmt.Sprintf("File system '%s' failed to create", fs.id)) } event.EventManager.PublishResourceEvent(msgreg.ResourceCreatedResourceEvent(), fs) + log.Info("Created file system", fileSystemIdKey, fs.id) + return s.StorageServiceIdFileSystemIdGet(storageServiceId, fs.id, model) } @@ -1265,12 +1303,20 @@ func (*StorageService) StorageServiceIdFileSystemIdGet(storageServiceId, fileSys } // StorageServiceIdFileSystemIdDelete - -func (*StorageService) StorageServiceIdFileSystemIdDelete(storageServiceId, fileSystemId string) error { +func (*StorageService) StorageServiceIdFileSystemIdDelete(storageServiceId, fileSystemId string) (err error) { s, fs := findFileSystem(storageServiceId, fileSystemId) if fs == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(FileSystemOdataType, fileSystemId)) } + log := s.log.WithValues(fileSystemIdKey, fs.id) + log.V(2).Info("Deleting file system") + defer func() { + if err != nil { + log.Error(err, "Delete file system failed") + } + }() + // Create a copy of file share IDs; The deletion of a share will modify the fs.shares[] so we cannot // iterate on that array directly as it is editted in place. shareIds := make([]string, len(fs.shares)) @@ -1285,8 +1331,6 @@ func (*StorageService) StorageServiceIdFileSystemIdDelete(storageServiceId, file } if err := s.persistentController.DeletePersistentObject(fs, func() error { return nil }, fileSystemDeleteStartLogEntryType, fileSystemDeleteCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to delete file system %s", fs.id) - return ec.NewErrInternalServerError().WithResourceType(FileSystemOdataType).WithError(err).WithCause("Failed to delete file system") } @@ -1294,6 +1338,8 @@ func (*StorageService) StorageServiceIdFileSystemIdDelete(storageServiceId, file s.deleteFileSystem(fs) + log.Info("Deleted file system") + return nil } @@ -1313,12 +1359,20 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedSharesGet(storageServ } // StorageServiceIdFileSystemIdExportedSharesPost - -func (*StorageService) StorageServiceIdFileSystemIdExportedSharesPost(storageServiceId, fileSystemId string, model *sf.FileShareV120FileShare) error { +func (*StorageService) StorageServiceIdFileSystemIdExportedSharesPost(storageServiceId, fileSystemId string, model *sf.FileShareV120FileShare) (err error) { s, fs := findFileSystem(storageServiceId, fileSystemId) if fs == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(FileSystemOdataType, fileSystemId)) } + log := s.log.WithValues(modelIdKey, model.Id) + log.V(2).Info("Creating file share") + defer func() { + if err != nil { + log.Error(err, "Create file share failed") + } + }() + fields := strings.Split(model.Links.Endpoint.OdataId, "/") if len(fields) != s.resourceIndex+1 { return ec.NewErrNotAcceptable().WithResourceType(FileSystemOdataType).WithEvent(msgreg.InvalidURIBase(model.Links.Endpoint.OdataId)) @@ -1328,7 +1382,6 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedSharesPost(storageSer ep := s.findEndpoint(endpointId) if ep == nil { return ec.NewErrNotAcceptable().WithResourceType(EndpointOdataType).WithEvent(msgreg.ResourceNotFoundBase(EndpointOdataType, endpointId)) - } sp := s.findStoragePool(fs.storagePoolId) @@ -1348,8 +1401,10 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedSharesPost(storageSer // Wait for the storage group to be ready (enabled) to ensure the disks are present on the system state := sg.status().State if state == sf.STARTING_RST { + log.V(2).Info("Storage group starting", storageGroupIdKey, sg.id) return ec.NewErrorNotReady().WithResourceType(StorageGroupOdataType).WithCause(fmt.Sprintf("Storage group '%s' is starting", sg.id)) } else if state != sf.ENABLED_RST { + log.Info("Storage group not enabled", storageGroupIdKey, sg.id) return ec.NewErrNotAcceptable().WithResourceType(StorageGroupOdataType).WithCause(fmt.Sprintf("Storage group '%s' is not ready: state %s", sg.id, state)) } @@ -1357,13 +1412,11 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedSharesPost(storageSer createFunc := func() error { if err := sg.serverStorage.CreateFileSystem(fs.fsApi, model.Oem); err != nil { - log.WithError(err).Errorf("Failed to create file share for path %s", model.FileSharePath) - return err + return ec.NewErrInternalServerError().WithResourceType(FileShareOdataType).WithError(err).WithCause(fmt.Sprintf("File share '%s' create failed", sh.id)) } if err := sg.serverStorage.MountFileSystem(fs.fsApi, sh.mountRoot); err != nil { - log.WithError(err).Errorf("Failed to mount file share for path %s", model.FileSharePath) - return err + return ec.NewErrInternalServerError().WithResourceType(FileShareOdataType).WithError(err).WithCause(fmt.Sprintf("File share '%s' mount failed", sh.id)) } return nil @@ -1375,21 +1428,31 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedSharesPost(storageSer event.EventManager.PublishResourceEvent(msgreg.ResourceCreatedResourceEvent(), sh) + log.Info("Created file share", fileShareIdKey, sh.id, "path", sh.mountRoot) + return s.StorageServiceIdFileSystemIdExportedShareIdGet(storageServiceId, fileSystemId, sh.id, model) } // StorageServiceIdFileSystemIdExportedShareIdPut - -func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdPut(storageServiceId, fileSystemId, exportedShareId string, model *sf.FileShareV120FileShare) error { +func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdPut(storageServiceId, fileSystemId, exportedShareId string, model *sf.FileShareV120FileShare) (err error) { s, fs, sh := findFileShare(storageServiceId, fileSystemId, exportedShareId) if fs == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(FileShareOdataType, exportedShareId)) } + if sh == nil { model.Id = exportedShareId - return s.StorageServiceIdFileSystemIdExportedSharesPost(storageServiceId, fileSystemId, model) } + log := s.log.WithValues(fileShareIdKey, sh.id, fileSystemIdKey, fs.id) + log.V(1).Info("Updating file share") + defer func() { + if err != nil { + log.Error(err, "Update file share failed") + } + }() + newPath := model.FileSharePath if err := s.StorageServiceIdFileSystemIdExportedShareIdGet(storageServiceId, fileSystemId, exportedShareId, model); err != nil { return err @@ -1434,8 +1497,7 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdPut(storageSer updateFunc = func() error { if err := sg.serverStorage.MountFileSystem(fs.fsApi, sh.mountRoot); err != nil { - log.WithError(err).Errorf("Failed to mount file share for path %s", model.FileSharePath) - return err + return ec.NewErrInternalServerError().WithResourceType(FileShareOdataType).WithError(err).WithCause(fmt.Sprintf("Failed to mount file share '%s' at path '%s'", sh.id, sh.mountRoot)) } return nil @@ -1444,8 +1506,7 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdPut(storageSer } else { updateFunc = func() error { if err := sg.serverStorage.UnmountFileSystem(fs.fsApi, sh.mountRoot); err != nil { - log.WithError(err).Errorf("Failed to unmount file share for path %s", model.FileSharePath) - return err + return ec.NewErrInternalServerError().WithResourceType(FileShareOdataType).WithError(err).WithCause(fmt.Sprintf("Failed to unmount file share '%s' at path '%s'", sh.id, sh.mountRoot)) } sh.mountRoot = "" @@ -1455,12 +1516,13 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdPut(storageSer } if err := s.persistentController.UpdatePersistentObject(sh, updateFunc, fileShareUpdateStartLogEntryType, fileShareUpdateCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to update file share %s", sh.id) return ec.NewErrInternalServerError().WithError(err).WithCause(fmt.Sprintf("File share '%s' failed to update", sh.id)) } event.EventManager.PublishResourceEvent(msgreg.ResourceChangedResourceEvent(), sh) + log.V(1).Info("Updated file share") + return s.StorageServiceIdFileSystemIdExportedShareIdGet(storageServiceId, fileSystemId, sh.id, model) } @@ -1489,19 +1551,27 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdGet(storageSer } // StorageServiceIdFileSystemIdExportedShareIdDelete - -func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdDelete(storageServiceId, fileSystemId, exportedShareId string) error { +func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdDelete(storageServiceId, fileSystemId, exportedShareId string) (err error) { s, fs, sh := findFileShare(storageServiceId, fileSystemId, exportedShareId) + if sh == nil { return ec.NewErrNotFound().WithEvent(msgreg.ResourceNotFoundBase(FileShareOdataType, exportedShareId)) } + log := s.log.WithValues(fileShareIdKey, sh.id) + log.V(2).Info("Deleting file share") + defer func() { + if err != nil { + log.Error(err, "Delete file share failed") + } + }() + sg := s.findStorageGroup(sh.storageGroupId) if sg == nil { return ec.NewErrInternalServerError().WithResourceType(FileShareOdataType).WithCause(fmt.Sprintf("File share '%s' does not have associated storage group '%s'", exportedShareId, sh.storageGroupId)) } deleteFunc := func() error { - if err := sg.serverStorage.UnmountFileSystem(fs.fsApi, sh.mountRoot); err != nil { return ec.NewErrInternalServerError().WithResourceType(FileShareOdataType).WithError(err).WithCause(fmt.Sprintf("File share '%s' failed unmount", exportedShareId)) } @@ -1514,8 +1584,6 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdDelete(storage } if err := s.persistentController.DeletePersistentObject(sh, deleteFunc, fileShareDeleteStartLogEntryType, fileShareDeleteCompleteLogEntryType); err != nil { - log.WithError(err).Errorf("Failed to delete file share %s", sh.id) - return ec.NewErrInternalServerError().WithError(err).WithResourceType(FileShareOdataType).WithCause("Failed to delete file share") } @@ -1523,5 +1591,7 @@ func (*StorageService) StorageServiceIdFileSystemIdExportedShareIdDelete(storage fs.deleteFileShare(sh) + log.Info("Deleted file share") + return nil } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/persistent.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/persistent.go index 6e7292a98..5429b8d1f 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/persistent.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/persistent.go @@ -20,8 +20,9 @@ package nnf import ( + logr "github.com/sirupsen/logrus" + "github.com/NearNodeFlash/nnf-ec/pkg/persistent" - log "github.com/sirupsen/logrus" ) // Persistent Controller API provides an interface for creating, updating, and deleting persistent objects. @@ -112,20 +113,20 @@ func executePersistentObjectTransaction(ledger *persistent.Ledger, obj Persisten data, err := obj.GenerateStateData(startingState) if err != nil { - log.WithError(err).Warnf("Object %s failed to generate starting state %d data", obj.GetKey(), startingState) + logr.WithError(err).Warnf("Object %s failed to generate starting state %d data", obj.GetKey(), startingState) return err } if err := ledger.Log(startingState, data); err != nil { - log.WithError(err).Warnf("Object %s failed to log starting state %d", obj.GetKey(), startingState) + logr.WithError(err).Warnf("Object %s failed to log starting state %d", obj.GetKey(), startingState) return err } if err := updateFunc(); err != nil { - log.WithError(err).Warnf("Object %s failed update to state %d", obj.GetKey(), startingState) + logr.WithError(err).Warnf("Object %s failed update to state %d", obj.GetKey(), startingState) if rollbackErr := obj.Rollback(startingState); rollbackErr != nil { - log.WithError(rollbackErr).Errorf("Object %s failed rollback to state %d", obj.GetKey(), startingState) + logr.WithError(rollbackErr).Errorf("Object %s failed rollback to state %d", obj.GetKey(), startingState) } return err @@ -133,12 +134,12 @@ func executePersistentObjectTransaction(ledger *persistent.Ledger, obj Persisten data, err = obj.GenerateStateData(endingState) if err != nil { - log.WithError(err).Warnf("Object %s failed to generate ending state %d data", obj.GetKey(), endingState) + logr.WithError(err).Warnf("Object %s failed to generate ending state %d data", obj.GetKey(), endingState) return err } if err := ledger.Log(endingState, data); err != nil { - log.WithError(err).Warnf("Object %s failed to log ending state %d", obj.GetKey(), endingState) + logr.WithError(err).Warnf("Object %s failed to log ending state %d", obj.GetKey(), endingState) return err } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go index 78a1691db..243d0c263 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/router.go @@ -36,8 +36,8 @@ func (*DefaultApiRouter) Name() string { return "NNF Storage Service Manager" } -func (r *DefaultApiRouter) Init() error { - return r.servicer.Initialize(r.controller) +func (r *DefaultApiRouter) Init(log ec.Logger) error { + return r.servicer.Initialize(log, r.controller) } func (r *DefaultApiRouter) Start() error { diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/servicer.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/servicer.go index 2aedb8e05..4059cf69a 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/servicer.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/servicer.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" + ec "github.com/NearNodeFlash/nnf-ec/pkg/ec" sf "github.com/NearNodeFlash/nnf-ec/pkg/rfsf/pkg/models" . "github.com/NearNodeFlash/nnf-ec/pkg/common" @@ -53,8 +54,8 @@ func (s *DefaultApiService) Id() string { return s.ss.Id() } -func (s *DefaultApiService) Initialize(ctrl NnfControllerInterface) error { - return s.ss.Initialize(ctrl) +func (s *DefaultApiService) Initialize(log ec.Logger, ctrl NnfControllerInterface) error { + return s.ss.Initialize(log, ctrl) } func (s *DefaultApiService) Close() error { @@ -359,7 +360,7 @@ func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdStorageGroup EncodeResponse(model, err, w) } -//RedfishV1StorageServicesStorageServiceIdStorageGroupsStorageGroupIdDelete(w http.ResponseWriter, r *http.Request) +// RedfishV1StorageServicesStorageServiceIdStorageGroupsStorageGroupIdDelete(w http.ResponseWriter, r *http.Request) func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdStorageGroupsStorageGroupIdDelete(w http.ResponseWriter, r *http.Request) { params := Params(r) storageServiceId := params["StorageServiceId"] @@ -376,7 +377,7 @@ func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdStorageGroup EncodeResponse(model, err, w) } -// RedfishV1StorageServicesStorageServiceIdEndpointsGet - +// RedfishV1StorageServicesStorageServiceIdEndpointsGet - func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdEndpointsGet(w http.ResponseWriter, r *http.Request) { params := Params(r) storageServiceId := params["StorageServiceId"] @@ -483,7 +484,7 @@ func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdFileSystemsF EncodeResponse(model, err, w) } -// RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemIdDelete - +// RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemIdDelete - func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemIdDelete(w http.ResponseWriter, r *http.Request) { params := Params(r) storageServiceId := params["StorageServiceId"] @@ -500,7 +501,7 @@ func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdFileSystemsF EncodeResponse(model, err, w) } -// RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemsIdExportedFileSharesGet - +// RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemsIdExportedFileSharesGet - func (s *DefaultApiService) RedfishV1StorageServicesStorageServiceIdFileSystemsFileSystemsIdExportedFileSharesGet(w http.ResponseWriter, r *http.Request) { params := Params(r) storageServiceId := params["StorageServiceId"] diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go index 7cbfdea71..f091cf678 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nnf/storage_pool.go @@ -24,7 +24,6 @@ import ( "fmt" "github.com/google/uuid" - log "github.com/sirupsen/logrus" nvme2 "github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme" nvme "github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme" @@ -106,22 +105,25 @@ func (p *StoragePool) findStorageGroupByEndpoint(endpoint *Endpoint) *StorageGro } func (p *StoragePool) recoverVolumes(volumes []storagePoolPersistentVolumeInfo, ignoreErrors bool) error { + log := p.storageService.log - log.Infof("Storage Pool %s: Recover Volumes", p.id) + log.WithValues(storagePoolIdKey, p.id) + log.Info("recover volumes") for _, volumeInfo := range volumes { + log := log.WithValues("serialNumber", volumeInfo.SerialNumber, "namespaceId", volumeInfo.NamespaceId) // Locate the NVMe Storage device by Serial Number storage := p.storageService.findStorage(volumeInfo.SerialNumber) if storage == nil { - log.Warnf("Storage %s not found", volumeInfo.SerialNumber) + log.Info("storage device not found") continue } // Locate the Volume by Namespace ID volume, err := storage.FindVolumeByNamespaceId(volumeInfo.NamespaceId) if err != nil { - log.Errorf("Volume %d not found", volumeInfo.NamespaceId) + log.Error(err, "namespace not found") if ignoreErrors { continue } @@ -145,6 +147,7 @@ func (p *StoragePool) recoverVolumes(volumes []storagePoolPersistentVolumeInfo, } func (p *StoragePool) deallocateVolumes() error { + log := p.storageService.log.WithValues(storagePoolIdKey, p.id) // In order to speed up deleting volumes, we format them first. Format runs asynchronously, so after // each format call, wait for completion before deleting the volume. @@ -163,14 +166,17 @@ func (p *StoragePool) deallocateVolumes() error { return nil } + log.V(3).Info("Formatting volumes") if err := runOnProvidingVolumes(func(v *nvme.Volume) error { return v.Format() }); err != nil { return fmt.Errorf("Failed to format volumes: %v", err) } + log.V(3).Info("Wait for format complete") if err := runOnProvidingVolumes(func(v *nvme.Volume) error { return v.WaitFormatComplete() }); err != nil { return fmt.Errorf("Failed to wait on format completions: %v", err) } + log.V(3).Info("Deleting volumes") if err := runOnProvidingVolumes(func(v *nvme.Volume) error { return v.Delete() }); err != nil { return fmt.Errorf("Failed to delete volumes: %v", err) } @@ -270,7 +276,6 @@ func NewStoragePoolRecoveryRegistry(s *StorageService) persistent.Registry { func (*storagePoolRecoveryRegistry) Prefix() string { return storagePoolRegistryPrefix } func (r *storagePoolRecoveryRegistry) NewReplay(id string) persistent.ReplayHandler { - log.Infof("Storage Pool %s: New Replay", id) return &storagePoolRecoveryReplayHandler{storageService: r.storageService, id: id} } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.go index 2fb1aaf18..bc2034caa 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.go @@ -43,7 +43,6 @@ type ConfigFile struct { Metadata struct { Name string } - DebugLevel string `yaml:"debugLevel"` Storage StorageConfig } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.yaml b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.yaml index 8975d47e9..f69a90b3e 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.yaml +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/config.yaml @@ -1,7 +1,6 @@ version: v1 metadata: name: Rabbit -debugLevel: info storage: controller: functions: 17 diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go index d7b49767c..b13691069 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/manager.go @@ -33,8 +33,6 @@ import ( fabric "github.com/NearNodeFlash/nnf-ec/pkg/manager-fabric" msgreg "github.com/NearNodeFlash/nnf-ec/pkg/manager-message-registry/registries" - log "github.com/sirupsen/logrus" - ec "github.com/NearNodeFlash/nnf-ec/pkg/ec" "github.com/NearNodeFlash/nnf-ec/internal/switchtec/pkg/nvme" @@ -52,6 +50,18 @@ const ( DefaultStoragePoolId = "0" ) +const ( + switchIdKey = "switchId" + portIdKey = "portId" + slotKey = "slot" + + storageIdKey = "storageId" + volumeIdKey = "volumeId" + + namespaceIdKey = "namespaceId" + controllerIdKey = "controllerId" +) + var mgr = Manager{id: ResourceBlockId} func NewDefaultStorageService() StorageApi { @@ -70,6 +80,8 @@ type Manager struct { // Command-Line Options purge bool // Purge existing namespaces on storage controllers purgeMockDb bool // Purge the persistent mock databse + + log ec.Logger } // Storage - Storage defines a generic storage device in the Redfish / Swordfish specification. @@ -121,6 +133,8 @@ type Storage struct { config *ControllerConfig // Link to the storage configuration device NvmeDeviceApi // Device interface for interaction with the underlying NVMe device + + log ec.Logger } // StorageController - @@ -145,6 +159,7 @@ type Volume struct { capacityBytes uint64 storage *Storage + log ec.Logger } type ProvidingVolume struct { @@ -213,7 +228,7 @@ func CleanupVolumes(providingVolumes []ProvidingVolume) { } if err := storage.purgeVolumes(volIdsToKeep); err != nil { - log.Errorf("Cleanup Volumes: Failed to remove abandoned volumes on storage %s: %v", storage.id, err) + mgr.log.Error(err, "Failed to remove abandoned volumes", "storage", storage) } } } @@ -319,8 +334,10 @@ func (s *Storage) OdataIdRef(ref string) sf.OdataV4IdRef { func (s *Storage) initialize() error { - log.Infof("Storage %s Initialize", s.id) + log := s.manager.log.WithName(s.id).WithValues(storageIdKey, s.id, slotKey, s.slot) + log.Info("Initialize storage device") + s.log = log s.state = sf.STARTING_RST ctrl, err := s.device.IdentifyController(0) @@ -366,6 +383,14 @@ func (s *Storage) initialize() error { s.virtManagementEnabled = ctrl.GetCapability(nvme.VirtualiztionManagementSupport) + log.V(1).Info("Identified controller", + "serialNumber", s.serialNumber, + "modelNumber", s.modelNumber, + "firmwareRevision", s.firmwareRevision, + "capacityInBytes", s.capacityBytes, + "unallocatedBytes", s.unallocatedBytes, + "virtualizationManagement", s.virtManagementEnabled) + ns, err := s.device.IdentifyNamespace(CommonNamespaceIdentifier) if err != nil { return fmt.Errorf("Initialize Storage %s: Failed to identify common namespace: Error: %w", s.id, err) @@ -376,7 +401,7 @@ func (s *Storage) initialize() error { if ns.NumberOfLBAFormats == 1 { if ((1 << ns.LBAFormats[1].LBADataSize) == 4096) && (ns.LBAFormats[1].RelativePerformance == 0) { - log.Warnf("Initialize Storage %s: Detected Device %s; Incorrect number of LBA Formats. Expected: 2 Actual: %d", s.id, s.serialNumber, ns.NumberOfLBAFormats) + log.Info("Incorrect number of LBA formats", "expected", 2, "actual", ns.NumberOfLBAFormats) ns.NumberOfLBAFormats = 2 } } @@ -392,7 +417,9 @@ func (s *Storage) initialize() error { s.lbaFormatIndex = uint8(bestIndex) s.blockSizeBytes = 1 << ns.LBAFormats[bestIndex].LBADataSize - log.Infof("Storage %s Initialized: SerialNumber: %s", s.id, s.serialNumber) + s.log = s.log.WithValues("serialNumber", s.serialNumber) + s.log.Info("Initialized storage device") + return nil } @@ -434,7 +461,6 @@ vol_loop: } for _, volId := range volIdsToDelete { - log.Infof("Storage %s Volume ID %s is being removed", s.id, volId) if err := s.deleteVolume(volId); err != nil { return err } @@ -473,31 +499,45 @@ func (s *Storage) createVolume(desiredCapacityInBytes uint64) (*Volume, error) { actualCapacityBytes := roundUpToMultiple(desiredCapacityInBytes, s.blockSizeBytes) + s.log.V(2).Info("Creating namespace", "capacityInBytes", actualCapacityBytes, "formatIndex", s.lbaFormatIndex) namespaceId, guid, err := s.device.CreateNamespace(actualCapacityBytes/s.blockSizeBytes, s.lbaFormatIndex) if err != nil { return nil, err } - s.unallocatedBytes -= actualCapacityBytes - id := strconv.Itoa(int(namespaceId)) - s.volumes = append(s.volumes, Volume{ + + log := s.log.WithName(id).WithValues(namespaceIdKey, namespaceId) + log.V(1).Info("Created namespace") + + volume := Volume{ id: id, namespaceId: namespaceId, guid: guid, capacityBytes: actualCapacityBytes, storage: s, - }) + log: log, + } + + s.volumes = append(s.volumes, volume) + + s.unallocatedBytes -= actualCapacityBytes return &s.volumes[len(s.volumes)-1], nil } func (s *Storage) deleteVolume(volumeId string) error { + for idx, volume := range s.volumes { if volume.id == volumeId { + log := volume.log + + log.V(2).Info("Deleting namespace") if err := s.device.DeleteNamespace(volume.namespaceId); err != nil { + log.Error(err, "Delete namespace failed") return err } + log.V(1).Info("Deleted namespace") s.unallocatedBytes += volume.capacityBytes @@ -515,9 +555,14 @@ func (s *Storage) deleteVolume(volumeId string) error { func (s *Storage) formatVolume(volumeID string) error { for _, volume := range s.volumes { if volume.id == volumeID { + log := volume.log + + log.V(2).Info("Format namespace") if err := s.device.FormatNamespace(volume.namespaceId); err != nil { + log.Error(err, "Format namespace failure") return err } + log.V(1).Info("Formatted namespace") return nil } @@ -527,26 +572,38 @@ func (s *Storage) formatVolume(volumeID string) error { } func (s *Storage) recoverVolumes() error { + s.log.V(1).Info("Recovering volumes") + namespaces, err := s.device.ListNamespaces(0) if err != nil { - log.WithError(err).Errorf("Storage %s Failed to list device namespaces", s.id) + s.log.Error(err, "Failed to list device namespaces") } + s.volumes = make([]Volume, 0) for _, nsid := range namespaces { + log := s.log.WithValues(namespaceIdKey, nsid) + + log.V(2).Info("Identify namespace") ns, err := s.device.IdentifyNamespace(nsid) if err != nil { - log.WithError(err).Errorf("Storage %s Failed to identify namespaces %d", s.id, nsid) + log.Error(err, "Failed to identify namespace") } blockSizeBytes := uint64(1 << ns.LBAFormats[ns.FormattedLBASize.Format].LBADataSize) - s.volumes = append(s.volumes, Volume{ - id: strconv.Itoa(int(nsid)), + id := strconv.Itoa(int(nsid)) + volume := Volume{ + id: id, namespaceId: nsid, guid: ns.GloballyUniqueIdentifier, capacityBytes: ns.Capacity * blockSizeBytes, storage: s, - }) + log: log.WithName(id), + } + + s.volumes = append(s.volumes, volume) + + s.log.V(1).Info("Recovered Volume", "volume", volume) } return nil @@ -612,13 +669,16 @@ func (v *Volume) runInAttachDetachBlock(fn func() error) error { // Wait for Format Completion by polling on the namespace Utilization value to reach zero. func (v *Volume) WaitFormatComplete() error { + log := v.log + log.V(2).Info("Wait for format completion") ns, err := v.storage.device.IdentifyNamespace(v.namespaceId) if err != nil { return err } for ns.Utilization != 0 { + log.V(3).Info("Namespace in use", "utilization", ns.Utilization) const delay = 100 * time.Millisecond @@ -633,10 +693,12 @@ func (v *Volume) WaitFormatComplete() error { } if lastUtilization == ns.Utilization { - return fmt.Errorf("Device %s Format Stalled: Namespace: %d Delay: %s", v.storage.id, v.namespaceId, delay.String()) + return fmt.Errorf("Device %s Format Stalled: Namespace: %d Delay: %s Utilization: %d", v.storage.id, v.namespaceId, delay.String(), ns.Utilization) } } + log.V(1).Info("Format completed", "utilization", ns.Utilization) + return nil } @@ -650,20 +712,25 @@ func (v *Volume) attach(controllerId uint16) error { // still use the secondary controller values for all other ports, but we need to remap the // first index to the physical function. // - if v.storage.device.IsDirectDevice() { - if controllerId == 1 { - controllerId = v.storage.physicalFunctionControllerId + if controllerId != v.storage.physicalFunctionControllerId { + if v.storage.device.IsDirectDevice() { + if controllerId == 1 { + controllerId = v.storage.physicalFunctionControllerId + } + } else if v.storage.virtManagementEnabled { + controllerId = v.storage.controllers[controllerId].controllerId + } else if v.storage.IsKioxiaDualPortConfiguration() { + controllerId = controllerId + 2 } - } else if v.storage.virtManagementEnabled { - controllerId = v.storage.controllers[controllerId].controllerId - } else if v.storage.IsKioxiaDualPortConfiguration() { - controllerId = controllerId + 2 } + log := v.log.WithValues(controllerIdKey, controllerId) + log.V(2).Info("Attach namespace") + err := v.storage.device.AttachNamespace(v.namespaceId, []uint16{controllerId}) if err != nil { - log.Infof("Device %s Attach Namespace: %d Controller: %d Error: %s", v.storage.id, v.namespaceId, controllerId, err) + log.Error(err, "Attach namespace failed") var cmdErr *nvme.CommandError if errors.As(err, &cmdErr) { @@ -675,26 +742,33 @@ func (v *Volume) attach(controllerId uint16) error { } } + log.V(1).Info("Attached namespace") + return nil } func (v *Volume) detach(controllerId uint16) error { // See the note on "attach" above - if v.storage.device.IsDirectDevice() { - if controllerId == 1 { - controllerId = v.storage.physicalFunctionControllerId + if controllerId != v.storage.physicalFunctionControllerId { + if v.storage.device.IsDirectDevice() { + if controllerId == 1 { + controllerId = v.storage.physicalFunctionControllerId + } + } else if v.storage.virtManagementEnabled { + controllerId = v.storage.controllers[controllerId].controllerId + } else if v.storage.IsKioxiaDualPortConfiguration() { + controllerId = controllerId + 2 } - } else if v.storage.virtManagementEnabled { - controllerId = v.storage.controllers[controllerId].controllerId - } else if v.storage.IsKioxiaDualPortConfiguration() { - controllerId = controllerId + 2 } + log := v.log.WithValues(controllerIdKey, controllerId) + log.V(2).Info("Detach namespace") + err := v.storage.device.DetachNamespace(v.namespaceId, []uint16{controllerId}) if err != nil { - log.Infof("Device %s Detach Namespace: %d Controller: %d Error: %s", v.storage.id, v.namespaceId, controllerId, err) + log.Error(err, "Detach namespace failed") var cmdErr *nvme.CommandError if errors.As(err, &cmdErr) { @@ -706,37 +780,33 @@ func (v *Volume) detach(controllerId uint16) error { } } + log.V(1).Info("Detached namespace") + return nil } // Initialize -func Initialize(ctrl NvmeController) error { +func Initialize(log ec.Logger, ctrl NvmeController) error { + log.Info("Initialize NVMe Manager") - log.SetLevel(log.DebugLevel) // TODO: Config file or command-line option - log.Infof("Initialize %s NVMe Namespace Manager", mgr.id) + const name = "nvme" + log.V(2).Info("Creating logger", "name", name) + log = log.WithName(name) + mgr.log = log conf, err := loadConfig() if err != nil { - log.WithError(err).Errorf("Failed to load %s configuration", mgr.id) + log.Error(err, "failed to load configuration", "id", mgr.id) return err } mgr.config = conf - log.Debugf("NVMe Configuration '%s' Loaded...", conf.Metadata.Name) - log.Debugf(" Debug Level: %s", conf.DebugLevel) - log.Debugf(" Controller Config:") - log.Debugf(" Virtual Functions: %d", conf.Storage.Controller.Functions) - log.Debugf(" Num Resources: %d", conf.Storage.Controller.Resources) - log.Debugf(" Device List: %+v", conf.Storage.Devices) - - level, err := log.ParseLevel(conf.DebugLevel) - if err != nil { - log.WithError(err).Errorf("Failed to parse debug level: %s", conf.DebugLevel) - return err - } + log.V(1).Info("Loaded configuration", + "virtualFunctions", conf.Storage.Controller.Functions, + "resources", conf.Storage.Controller.Resources) - log.SetLevel(level) + mgr.log = log mgr.storage = make([]Storage, len(conf.Storage.Devices)) for storageIdx, storageDevice := range conf.Storage.Devices { @@ -751,7 +821,7 @@ func Initialize(ctrl NvmeController) error { mgr.ctrl = ctrl.NewNvmeDeviceController() if err := mgr.ctrl.Initialize(); err != nil { - log.WithError(err).Errorf("Failed to initialize NVMe Device Controller") + log.Error(err, "Failed to initialize NVMe Device Controller") return err } @@ -765,12 +835,13 @@ func Close() error { } func (m *Manager) EventHandler(e event.Event) error { + log := m.log.WithValues("eventId", e.Id, "eventMessage", e.Message, "eventArgs", e.MessageArgs) linkEstablished := e.Is(msgreg.DownstreamLinkEstablishedFabric("", "")) || e.Is(msgreg.DegradedDownstreamLinkEstablishedFabric("", "")) linkDropped := e.Is(msgreg.DownstreamLinkDroppedFabric("", "")) if linkEstablished || linkDropped { - log.Infof("NVMe Manager: Event Received %+v", e) + log.V(2).Info("Link event received") var switchId, portId string if err := e.Args(&switchId, &portId); err != nil { @@ -797,6 +868,7 @@ func (m *Manager) EventHandler(e event.Event) error { } storage.slot = port.GetSlot() + storage.log = m.log.WithValues(slotKey, storage.slot) if linkEstablished { return storage.LinkEstablishedEventHandler(switchId, portId) @@ -811,24 +883,28 @@ func (m *Manager) EventHandler(e event.Event) error { } func (s *Storage) LinkEstablishedEventHandler(switchId, portId string) error { + log := s.log.WithValues(switchIdKey, switchId, portIdKey, portId) + // Connect device, err := s.manager.ctrl.NewNvmeDevice(fabric.FabricId, switchId, portId) if err != nil { - log.WithError(err).Errorf("Storage %s - Could not allocate storage controller", s.id) + log.Error(err, "Could not allocate storage controller") return err } s.device = device if err := s.initialize(); err != nil { - log.WithError(err).Errorf("Storage %s - Failed to initialize device controller", s.id) + log.Error(err, "Failed to initialize storage device") return err } + log = s.log // switch to using the storage logger + if s.manager.purge { - log.Warnf("Storage %s - Starting purge of existing volumes", s.id) + log.Info("Purging existing volumes") if err := s.purge(); err != nil { - log.WithError(err).Errorf("Storage %s - Failed to purge storage device", s.id) + log.Error(err, "Failed to purge storage volumes") } } @@ -859,10 +935,11 @@ func (s *Storage) LinkEstablishedEventHandler(switchId, portId string) error { } } else { + log.V(2).Info("List Secondary Controllers") ls, err := device.ListSecondary() if err != nil { - log.WithError(err).Errorf("List Secondary command failed") + log.Error(err, "List Secondary failed") return err } @@ -881,8 +958,9 @@ func (s *Storage) LinkEstablishedEventHandler(switchId, portId string) error { } for idx, sc := range ls.Entries[:count] { + if sc.SecondaryControllerID == 0 { - log.Errorf("Secondary Controller ID overlaps with PF Controller ID") + log.Info("Secondary Controller ID overlaps with PF Controller ID") break } @@ -897,21 +975,23 @@ func (s *Storage) LinkEstablishedEventHandler(switchId, portId string) error { } ctrl := &s.controllers[idx+1] + log := log.WithValues(controllerIdKey, ctrl.id) if !s.IsKioxiaDualPortConfiguration() { - log.Debugf("Storage %s Initialize Secondary Controller %s", s.id, ctrl.id) + log.V(2).Info("Initialize VQ resources", "resources", s.config.Resources) if sc.VQFlexibleResourcesAssigned != uint16(s.config.Resources) { - if err := s.device.AssignControllerResources(sc.SecondaryControllerID, VQResourceType, s.config.Resources-uint32(sc.VQFlexibleResourcesAssigned)); err != nil { - log.WithError(err).Errorf("Secondary Controller %d: Failed to assign VQ Resources", sc.SecondaryControllerID) + if err := s.device.AssignControllerResources(sc.SecondaryControllerID, VQResourceType, s.config.Resources); err != nil { + log.Error(err, "Failed to assign VQ Resources") break } ctrl.vqResources = uint16(s.config.Resources) } + log.V(2).Info("Initialize VI resources", "resources", s.config.Resources) if sc.VIFlexibleResourcesAssigned != uint16(s.config.Resources) { - if err := s.device.AssignControllerResources(sc.SecondaryControllerID, VIResourceType, s.config.Resources-uint32(sc.VIFlexibleResourcesAssigned)); err != nil { - log.WithError(err).Errorf("Secondary Controller %d: Failed to assign VI Resources", sc.SecondaryControllerID) + if err := s.device.AssignControllerResources(sc.SecondaryControllerID, VIResourceType, s.config.Resources); err != nil { + log.Error(err, "Failed to assign VI resources") break } @@ -919,41 +999,46 @@ func (s *Storage) LinkEstablishedEventHandler(switchId, portId string) error { } if sc.SecondaryControllerState&0x01 == 0 { + log.V(2).Info("Reset controller") if err := fabric.FabricController.ResetEndpoint(switchId, portId, idx); err != nil { - log.WithError(err).Errorf("Secondary Controller %d: Failed to reset controller", sc.SecondaryControllerID) + log.Error(err, "Failed to reset controller") } } } if sc.SecondaryControllerState&0x01 == 0 { + log.V(2).Info("Online controller") if err := s.device.OnlineController(sc.SecondaryControllerID); err != nil { - log.WithError(err).Errorf("Secondary Controller %d: Failed to online controller", sc.SecondaryControllerID) + log.Error(err, "Failed to online controller") break } ctrl.online = true } - log.Infof("Storage %s Secondary Controller %s Initialized", s.id, ctrl.id) + log.V(2).Info("Secondary controller initialized") } // for := secondary controllers for _, ctrl := range s.controllers[1:] { if !ctrl.online { s.state = sf.DISABLED_RST - log.Errorf("Secondary Controller %s Offline - Storage %s Not Ready.", ctrl.id, s.id) + log.Info("Secondary Controller Offline") return nil } } } + log.V(1).Info("Storage initialized") + // Recover existing volumes + log.V(2).Info("Recovering volumes") if err := s.recoverVolumes(); err != nil { - log.WithError(err).Errorf("Storage %s - Failed to recover existing volumes", s.id) + log.Error(err, "Failed to recover existing volumes") return err } - log.Infof("Storage %s - Ready", s.id) + log.Info("Storage Ready") s.state = sf.ENABLED_RST event.EventManager.Publish(msgreg.PortAutomaticallyEnabledFabric(switchId, portId)) @@ -1146,7 +1231,7 @@ func (mgr *Manager) StorageIdVolumeIdGet(storageId, volumeId string, model *sf.V ns, err := s.device.IdentifyNamespace(nvme.NamespaceIdentifier(v.namespaceId)) if err != nil { - log.WithError(err).Errorf("Identify Namespace Failed: NSID %d", v.namespaceId) + v.log.Error(err, "Identify Namespace Failed") return ec.NewErrInternalServerError() } diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/router.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/router.go index c79ccc737..5cfcc2578 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/router.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-nvme/router.go @@ -48,8 +48,8 @@ func (*DefaultApiRouter) Name() string { } // Init - -func (r *DefaultApiRouter) Init() error { - return Initialize(r.controller) +func (r *DefaultApiRouter) Init(log ec.Logger) error { + return Initialize(log, r.controller) } // Start - diff --git a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-telemetry/router.go b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-telemetry/router.go index 033478fd1..e0b3d7bd7 100644 --- a/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-telemetry/router.go +++ b/vendor/github.com/NearNodeFlash/nnf-ec/pkg/manager-telemetry/router.go @@ -36,7 +36,7 @@ func (*DefaultApiRouter) Name() string { return "Telemetry Manager" } -func (*DefaultApiRouter) Init() error { +func (*DefaultApiRouter) Init(ec.Logger) error { return TelemetryManager.Initialize() } diff --git a/vendor/github.com/cespare/xxhash/v2/README.md b/vendor/github.com/cespare/xxhash/v2/README.md index 792b4a60b..8bf0e5b78 100644 --- a/vendor/github.com/cespare/xxhash/v2/README.md +++ b/vendor/github.com/cespare/xxhash/v2/README.md @@ -3,8 +3,7 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/cespare/xxhash/v2.svg)](https://pkg.go.dev/github.com/cespare/xxhash/v2) [![Test](https://github.com/cespare/xxhash/actions/workflows/test.yml/badge.svg)](https://github.com/cespare/xxhash/actions/workflows/test.yml) -xxhash is a Go implementation of the 64-bit -[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a +xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a high-quality hashing algorithm that is much faster than anything in the Go standard library. @@ -25,8 +24,11 @@ func (*Digest) WriteString(string) (int, error) func (*Digest) Sum64() uint64 ``` -This implementation provides a fast pure-Go implementation and an even faster -assembly implementation for amd64. +The package is written with optimized pure Go and also contains even faster +assembly implementations for amd64 and arm64. If desired, the `purego` build tag +opts into using the Go code even on those architectures. + +[xxHash]: http://cyan4973.github.io/xxHash/ ## Compatibility @@ -45,19 +47,20 @@ I recommend using the latest release of Go. Here are some quick benchmarks comparing the pure-Go and assembly implementations of Sum64. -| input size | purego | asm | -| --- | --- | --- | -| 5 B | 979.66 MB/s | 1291.17 MB/s | -| 100 B | 7475.26 MB/s | 7973.40 MB/s | -| 4 KB | 17573.46 MB/s | 17602.65 MB/s | -| 10 MB | 17131.46 MB/s | 17142.16 MB/s | +| input size | purego | asm | +| ---------- | --------- | --------- | +| 4 B | 1.3 GB/s | 1.2 GB/s | +| 16 B | 2.9 GB/s | 3.5 GB/s | +| 100 B | 6.9 GB/s | 8.1 GB/s | +| 4 KB | 11.7 GB/s | 16.7 GB/s | +| 10 MB | 12.0 GB/s | 17.3 GB/s | -These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using -the following commands under Go 1.11.2: +These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C +CPU using the following commands under Go 1.19.2: ``` -$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes' -$ go test -benchtime 10s -bench '/xxhash,direct,bytes' +benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$') +benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$') ``` ## Projects using this package diff --git a/vendor/github.com/cespare/xxhash/v2/testall.sh b/vendor/github.com/cespare/xxhash/v2/testall.sh new file mode 100644 index 000000000..94b9c4439 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/v2/testall.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -eu -o pipefail + +# Small convenience script for running the tests with various combinations of +# arch/tags. This assumes we're running on amd64 and have qemu available. + +go test ./... +go test -tags purego ./... +GOARCH=arm64 go test +GOARCH=arm64 go test -tags purego diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash.go b/vendor/github.com/cespare/xxhash/v2/xxhash.go index 15c835d54..a9e0d45c9 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash.go @@ -16,19 +16,11 @@ const ( prime5 uint64 = 2870177450012600261 ) -// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where -// possible in the Go code is worth a small (but measurable) performance boost -// by avoiding some MOVQs. Vars are needed for the asm and also are useful for -// convenience in the Go code in a few places where we need to intentionally -// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the -// result overflows a uint64). -var ( - prime1v = prime1 - prime2v = prime2 - prime3v = prime3 - prime4v = prime4 - prime5v = prime5 -) +// Store the primes in an array as well. +// +// The consts are used when possible in Go code to avoid MOVs but we need a +// contiguous array of the assembly code. +var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5} // Digest implements hash.Hash64. type Digest struct { @@ -50,10 +42,10 @@ func New() *Digest { // Reset clears the Digest's state so that it can be reused. func (d *Digest) Reset() { - d.v1 = prime1v + prime2 + d.v1 = primes[0] + prime2 d.v2 = prime2 d.v3 = 0 - d.v4 = -prime1v + d.v4 = -primes[0] d.total = 0 d.n = 0 } @@ -69,21 +61,23 @@ func (d *Digest) Write(b []byte) (n int, err error) { n = len(b) d.total += uint64(n) + memleft := d.mem[d.n&(len(d.mem)-1):] + if d.n+n < 32 { // This new data doesn't even fill the current block. - copy(d.mem[d.n:], b) + copy(memleft, b) d.n += n return } if d.n > 0 { // Finish off the partial block. - copy(d.mem[d.n:], b) + c := copy(memleft, b) d.v1 = round(d.v1, u64(d.mem[0:8])) d.v2 = round(d.v2, u64(d.mem[8:16])) d.v3 = round(d.v3, u64(d.mem[16:24])) d.v4 = round(d.v4, u64(d.mem[24:32])) - b = b[32-d.n:] + b = b[c:] d.n = 0 } @@ -133,21 +127,20 @@ func (d *Digest) Sum64() uint64 { h += d.total - i, end := 0, d.n - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(d.mem[i:i+8])) + b := d.mem[:d.n&(len(d.mem)-1)] + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(d.mem[i:i+4])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for i < end { - h ^= uint64(d.mem[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 - i++ } h ^= h >> 33 diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s b/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s index be8db5bf7..3e8b13257 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s @@ -1,215 +1,209 @@ +//go:build !appengine && gc && !purego // +build !appengine // +build gc // +build !purego #include "textflag.h" -// Register allocation: -// AX h -// SI pointer to advance through b -// DX n -// BX loop end -// R8 v1, k1 -// R9 v2 -// R10 v3 -// R11 v4 -// R12 tmp -// R13 prime1v -// R14 prime2v -// DI prime4v - -// round reads from and advances the buffer pointer in SI. -// It assumes that R13 has prime1v and R14 has prime2v. -#define round(r) \ - MOVQ (SI), R12 \ - ADDQ $8, SI \ - IMULQ R14, R12 \ - ADDQ R12, r \ - ROLQ $31, r \ - IMULQ R13, r - -// mergeRound applies a merge round on the two registers acc and val. -// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v. -#define mergeRound(acc, val) \ - IMULQ R14, val \ - ROLQ $31, val \ - IMULQ R13, val \ - XORQ val, acc \ - IMULQ R13, acc \ - ADDQ DI, acc +// Registers: +#define h AX +#define d AX +#define p SI // pointer to advance through b +#define n DX +#define end BX // loop end +#define v1 R8 +#define v2 R9 +#define v3 R10 +#define v4 R11 +#define x R12 +#define prime1 R13 +#define prime2 R14 +#define prime4 DI + +#define round(acc, x) \ + IMULQ prime2, x \ + ADDQ x, acc \ + ROLQ $31, acc \ + IMULQ prime1, acc + +// round0 performs the operation x = round(0, x). +#define round0(x) \ + IMULQ prime2, x \ + ROLQ $31, x \ + IMULQ prime1, x + +// mergeRound applies a merge round on the two registers acc and x. +// It assumes that prime1, prime2, and prime4 have been loaded. +#define mergeRound(acc, x) \ + round0(x) \ + XORQ x, acc \ + IMULQ prime1, acc \ + ADDQ prime4, acc + +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that there is at least one block +// to process. +#define blockLoop() \ +loop: \ + MOVQ +0(p), x \ + round(v1, x) \ + MOVQ +8(p), x \ + round(v2, x) \ + MOVQ +16(p), x \ + round(v3, x) \ + MOVQ +24(p), x \ + round(v4, x) \ + ADDQ $32, p \ + CMPQ p, end \ + JLE loop // func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOSPLIT, $0-32 +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 // Load fixed primes. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 - MOVQ ·prime4v(SB), DI + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 + MOVQ ·primes+24(SB), prime4 // Load slice. - MOVQ b_base+0(FP), SI - MOVQ b_len+8(FP), DX - LEAQ (SI)(DX*1), BX + MOVQ b_base+0(FP), p + MOVQ b_len+8(FP), n + LEAQ (p)(n*1), end // The first loop limit will be len(b)-32. - SUBQ $32, BX + SUBQ $32, end // Check whether we have at least one block. - CMPQ DX, $32 + CMPQ n, $32 JLT noBlocks // Set up initial state (v1, v2, v3, v4). - MOVQ R13, R8 - ADDQ R14, R8 - MOVQ R14, R9 - XORQ R10, R10 - XORQ R11, R11 - SUBQ R13, R11 - - // Loop until SI > BX. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ SI, BX - JLE blockLoop - - MOVQ R8, AX - ROLQ $1, AX - MOVQ R9, R12 - ROLQ $7, R12 - ADDQ R12, AX - MOVQ R10, R12 - ROLQ $12, R12 - ADDQ R12, AX - MOVQ R11, R12 - ROLQ $18, R12 - ADDQ R12, AX - - mergeRound(AX, R8) - mergeRound(AX, R9) - mergeRound(AX, R10) - mergeRound(AX, R11) + MOVQ prime1, v1 + ADDQ prime2, v1 + MOVQ prime2, v2 + XORQ v3, v3 + XORQ v4, v4 + SUBQ prime1, v4 + + blockLoop() + + MOVQ v1, h + ROLQ $1, h + MOVQ v2, x + ROLQ $7, x + ADDQ x, h + MOVQ v3, x + ROLQ $12, x + ADDQ x, h + MOVQ v4, x + ROLQ $18, x + ADDQ x, h + + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) JMP afterBlocks noBlocks: - MOVQ ·prime5v(SB), AX + MOVQ ·primes+32(SB), h afterBlocks: - ADDQ DX, AX - - // Right now BX has len(b)-32, and we want to loop until SI > len(b)-8. - ADDQ $24, BX - - CMPQ SI, BX - JG fourByte - -wordLoop: - // Calculate k1. - MOVQ (SI), R8 - ADDQ $8, SI - IMULQ R14, R8 - ROLQ $31, R8 - IMULQ R13, R8 - - XORQ R8, AX - ROLQ $27, AX - IMULQ R13, AX - ADDQ DI, AX - - CMPQ SI, BX - JLE wordLoop - -fourByte: - ADDQ $4, BX - CMPQ SI, BX - JG singles - - MOVL (SI), R8 - ADDQ $4, SI - IMULQ R13, R8 - XORQ R8, AX - - ROLQ $23, AX - IMULQ R14, AX - ADDQ ·prime3v(SB), AX - -singles: - ADDQ $4, BX - CMPQ SI, BX + ADDQ n, h + + ADDQ $24, end + CMPQ p, end + JG try4 + +loop8: + MOVQ (p), x + ADDQ $8, p + round0(x) + XORQ x, h + ROLQ $27, h + IMULQ prime1, h + ADDQ prime4, h + + CMPQ p, end + JLE loop8 + +try4: + ADDQ $4, end + CMPQ p, end + JG try1 + + MOVL (p), x + ADDQ $4, p + IMULQ prime1, x + XORQ x, h + + ROLQ $23, h + IMULQ prime2, h + ADDQ ·primes+16(SB), h + +try1: + ADDQ $4, end + CMPQ p, end JGE finalize -singlesLoop: - MOVBQZX (SI), R12 - ADDQ $1, SI - IMULQ ·prime5v(SB), R12 - XORQ R12, AX +loop1: + MOVBQZX (p), x + ADDQ $1, p + IMULQ ·primes+32(SB), x + XORQ x, h + ROLQ $11, h + IMULQ prime1, h - ROLQ $11, AX - IMULQ R13, AX - - CMPQ SI, BX - JL singlesLoop + CMPQ p, end + JL loop1 finalize: - MOVQ AX, R12 - SHRQ $33, R12 - XORQ R12, AX - IMULQ R14, AX - MOVQ AX, R12 - SHRQ $29, R12 - XORQ R12, AX - IMULQ ·prime3v(SB), AX - MOVQ AX, R12 - SHRQ $32, R12 - XORQ R12, AX - - MOVQ AX, ret+24(FP) + MOVQ h, x + SHRQ $33, x + XORQ x, h + IMULQ prime2, h + MOVQ h, x + SHRQ $29, x + XORQ x, h + IMULQ ·primes+16(SB), h + MOVQ h, x + SHRQ $32, x + XORQ x, h + + MOVQ h, ret+24(FP) RET -// writeBlocks uses the same registers as above except that it uses AX to store -// the d pointer. - // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT, $0-40 +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 // Load fixed primes needed for round. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 // Load slice. - MOVQ b_base+8(FP), SI - MOVQ b_len+16(FP), DX - LEAQ (SI)(DX*1), BX - SUBQ $32, BX + MOVQ b_base+8(FP), p + MOVQ b_len+16(FP), n + LEAQ (p)(n*1), end + SUBQ $32, end // Load vN from d. - MOVQ d+0(FP), AX - MOVQ 0(AX), R8 // v1 - MOVQ 8(AX), R9 // v2 - MOVQ 16(AX), R10 // v3 - MOVQ 24(AX), R11 // v4 + MOVQ s+0(FP), d + MOVQ 0(d), v1 + MOVQ 8(d), v2 + MOVQ 16(d), v3 + MOVQ 24(d), v4 // We don't need to check the loop condition here; this function is // always called with at least one block of data to process. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ SI, BX - JLE blockLoop + blockLoop() // Copy vN back to d. - MOVQ R8, 0(AX) - MOVQ R9, 8(AX) - MOVQ R10, 16(AX) - MOVQ R11, 24(AX) - - // The number of bytes written is SI minus the old base pointer. - SUBQ b_base+8(FP), SI - MOVQ SI, ret+32(FP) + MOVQ v1, 0(d) + MOVQ v2, 8(d) + MOVQ v3, 16(d) + MOVQ v4, 24(d) + + // The number of bytes written is p minus the old base pointer. + SUBQ b_base+8(FP), p + MOVQ p, ret+32(FP) RET diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s b/vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s new file mode 100644 index 000000000..7e3145a22 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s @@ -0,0 +1,183 @@ +//go:build !appengine && gc && !purego +// +build !appengine +// +build gc +// +build !purego + +#include "textflag.h" + +// Registers: +#define digest R1 +#define h R2 // return value +#define p R3 // input pointer +#define n R4 // input length +#define nblocks R5 // n / 32 +#define prime1 R7 +#define prime2 R8 +#define prime3 R9 +#define prime4 R10 +#define prime5 R11 +#define v1 R12 +#define v2 R13 +#define v3 R14 +#define v4 R15 +#define x1 R20 +#define x2 R21 +#define x3 R22 +#define x4 R23 + +#define round(acc, x) \ + MADD prime2, acc, x, acc \ + ROR $64-31, acc \ + MUL prime1, acc + +// round0 performs the operation x = round(0, x). +#define round0(x) \ + MUL prime2, x \ + ROR $64-31, x \ + MUL prime1, x + +#define mergeRound(acc, x) \ + round0(x) \ + EOR x, acc \ + MADD acc, prime4, prime1, acc + +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that n >= 32. +#define blockLoop() \ + LSR $5, n, nblocks \ + PCALIGN $16 \ + loop: \ + LDP.P 16(p), (x1, x2) \ + LDP.P 16(p), (x3, x4) \ + round(v1, x1) \ + round(v2, x2) \ + round(v3, x3) \ + round(v4, x4) \ + SUB $1, nblocks \ + CBNZ nblocks, loop + +// func Sum64(b []byte) uint64 +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 + LDP b_base+0(FP), (p, n) + + LDP ·primes+0(SB), (prime1, prime2) + LDP ·primes+16(SB), (prime3, prime4) + MOVD ·primes+32(SB), prime5 + + CMP $32, n + CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 } + BLT afterLoop + + ADD prime1, prime2, v1 + MOVD prime2, v2 + MOVD $0, v3 + NEG prime1, v4 + + blockLoop() + + ROR $64-1, v1, x1 + ROR $64-7, v2, x2 + ADD x1, x2 + ROR $64-12, v3, x3 + ROR $64-18, v4, x4 + ADD x3, x4 + ADD x2, x4, h + + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) + +afterLoop: + ADD n, h + + TBZ $4, n, try8 + LDP.P 16(p), (x1, x2) + + round0(x1) + + // NOTE: here and below, sequencing the EOR after the ROR (using a + // rotated register) is worth a small but measurable speedup for small + // inputs. + ROR $64-27, h + EOR x1 @> 64-27, h, h + MADD h, prime4, prime1, h + + round0(x2) + ROR $64-27, h + EOR x2 @> 64-27, h, h + MADD h, prime4, prime1, h + +try8: + TBZ $3, n, try4 + MOVD.P 8(p), x1 + + round0(x1) + ROR $64-27, h + EOR x1 @> 64-27, h, h + MADD h, prime4, prime1, h + +try4: + TBZ $2, n, try2 + MOVWU.P 4(p), x2 + + MUL prime1, x2 + ROR $64-23, h + EOR x2 @> 64-23, h, h + MADD h, prime3, prime2, h + +try2: + TBZ $1, n, try1 + MOVHU.P 2(p), x3 + AND $255, x3, x1 + LSR $8, x3, x2 + + MUL prime5, x1 + ROR $64-11, h + EOR x1 @> 64-11, h, h + MUL prime1, h + + MUL prime5, x2 + ROR $64-11, h + EOR x2 @> 64-11, h, h + MUL prime1, h + +try1: + TBZ $0, n, finalize + MOVBU (p), x4 + + MUL prime5, x4 + ROR $64-11, h + EOR x4 @> 64-11, h, h + MUL prime1, h + +finalize: + EOR h >> 33, h + MUL prime2, h + EOR h >> 29, h + MUL prime3, h + EOR h >> 32, h + + MOVD h, ret+24(FP) + RET + +// func writeBlocks(d *Digest, b []byte) int +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 + LDP ·primes+0(SB), (prime1, prime2) + + // Load state. Assume v[1-4] are stored contiguously. + MOVD d+0(FP), digest + LDP 0(digest), (v1, v2) + LDP 16(digest), (v3, v4) + + LDP b_base+8(FP), (p, n) + + blockLoop() + + // Store updated state. + STP (v1, v2), 0(digest) + STP (v3, v4), 16(digest) + + BIC $31, n + MOVD n, ret+32(FP) + RET diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.go b/vendor/github.com/cespare/xxhash/v2/xxhash_asm.go similarity index 73% rename from vendor/github.com/cespare/xxhash/v2/xxhash_amd64.go rename to vendor/github.com/cespare/xxhash/v2/xxhash_asm.go index ad14b807f..9216e0a40 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_asm.go @@ -1,3 +1,5 @@ +//go:build (amd64 || arm64) && !appengine && gc && !purego +// +build amd64 arm64 // +build !appengine // +build gc // +build !purego diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_other.go b/vendor/github.com/cespare/xxhash/v2/xxhash_other.go index 4a5a82160..26df13bba 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_other.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_other.go @@ -1,4 +1,5 @@ -// +build !amd64 appengine !gc purego +//go:build (!amd64 && !arm64) || appengine || !gc || purego +// +build !amd64,!arm64 appengine !gc purego package xxhash @@ -14,10 +15,10 @@ func Sum64(b []byte) uint64 { var h uint64 if n >= 32 { - v1 := prime1v + prime2 + v1 := primes[0] + prime2 v2 := prime2 v3 := uint64(0) - v4 := -prime1v + v4 := -primes[0] for len(b) >= 32 { v1 = round(v1, u64(b[0:8:len(b)])) v2 = round(v2, u64(b[8:16:len(b)])) @@ -36,19 +37,18 @@ func Sum64(b []byte) uint64 { h += uint64(n) - i, end := 0, len(b) - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(b[i:i+8:len(b)])) + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for ; i < end; i++ { - h ^= uint64(b[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 } diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go b/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go index fc9bea7a3..e86f1b5fd 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go @@ -1,3 +1,4 @@ +//go:build appengine // +build appengine // This file contains the safe implementations of otherwise unsafe-using code. diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go b/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go index 376e0ca2e..1c1638fd8 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go @@ -1,3 +1,4 @@ +//go:build !appengine // +build !appengine // This file encapsulates usage of unsafe. @@ -11,7 +12,7 @@ import ( // In the future it's possible that compiler optimizations will make these // XxxString functions unnecessary by realizing that calls such as -// Sum64([]byte(s)) don't need to copy s. See https://golang.org/issue/2205. +// Sum64([]byte(s)) don't need to copy s. See https://go.dev/issue/2205. // If that happens, even if we keep these functions they can be replaced with // the trivial safe code. diff --git a/vendor/github.com/dgraph-io/badger/v3/.go-version b/vendor/github.com/dgraph-io/badger/v3/.go-version new file mode 100644 index 000000000..adc97d8e2 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/v3/.go-version @@ -0,0 +1 @@ +1.18 diff --git a/vendor/github.com/dgraph-io/badger/v3/.golangci.yml b/vendor/github.com/dgraph-io/badger/v3/.golangci.yml index fecb8644b..81060fb1c 100644 --- a/vendor/github.com/dgraph-io/badger/v3/.golangci.yml +++ b/vendor/github.com/dgraph-io/badger/v3/.golangci.yml @@ -3,25 +3,19 @@ run: linters-settings: lll: - line-length: 100 + line-length: 120 linters: disable-all: true enable: - - errcheck - - ineffassign - - gas +# - errcheck +# - ineffassign +# - gas - gofmt - - golint +# - golint - gosimple - govet - lll - - varcheck - - unused - -issues: - exclude-rules: - - linters: - - gosec - text: "G404: " - \ No newline at end of file +# - varcheck +# - unused +# - gosec diff --git a/vendor/github.com/dgraph-io/badger/v3/.travis.yml b/vendor/github.com/dgraph-io/badger/v3/.travis.yml deleted file mode 100644 index b671868f3..000000000 --- a/vendor/github.com/dgraph-io/badger/v3/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -language: go - -go: - - "1.12" - - "1.13" - - tip -os: - - osx -env: - jobs: - - GOARCH=386 - - GOARCH=amd64 - global: - - secure: CRkV2+/jlO0gXzzS50XGxfMS117FNwiVjxNY/LeWq06RKD+dDCPxTJl3JCNe3l0cYEPAglV2uMMYukDiTqJ7e+HI4nh4N4mv6lwx39N8dAvJe1x5ITS2T4qk4kTjuQb1Q1vw/ZOxoQqmvNKj2uRmBdJ/HHmysbRJ1OzCWML3OXdUwJf0AYlJzTjpMfkOKr7sTtE4rwyyQtd4tKH1fGdurgI9ZuFd9qvYxK2qcJhsQ6CNqMXt+7FkVkN1rIPmofjjBTNryzUr4COFXuWH95aDAif19DeBW4lbNgo1+FpDsrgmqtuhl6NAuptI8q/imow2KXBYJ8JPXsxW8DVFj0IIp0RCd3GjaEnwBEbxAyiIHLfW7AudyTS/dJOvZffPqXnuJ8xj3OPIdNe4xY0hWl8Ju2HhKfLOAHq7VadHZWd3IHLil70EiL4/JLD1rNbMImUZisFaA8pyrcIvYYebjOnk4TscwKFLedClRSX1XsMjWWd0oykQtrdkHM2IxknnBpaLu7mFnfE07f6dkG0nlpyu4SCLey7hr5FdcEmljA0nIxTSYDg6035fQkBEAbe7hlESOekkVNT9IZPwG+lmt3vU4ofi6NqNbJecOuSB+h36IiZ9s4YQtxYNnLgW14zjuFGGyT5smc3IjBT7qngDjKIgyrSVoRkY/8udy9qbUgvBeW8= - - -jobs: - allow_failures: - - go: tip - exclude: - # Exclude builds for 386 architecture on go 1.12 and tip - # Since we don't want it to run for 32 bit - - go: "1.12" - env: GOARCH=386 - - go: tip - env: GOARCH=386 - include: - # Define one extra linux build, which we use to run cross - # compiled 32 bit tests - - os: linux - arch: arm64 - go: "1.14" - env: go_32=yes - -notifications: - email: false - slack: - secure: X7uBLWYbuUhf8QFE16CoS5z7WvFR8EN9j6cEectMW6mKZ3vwXGwVXRIPsgUq/606DsQdCCx34MR8MRWYGlu6TBolbSe9y0EP0i46yipPz22YtuT7umcVUbGEyx8MZKgG0v1u/zA0O4aCsOBpGAA3gxz8h3JlEHDt+hv6U8xRsSllVLzLSNb5lwxDtcfEDxVVqP47GMEgjLPM28Pyt5qwjk7o5a4YSVzkfdxBXxd3gWzFUWzJ5E3cTacli50dK4GVfiLcQY2aQYoYO7AAvDnvP+TPfjDkBlUEE4MUz5CDIN51Xb+WW33sX7g+r3Bj7V5IRcF973RiYkpEh+3eoiPnyWyxhDZBYilty3b+Hysp6d4Ov/3I3ll7Bcny5+cYjakjkMH3l9w3gs6Y82GlpSLSJshKWS8vPRsxFe0Pstj6QSJXTd9EBaFr+l1ScXjJv/Sya9j8N9FfTuOTESWuaL1auX4Y7zEEVHlA8SCNOO8K0eTfxGZnC/YcIHsR8rePEAcFxfOYQppkyLF/XvAtnb/LMUuu0g4y2qNdme6Oelvyar1tFEMRtbl4mRCdu/krXBFtkrsfUaVY6WTPdvXAGotsFJ0wuA53zGVhlcd3+xAlSlR3c1QX95HIMeivJKb5L4nTjP+xnrmQNtnVk+tG4LSH2ltuwcZSSczModtcBmRefrk= - -script: >- - if [ $TRAVIS_OS_NAME = "linux" ] && [ $go_32 ]; then - uname -a - GOOS=linux GOARCH=arm go test -v ./... - else - go test -v ./... - # Cross-compile for Plan 9 - GOOS=plan9 go build ./... - fi diff --git a/vendor/github.com/dgraph-io/badger/v3/CHANGELOG.md b/vendor/github.com/dgraph-io/badger/v3/CHANGELOG.md index 9dc49bac2..28648bbea 100644 --- a/vendor/github.com/dgraph-io/badger/v3/CHANGELOG.md +++ b/vendor/github.com/dgraph-io/badger/v3/CHANGELOG.md @@ -3,6 +3,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [3.2103.3] - 2022-10-14 + +### Remarks + + - This is a minor patch release that fixes arm64 related issues. The issues in the `z` package in Ristretto were resolved in Ristretto v0.1.1. + +### Fixed + + - fix(arm64): bump ristretto v0.1.0 --> v0.1.1 (#1806) + +## [3.2103.2] - 2021-10-07 + +### Fixed + + - fix(compact): close vlog after the compaction at L0 has completed (#1752) + - fix(builder): put the upper limit on reallocation (#1748) + - deps: Bump github.com/google/flatbuffers to v1.12.1 (#1746) + - fix(levels): Avoid a deadlock when acquiring read locks in levels (#1744) + - fix(pubsub): avoid deadlock in publisher and subscriber (#1749) (#1751) + ## [3.2103.1] - 2021-07-08 ### Fixed diff --git a/vendor/github.com/dgraph-io/badger/v3/Makefile b/vendor/github.com/dgraph-io/badger/v3/Makefile new file mode 100644 index 000000000..b3e2ac122 --- /dev/null +++ b/vendor/github.com/dgraph-io/badger/v3/Makefile @@ -0,0 +1,60 @@ +# +# Copyright 2022 Dgraph Labs, Inc. and Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +USER_ID = $(shell id -u) +HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc") +JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" + + +.PHONY: all badger test jemalloc dependency + +badger: jemalloc + @echo "Compiling Badger binary..." + @$(MAKE) -C badger badger + @echo "Badger binary located in badger directory." + +test: jemalloc + @echo "Running Badger tests..." + @./test.sh + +jemalloc: + @if [ -z "$(HAS_JEMALLOC)" ] ; then \ + mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; \ + echo "Downloading jemalloc..." ; \ + curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ; \ + tar xjf ./jemalloc.tar.bz2 ; \ + cd jemalloc-5.2.1 ; \ + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'; \ + make ; \ + if [ "$(USER_ID)" -eq "0" ]; then \ + make install ; \ + else \ + echo "==== Need sudo access to install jemalloc" ; \ + sudo make install ; \ + fi \ + fi + +dependency: + @echo "Installing dependencies..." + @sudo apt-get update + @sudo apt-get -y upgrade + @sudo apt-get -y install \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + build-essential \ + protobuf-compiler \ diff --git a/vendor/github.com/dgraph-io/badger/v3/README.md b/vendor/github.com/dgraph-io/badger/v3/README.md index aef00efc0..e6a7cf4cb 100644 --- a/vendor/github.com/dgraph-io/badger/v3/README.md +++ b/vendor/github.com/dgraph-io/badger/v3/README.md @@ -1,4 +1,12 @@ -# BadgerDB [![Go Reference](https://pkg.go.dev/badge/github.com/dgraph-io/badger/v3.svg)](https://pkg.go.dev/github.com/dgraph-io/badger/v3) [![Go Report Card](https://goreportcard.com/badge/github.com/dgraph-io/badger)](https://goreportcard.com/report/github.com/dgraph-io/badger) [![Sourcegraph](https://sourcegraph.com/github.com/dgraph-io/badger/-/badge.svg)](https://sourcegraph.com/github.com/dgraph-io/badger?badge) [![Build Status](https://teamcity.dgraph.io/guestAuth/app/rest/builds/buildType:(id:Badger_UnitTests)/statusIcon.svg)](https://teamcity.dgraph.io/viewLog.html?buildTypeId=Badger_UnitTests&buildId=lastFinished&guest=1) ![Appveyor](https://ci.appveyor.com/api/projects/status/github/dgraph-io/badger?branch=master&svg=true) [![Coverage Status](https://coveralls.io/repos/github/dgraph-io/badger/badge.svg?branch=master)](https://coveralls.io/github/dgraph-io/badger?branch=master) +# BadgerDB + +[![Go Reference](https://pkg.go.dev/badge/github.com/dgraph-io/badger/v3.svg)](https://pkg.go.dev/github.com/dgraph-io/badger/v3) +[![Go Report Card](https://goreportcard.com/badge/github.com/dgraph-io/badger/v3)](https://goreportcard.com/report/github.com/dgraph-io/badger/v3) +[![Sourcegraph](https://sourcegraph.com/github.com/dgraph-io/badger/-/badge.svg)](https://sourcegraph.com/github.com/dgraph-io/badger?badge) +[![ci-badger-tests](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-tests.yml/badge.svg)](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-tests.yml) +[![ci-badger-bank-tests](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-bank-tests.yml/badge.svg)](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-bank-tests.yml) +[![ci-golang-lint](https://github.com/dgraph-io/badger/actions/workflows/ci-golang-lint.yml/badge.svg)](https://github.com/dgraph-io/badger/actions/workflows/ci-golang-lint.yml) + ![Badger mascot](images/diggy-shadow.png) @@ -7,9 +15,7 @@ in pure Go. It is the underlying database for [Dgraph](https://dgraph.io), a fast, distributed graph database. It's meant to be a performant alternative to non-Go-based key-value stores like RocksDB. -**Use [Discuss Issues](https://discuss.dgraph.io/c/issues/badger/37) for reporting issues about this repository.** - -## Project Status [March 24, 2020] +## Project Status Badger is stable and is being used to serve data sets worth hundreds of terabytes. Badger supports concurrent ACID transactions with serializable @@ -27,7 +33,9 @@ with v1.0 is v1.6.0. Badger v2.0 was released in Nov 2019 with a new storage format which won't be compatible with all of the v1.x. Badger v2.0 supports compression, encryption and uses a cache to speed up lookup. -The [Changelog] is kept fairly up-to-date. +Badger v3.0 was released in January 2021. This release improves compaction performance. + +Please consult the [Changelog] for more detailed information on releases. For more details on our version naming schema please read [Choosing a version](#choosing-a-version). @@ -51,7 +59,7 @@ For more details on our version naming schema please read [Choosing a version](# ## Getting Started ### Installing -To start using Badger, install Go 1.12 or above. Badger v2 needs go modules. Run the following command to retrieve the library. +To start using Badger, install Go 1.12 or above. Badger v3 needs go modules. From your project, run the following command ```sh $ go get github.com/dgraph-io/badger/v3 @@ -60,13 +68,14 @@ This will retrieve the library. #### Installing Badger Command Line Tool -Download and extract the latest Badger DB release from https://github.com/dgraph-io/badger/releases and then run the following commands. +Badger provides a CLI tool which can perform certain operations like offline backup/restore. To install the Badger CLI, +retrieve the repository and checkout the desired version. Then run ```sh -$ cd badger-/badger -$ go install +$ cd badger +$ go install . ``` -This will install the badger command line utility into your $GOBIN path. +This will install the badger command line utility into your $GOBIN path. #### Choosing a version @@ -86,9 +95,12 @@ Following these rules: version is the same, therefore the data format on disk is compatible. - v1.6.0 and v2.0.0 are data incompatible as their major version implies, so files created with v1.6.0 will need to be converted into the new format before they can be used by v2.0.0. + - v2.x.x and v3.x.x are data incompatible as their major version implies, so files created with + v2.x.x will need to be converted into the new format before they can be used by v3.0.0. + For a longer explanation on the reasons behind using a new versioning naming schema, you can read -[VERSIONING.md](VERSIONING.md). +[VERSIONING](VERSIONING.md). ## Badger Documentation @@ -99,9 +111,9 @@ Badger Documentation is available at https://dgraph.io/docs/badger ### Blog Posts 1. [Introducing Badger: A fast key-value store written natively in Go](https://open.dgraph.io/post/badger/) -2. [Make Badger crash resilient with ALICE](https://blog.dgraph.io/post/alice/) -3. [Badger vs LMDB vs BoltDB: Benchmarking key-value databases in Go](https://blog.dgraph.io/post/badger-lmdb-boltdb/) -4. [Concurrent ACID Transactions in Badger](https://blog.dgraph.io/post/badger-txn/) +2. [Make Badger crash resilient with ALICE](https://open.dgraph.io/post/alice/) +3. [Badger vs LMDB vs BoltDB: Benchmarking key-value databases in Go](https://open.dgraph.io/post/badger-lmdb-boltdb/) +4. [Concurrent ACID Transactions in Badger](https://open.dgraph.io/post/badger-txn/) ## Design Badger was written with these design goals in mind: @@ -167,6 +179,7 @@ Below is a list of known projects that use Badger: * [Immudb](https://github.com/codenotary/immudb) - Lightweight, high-speed immutable database for systems and applications. * [Usenet Express](https://usenetexpress.com/) - Serving over 300TB of data with Badger. * [gorush](https://github.com/appleboy/gorush) - A push notification server written in Go. +* [0-stor](https://github.com/zero-os/0-stor) - Single device object store. * [Dispatch Protocol](https://github.com/dispatchlabs/disgo) - Blockchain protocol for distributed application data analytics. * [GarageMQ](https://github.com/valinurovam/garagemq) - AMQP server written in Go. * [RedixDB](https://alash3al.github.io/redix/) - A real-time persistent key-value store with the same redis protocol. @@ -189,15 +202,23 @@ Below is a list of known projects that use Badger: * [Terminotes](https://gitlab.com/asad-awadia/terminotes) - Self hosted notes storage and search server - storage powered by BadgerDB * [Pyroscope](https://github.com/pyroscope-io/pyroscope) - Open source confinuous profiling platform built with BadgerDB * [Veri](https://github.com/bgokden/veri) - A distributed feature store optimized for Search and Recommendation tasks. +* [bIter](https://github.com/MikkelHJuul/bIter) - A library and Iterator interface for working with the `badger.Iterator`, simplifying from-to, and prefix mechanics. +* [ld](https://github.com/MikkelHJuul/ld) - (Lean Database) A very simple gRPC-only key-value database, exposing BadgerDB with key-range scanning semantics. +* [Souin](https://github.com/darkweak/Souin) - A RFC compliant HTTP cache with lot of other features based on Badger for the storage. Compatible with all existing reverse-proxies. +* [Xuperchain](https://github.com/xuperchain/xupercore) - A highly flexible blockchain architecture with great transaction performance. +* [m2](https://github.com/qichengzx/m2) - A simple http key/value store based on the raft protocol. +* [chaindb](https://github.com/ChainSafe/chaindb) - A blockchain storage layer used by [Gossamer](https://chainsafe.github.io/gossamer/), a Go client for the [Polkadot Network](https://polkadot.network/). +* [vxdb](https://github.com/vitalvas/vxdb) - Simple schema-less Key-Value NoSQL database with simplest API interface. +* [Opacity](https://github.com/opacity/storage-node) - Backend implementation for the Opacity storage project +* [Vephar](https://github.com/vaccovecrana/vephar) - A minimal key/value store using hashicorp-raft for cluster coordination and Badger for data storage. If you are using Badger in a project please send a pull request to add it to the list. ## Contributing -If you're interested in contributing to Badger see [CONTRIBUTING.md](./CONTRIBUTING.md). +If you're interested in contributing to Badger see [CONTRIBUTING](./CONTRIBUTING.md). ## Contact - Please use [discuss.dgraph.io](https://discuss.dgraph.io) for questions, feature requests and discussions. -- Please use [Github issue tracker](https://github.com/dgraph-io/badger/issues) for filing bugs or feature requests. -- Follow us on Twitter [@dgraphlabs](https://twitter.com/dgraphlabs). - +- Please use [discuss.dgraph.io](https://discuss.dgraph.io) for filing bugs or feature requests. +- Follow us on Twitter [@dgraphlabs](https://twitter.com/dgraphlabs). \ No newline at end of file diff --git a/vendor/github.com/dgraph-io/badger/v3/db.go b/vendor/github.com/dgraph-io/badger/v3/db.go index 7bedd1dec..3ac6a2d95 100644 --- a/vendor/github.com/dgraph-io/badger/v3/db.go +++ b/vendor/github.com/dgraph-io/badger/v3/db.go @@ -559,11 +559,6 @@ func (db *DB) close() (err error) { db.closers.pub.SignalAndWait() db.closers.cacheHealth.Signal() - // Now close the value log. - if vlogErr := db.vlog.Close(); vlogErr != nil { - err = y.Wrap(vlogErr, "DB.Close") - } - // Make sure that block writer is done pushing stuff into memtable! // Otherwise, you will have a race condition: we are trying to flush memtables // and remove them completely, while the block / memtable writer is still @@ -619,6 +614,11 @@ func (db *DB) close() (err error) { } } + // Now close the value log. + if vlogErr := db.vlog.Close(); vlogErr != nil { + err = y.Wrap(vlogErr, "DB.Close") + } + db.opt.Infof(db.LevelsToString()) if lcErr := db.lc.close(); err == nil { err = y.Wrap(lcErr, "DB.Close") @@ -1869,11 +1869,11 @@ func (db *DB) Subscribe(ctx context.Context, cb func(kv *KVList) error, matches } c := z.NewCloser(1) - recvCh, id := db.pub.newSubscriber(c, matches) + s := db.pub.newSubscriber(c, matches) slurp := func(batch *pb.KVList) error { for { select { - case kvs := <-recvCh: + case kvs := <-s.sendCh: batch.Kv = append(batch.Kv, kvs.Kv...) default: if len(batch.GetKv()) > 0 { @@ -1883,6 +1883,16 @@ func (db *DB) Subscribe(ctx context.Context, cb func(kv *KVList) error, matches } } } + + drain := func() { + for { + select { + case <-s.sendCh: + default: + return + } + } + } for { select { case <-c.HasBeenClosed(): @@ -1894,15 +1904,19 @@ func (db *DB) Subscribe(ctx context.Context, cb func(kv *KVList) error, matches return err case <-ctx.Done(): c.Done() - db.pub.deleteSubscriber(id) + atomic.StoreUint64(s.active, 0) + drain() + db.pub.deleteSubscriber(s.id) // Delete the subscriber to avoid further updates. return ctx.Err() - case batch := <-recvCh: + case batch := <-s.sendCh: err := slurp(batch) if err != nil { c.Done() + atomic.StoreUint64(s.active, 0) + drain() // Delete the subscriber if there is an error by the callback. - db.pub.deleteSubscriber(id) + db.pub.deleteSubscriber(s.id) return err } } diff --git a/vendor/github.com/dgraph-io/badger/v3/levels.go b/vendor/github.com/dgraph-io/badger/v3/levels.go index 637a3565d..1c625d1b3 100644 --- a/vendor/github.com/dgraph-io/badger/v3/levels.go +++ b/vendor/github.com/dgraph-io/badger/v3/levels.go @@ -1119,8 +1119,22 @@ func (s *levelsController) fillTablesL0ToL0(cd *compactDef) bool { cd.nextRange = keyRange{} cd.bot = nil - cd.lockLevels() - defer cd.unlockLevels() + // Because this level and next level are both level 0, we should NOT acquire + // the read lock twice, because it can result in a deadlock. So, we don't + // call compactDef.lockLevels, instead locking the level only once and + // directly here. + // + // As per godocs on RWMutex: + // If a goroutine holds a RWMutex for reading and another goroutine might + // call Lock, no goroutine should expect to be able to acquire a read lock + // until the initial read lock is released. In particular, this prohibits + // recursive read locking. This is to ensure that the lock eventually + // becomes available; a blocked Lock call excludes new readers from + // acquiring the lock. + y.AssertTrue(cd.thisLevel.level == 0) + y.AssertTrue(cd.nextLevel.level == 0) + s.levels[0].RLock() + defer s.levels[0].RUnlock() s.cstatus.Lock() defer s.cstatus.Unlock() diff --git a/vendor/github.com/dgraph-io/badger/v3/manifest.go b/vendor/github.com/dgraph-io/badger/v3/manifest.go index bb27fb247..7face8b47 100644 --- a/vendor/github.com/dgraph-io/badger/v3/manifest.go +++ b/vendor/github.com/dgraph-io/badger/v3/manifest.go @@ -208,15 +208,14 @@ func (mf *manifestFile) addChanges(changesParam []*pb.ManifestChange) error { // Maybe we could use O_APPEND instead (on certain file systems) mf.appendLock.Lock() + defer mf.appendLock.Unlock() if err := applyChangeSet(&mf.manifest, &changes); err != nil { - mf.appendLock.Unlock() return err } // Rewrite manifest if it'd shrink by 1/10 and it's big enough to care if mf.manifest.Deletions > mf.deletionsRewriteThreshold && mf.manifest.Deletions > manifestDeletionsRatio*(mf.manifest.Creations-mf.manifest.Deletions) { if err := mf.rewrite(); err != nil { - mf.appendLock.Unlock() return err } } else { @@ -225,15 +224,16 @@ func (mf *manifestFile) addChanges(changesParam []*pb.ManifestChange) error { binary.BigEndian.PutUint32(lenCrcBuf[4:8], crc32.Checksum(buf, y.CastagnoliCrcTable)) buf = append(lenCrcBuf[:], buf...) if _, err := mf.fp.Write(buf); err != nil { - mf.appendLock.Unlock() return err } } - mf.appendLock.Unlock() - return mf.fp.Sync() + return syncFunc(mf.fp) } +// this function is saved here to allow injection of fake filesystem latency at test time. +var syncFunc = func(f *os.File) error { return f.Sync() } + // Has to be 4 bytes. The value can never change, ever, anyway. var magicText = [4]byte{'B', 'd', 'g', 'r'} diff --git a/vendor/github.com/dgraph-io/badger/v3/publisher.go b/vendor/github.com/dgraph-io/badger/v3/publisher.go index 6694433f2..f4c31b2ea 100644 --- a/vendor/github.com/dgraph-io/badger/v3/publisher.go +++ b/vendor/github.com/dgraph-io/badger/v3/publisher.go @@ -18,6 +18,7 @@ package badger import ( "sync" + "sync/atomic" "github.com/dgraph-io/badger/v3/pb" "github.com/dgraph-io/badger/v3/trie" @@ -26,9 +27,13 @@ import ( ) type subscriber struct { + id uint64 matches []pb.Match - sendCh chan<- *pb.KVList + sendCh chan *pb.KVList subCloser *z.Closer + // this will be atomic pointer which will be used to + // track whether the subscriber is active or not + active *uint64 } type publisher struct { @@ -106,26 +111,32 @@ func (p *publisher) publishUpdates(reqs requests) { } for id, kvs := range batchedUpdates { - p.subscribers[id].sendCh <- kvs + if atomic.LoadUint64(p.subscribers[id].active) == 1 { + p.subscribers[id].sendCh <- kvs + } } } -func (p *publisher) newSubscriber(c *z.Closer, matches []pb.Match) (<-chan *pb.KVList, uint64) { +func (p *publisher) newSubscriber(c *z.Closer, matches []pb.Match) subscriber { p.Lock() defer p.Unlock() ch := make(chan *pb.KVList, 1000) id := p.nextID // Increment next ID. p.nextID++ - p.subscribers[id] = subscriber{ + active := uint64(1) + s := subscriber{ + active: &active, + id: id, matches: matches, sendCh: ch, subCloser: c, } + p.subscribers[id] = s for _, m := range matches { p.indexer.AddMatch(m, id) } - return ch, id + return s } // cleanSubscribers stops all the subscribers. Ideally, It should be called while closing DB. diff --git a/vendor/github.com/dgraph-io/badger/v3/table/builder.go b/vendor/github.com/dgraph-io/badger/v3/table/builder.go index 8322bb86f..5bab48a72 100644 --- a/vendor/github.com/dgraph-io/badger/v3/table/builder.go +++ b/vendor/github.com/dgraph-io/badger/v3/table/builder.go @@ -100,8 +100,12 @@ type Builder struct { func (b *Builder) allocate(need int) []byte { bb := b.curBlock if len(bb.data[bb.end:]) < need { - // We need to reallocate. + // We need to reallocate. 1GB is the max size that the allocator can allocate. + // While reallocating, if doubling exceeds that limit, then put the upper bound on it. sz := 2 * len(bb.data) + if sz > (1 << 30) { + sz = 1 << 30 + } if bb.end+need > sz { sz = bb.end + need } diff --git a/vendor/github.com/dgraph-io/badger/v3/test.sh b/vendor/github.com/dgraph-io/badger/v3/test.sh index 6922eed0b..15122767e 100644 --- a/vendor/github.com/dgraph-io/badger/v3/test.sh +++ b/vendor/github.com/dgraph-io/badger/v3/test.sh @@ -4,48 +4,27 @@ set -eo pipefail go version +# Check if Github Actions is running +if [ $CI = "true" ]; then + # Enable code coverage + # export because tests run in a subprocess + export covermode="-covermode=atomic" + export coverprofile="-coverprofile=cover_tmp.out" + echo "mode: atomic" >> cover.out +fi + # Run `go list` BEFORE setting GOFLAGS so that the output is in the right # format for grep. # export packages because the test will run in a sub process. export packages=$(go list ./... | grep "github.com/dgraph-io/badger/v3/") -if [[ ! -z "$TEAMCITY_VERSION" ]]; then - export GOFLAGS="-json" -fi - -function InstallJemalloc() { - pushd . - if [ ! -f /usr/local/lib/libjemalloc.a ]; then - USER_ID=`id -u` - JEMALLOC_URL="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" - - mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; - echo "Downloading jemalloc" ; - curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ; - tar xjf ./jemalloc.tar.bz2 ; - cd jemalloc-5.2.1 ; - ./configure --with-jemalloc-prefix='je_' ; - make ; - if [ "$USER_ID" -eq "0" ]; then - make install ; - else - echo "==== Need sudo access to install jemalloc" ; - sudo make install ; - fi - fi - popd -} - tags="-tags=jemalloc" -# Ensure that we can compile the binary. +# Compile the Badger binary pushd badger go build -v $tags . popd -# tags="" -InstallJemalloc - # Run the memory intensive tests first. manual() { timeout="-timeout 2m" @@ -53,7 +32,7 @@ manual() { set -e for pkg in $packages; do echo "===> Testing $pkg" - go test $tags -timeout=25m -race $pkg -parallel 16 + go test $tags -timeout=25m $covermode $coverprofile -race -parallel 16 $pkg && write_coverage done echo "==> DONE package tests" @@ -61,10 +40,10 @@ manual() { # Run the special Truncate test. rm -rf p set -e - go test $tags $timeout -run='TestTruncateVlogNoClose$' --manual=true + go test $tags $timeout $covermode $coverprofile -run='TestTruncateVlogNoClose$' --manual=true && write_coverage truncate --size=4096 p/000000.vlog - go test $tags $timeout -run='TestTruncateVlogNoClose2$' --manual=true - go test $tags $timeout -run='TestTruncateVlogNoClose3$' --manual=true + go test $tags $timeout $covermode $coverprofile -run='TestTruncateVlogNoClose2$' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestTruncateVlogNoClose3$' --manual=true && write_coverage rm -rf p # TODO(ibrahim): Let's make these tests have Manual prefix. @@ -73,14 +52,14 @@ manual() { # TestValueGCManaged # TestDropPrefix # TestDropAllManaged - go test $tags $timeout -run='TestBigKeyValuePairs$' --manual=true - go test $tags $timeout -run='TestPushValueLogLimit' --manual=true - go test $tags $timeout -run='TestKeyCount' --manual=true - go test $tags $timeout -run='TestIteratePrefix' --manual=true - go test $tags $timeout -run='TestIterateParallel' --manual=true - go test $tags $timeout -run='TestBigStream' --manual=true - go test $tags $timeout -run='TestGoroutineLeak' --manual=true - go test $tags $timeout -run='TestGetMore' --manual=true + go test $tags $timeout $covermode $coverprofile -run='TestBigKeyValuePairs$' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestPushValueLogLimit' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestKeyCount' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestIteratePrefix' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestIterateParallel' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestBigStream' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestGoroutineLeak' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestGetMore' --manual=true && write_coverage echo "==> DONE manual tests" } @@ -91,7 +70,7 @@ root() { echo "==> Running root level tests." set -e - go test $tags -timeout=25m . -v -race -parallel 16 + go test $tags -v -race -parallel=16 -timeout=25m $covermode $coverprofile . && write_coverage echo "==> DONE root level tests" } @@ -110,11 +89,23 @@ stream() { return 1 fi echo "==> DONE stream test" + popd return 0 } -export -f stream -export -f manual -export -f root +write_coverage() { + if [ $CI = "true" ]; then + if [ -f cover_tmp.out ]; then + sed -i '1d' cover_tmp.out + cat cover_tmp.out >> cover.out && rm cover_tmp.out + fi + fi + +} -parallel --halt now,fail=1 --progress --line-buffer ::: stream manual root +# parallel tests currently not working +# parallel --halt now,fail=1 --progress --line-buffer ::: stream manual root +# run tests in sequence +root +stream +manual diff --git a/vendor/github.com/dgraph-io/ristretto/.go-version b/vendor/github.com/dgraph-io/ristretto/.go-version new file mode 100644 index 000000000..b8f1e3fd3 --- /dev/null +++ b/vendor/github.com/dgraph-io/ristretto/.go-version @@ -0,0 +1 @@ +1.17.11 diff --git a/vendor/github.com/dgraph-io/ristretto/.golangci.yml b/vendor/github.com/dgraph-io/ristretto/.golangci.yml new file mode 100644 index 000000000..7318e9a3b --- /dev/null +++ b/vendor/github.com/dgraph-io/ristretto/.golangci.yml @@ -0,0 +1,23 @@ +run: + tests: false + skip-dirs: + - contrib + - sim + +linters-settings: + lll: + line-length: 120 + +linters: + disable-all: true + enable: + #- errcheck + #- ineffassign + - gas + #- gofmt + #- golint + #- gosimple + #- govet + - lll + #- varcheck + #- unused diff --git a/vendor/github.com/dgraph-io/ristretto/CHANGELOG.md b/vendor/github.com/dgraph-io/ristretto/CHANGELOG.md index da964bc0c..3d18e39ed 100644 --- a/vendor/github.com/dgraph-io/ristretto/CHANGELOG.md +++ b/vendor/github.com/dgraph-io/ristretto/CHANGELOG.md @@ -4,11 +4,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project will adhere to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) starting v1.0.0. -## Unreleased +## [0.1.1] - 2022-10-12 + +[0.1.1]: https://github.com/dgraph-io/ristretto/compare/v0.1.0..v0.1.1 +This release fixes certain arm64 build issues in the z package. It also +incorporates CI steps in our repository. + +### Changed +- [chore(docs): Include SpiceDB in the list of projects using Ristretto (#285)](https://github.com/dgraph-io/ristretto/pull/311) + +### Added +- [Run CI Jobs via Github Actions #304](https://github.com/dgraph-io/ristretto/pull/304) + +### Fixed +- [fix(build): update x/sys dependency](https://github.com/dgraph-io/ristretto/pull/308) +- [fix(z): Address inconsistent mremap return arguments with arm64](https://github.com/dgraph-io/ristretto/pull/309) +- [fix(z): runtime error: index out of range for !amd64 env #287](https://github.com/dgraph-io/ristretto/pull/307) ## [0.1.0] - 2021-06-03 -[0.1.0]: https://github.com/dgraph-io/ristretto/compare/v0.1.0..v0.0.3 +[0.1.0]: https://github.com/dgraph-io/ristretto/compare/v0.0.3..v0.1.0 This release contains bug fixes and improvements to Ristretto. It also contains major updates to the z package. The z package contains types such as Tree (B+ tree), Buffer, Mmap file, etc. All these types are used in Badger and Dgraph to diff --git a/vendor/github.com/dgraph-io/ristretto/README.md b/vendor/github.com/dgraph-io/ristretto/README.md index f4bb28cd7..e71ae3df9 100644 --- a/vendor/github.com/dgraph-io/ristretto/README.md +++ b/vendor/github.com/dgraph-io/ristretto/README.md @@ -1,16 +1,15 @@ # Ristretto [![Go Doc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/dgraph-io/ristretto) +[![ci-ristretto-tests](https://github.com/dgraph-io/ristretto/actions/workflows/ci-ristretto-tests.yml/badge.svg)](https://github.com/dgraph-io/ristretto/actions/workflows/ci-ristretto-tests.yml) +[![ci-ristretto-lint](https://github.com/dgraph-io/ristretto/actions/workflows/ci-ristretto-lint.yml/badge.svg)](https://github.com/dgraph-io/ristretto/actions/workflows/ci-ristretto-lint.yml) +[![Coverage Status](https://coveralls.io/repos/github/dgraph-io/ristretto/badge.svg?branch=main)](https://coveralls.io/github/dgraph-io/ristretto?branch=main) [![Go Report Card](https://img.shields.io/badge/go%20report-A%2B-brightgreen)](https://goreportcard.com/report/github.com/dgraph-io/ristretto) -[![Coverage](https://gocover.io/_badge/github.com/dgraph-io/ristretto)](https://gocover.io/github.com/dgraph-io/ristretto) -![Tests](https://github.com/dgraph-io/ristretto/workflows/tests/badge.svg) Ristretto is a fast, concurrent cache library built with a focus on performance and correctness. The motivation to build Ristretto comes from the need for a contention-free cache in [Dgraph][]. -**Use [Discuss Issues](https://discuss.dgraph.io/tags/c/issues/35/ristretto/40) for reporting issues about this repository.** - [Dgraph]: https://github.com/dgraph-io/dgraph ## Features @@ -205,7 +204,8 @@ Below is a list of known projects that use Ristretto: - [Badger](https://github.com/dgraph-io/badger) - Embeddable key-value DB in Go - [Dgraph](https://github.com/dgraph-io/dgraph) - Horizontally scalable and distributed GraphQL database with a graph backend -- [Vitess](https://github.com/vitessio/vitess) - database clustering system for horizontal scaling of MySQL +- [Vitess](https://github.com/vitessio/vitess) - Database clustering system for horizontal scaling of MySQL +- [SpiceDB](https://github.com/authzed/spicedb) - Horizontally scalable permissions database ## FAQ diff --git a/vendor/github.com/dgraph-io/ristretto/sketch.go b/vendor/github.com/dgraph-io/ristretto/sketch.go index 10f414689..6368d2bde 100644 --- a/vendor/github.com/dgraph-io/ristretto/sketch.go +++ b/vendor/github.com/dgraph-io/ristretto/sketch.go @@ -53,7 +53,8 @@ func newCmSketch(numCounters int64) *cmSketch { numCounters = next2Power(numCounters) sketch := &cmSketch{mask: uint64(numCounters - 1)} // Initialize rows of counters and seeds. - source := rand.New(rand.NewSource(time.Now().UnixNano())) + // Cryptographic precision not needed + source := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec for i := 0; i < cmDepth; i++ { sketch.seed[i] = source.Uint64() sketch.rows[i] = newCmRow(numCounters) diff --git a/vendor/github.com/dgraph-io/ristretto/test.sh b/vendor/github.com/dgraph-io/ristretto/test.sh deleted file mode 100644 index d53b32d42..000000000 --- a/vendor/github.com/dgraph-io/ristretto/test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/sh - -starttest() { - set -e - GO111MODULE=on go test -race ./... -} - -if [ -z "${TEAMCITY_VERSION}" ]; then - # running locally, so start test in a container - # TEAMCITY_VERSION=local will avoid recursive calls, when it would be running in container - docker run --rm --name ristretto-test -ti \ - -v `pwd`:/go/src/github.com/dgraph-io/ristretto \ - --workdir /go/src/github.com/dgraph-io/ristretto \ - --env TEAMCITY_VERSION=local \ - golang:1.13 \ - sh test.sh -else - # running in teamcity, since teamcity itself run this in container, let's simply run this - starttest -fi diff --git a/vendor/github.com/dgraph-io/ristretto/z/allocator.go b/vendor/github.com/dgraph-io/ristretto/z/allocator.go index db00ff5ec..eae0f8344 100644 --- a/vendor/github.com/dgraph-io/ristretto/z/allocator.go +++ b/vendor/github.com/dgraph-io/ristretto/z/allocator.go @@ -57,7 +57,7 @@ func init() { // Set up a unique Ref per process. rand.Seed(time.Now().UnixNano()) - allocRef = uint64(rand.Int63n(1<<16)) << 48 + allocRef = uint64(rand.Int63n(1<<16)) << 48 //nolint:gosec // cryptographic precision not needed calculatedLog2 = make([]int, 1025) for i := 1; i <= 1024; i++ { diff --git a/vendor/github.com/dgraph-io/ristretto/z/mmap_linux.go b/vendor/github.com/dgraph-io/ristretto/z/mmap_linux.go index 9cc3497a1..331330cff 100644 --- a/vendor/github.com/dgraph-io/ristretto/z/mmap_linux.go +++ b/vendor/github.com/dgraph-io/ristretto/z/mmap_linux.go @@ -17,9 +17,7 @@ package z import ( - "fmt" "os" - "reflect" "unsafe" "golang.org/x/sys/unix" @@ -35,34 +33,6 @@ func mmap(fd *os.File, writable bool, size int64) ([]byte, error) { return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED) } -// mremap is a Linux-specific system call to remap pages in memory. This can be used in place of munmap + mmap. -func mremap(data []byte, size int) ([]byte, error) { - // taken from - const MREMAP_MAYMOVE = 0x1 - - header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) - mmapAddr, mmapSize, errno := unix.Syscall6( - unix.SYS_MREMAP, - header.Data, - uintptr(header.Len), - uintptr(size), - uintptr(MREMAP_MAYMOVE), - 0, - 0, - ) - if errno != 0 { - return nil, errno - } - if mmapSize != uintptr(size) { - return nil, fmt.Errorf("mremap size mismatch: requested: %d got: %d", size, mmapSize) - } - - header.Data = mmapAddr - header.Cap = size - header.Len = size - return data, nil -} - // munmap unmaps a previously mapped slice. // // unix.Munmap maintains an internal list of mmapped addresses, and only calls munmap diff --git a/vendor/github.com/dgraph-io/ristretto/z/mremap_linux.go b/vendor/github.com/dgraph-io/ristretto/z/mremap_linux.go new file mode 100644 index 000000000..225678658 --- /dev/null +++ b/vendor/github.com/dgraph-io/ristretto/z/mremap_linux.go @@ -0,0 +1,56 @@ +// +build !arm64 + +/* + * Copyright 2020 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package z + +import ( + "fmt" + "reflect" + "unsafe" + + "golang.org/x/sys/unix" +) + +// mremap is a Linux-specific system call to remap pages in memory. This can be used in place of munmap + mmap. +func mremap(data []byte, size int) ([]byte, error) { + //nolint:lll + // taken from + const MREMAP_MAYMOVE = 0x1 + + header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) + mmapAddr, mmapSize, errno := unix.Syscall6( + unix.SYS_MREMAP, + header.Data, + uintptr(header.Len), + uintptr(size), + uintptr(MREMAP_MAYMOVE), + 0, + 0, + ) + if errno != 0 { + return nil, errno + } + if mmapSize != uintptr(size) { + return nil, fmt.Errorf("mremap size mismatch: requested: %d got: %d", size, mmapSize) + } + + header.Data = mmapAddr + header.Cap = size + header.Len = size + return data, nil +} diff --git a/vendor/github.com/dgraph-io/ristretto/z/mremap_linux_arm64.go b/vendor/github.com/dgraph-io/ristretto/z/mremap_linux_arm64.go new file mode 100644 index 000000000..09683cdfe --- /dev/null +++ b/vendor/github.com/dgraph-io/ristretto/z/mremap_linux_arm64.go @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package z + +import ( + "reflect" + "unsafe" + + "golang.org/x/sys/unix" +) + +// mremap is a Linux-specific system call to remap pages in memory. This can be used in place of munmap + mmap. +func mremap(data []byte, size int) ([]byte, error) { + //nolint:lll + // taken from + const MREMAP_MAYMOVE = 0x1 + + header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) + // For ARM64, the second return argument for SYS_MREMAP is inconsistent (prior allocated size) with + // other architectures, which return the size allocated + mmapAddr, _, errno := unix.Syscall6( + unix.SYS_MREMAP, + header.Data, + uintptr(header.Len), + uintptr(size), + uintptr(MREMAP_MAYMOVE), + 0, + 0, + ) + if errno != 0 { + return nil, errno + } + + header.Data = mmapAddr + header.Cap = size + header.Len = size + return data, nil +} diff --git a/vendor/github.com/dgraph-io/ristretto/z/simd/search.go b/vendor/github.com/dgraph-io/ristretto/z/simd/search.go index 0d001ee0c..b1e639225 100644 --- a/vendor/github.com/dgraph-io/ristretto/z/simd/search.go +++ b/vendor/github.com/dgraph-io/ristretto/z/simd/search.go @@ -20,7 +20,7 @@ package simd // Search uses the Clever search to find the correct key. func Search(xs []uint64, k uint64) int16 { - if len(xs) < 8 { + if len(xs) < 8 || (len(xs) % 8 != 0) { return Naive(xs, k) } var twos, pk [4]uint64 diff --git a/vendor/github.com/dustin/go-humanize/.travis.yml b/vendor/github.com/dustin/go-humanize/.travis.yml index ba95cdd15..ac12e485a 100644 --- a/vendor/github.com/dustin/go-humanize/.travis.yml +++ b/vendor/github.com/dustin/go-humanize/.travis.yml @@ -1,12 +1,12 @@ sudo: false language: go +go_import_path: github.com/dustin/go-humanize go: - - 1.3.x - - 1.5.x - - 1.6.x - - 1.7.x - - 1.8.x - - 1.9.x + - 1.13.x + - 1.14.x + - 1.15.x + - 1.16.x + - stable - master matrix: allow_failures: @@ -15,7 +15,7 @@ matrix: install: - # Do nothing. This is needed to prevent default install action "go get -t -v ./..." from happening here (we want it to happen inside script step). script: - - go get -t -v ./... - diff -u <(echo -n) <(gofmt -d -s .) - - go tool vet . + - go vet . + - go install -v -race ./... - go test -v -race ./... diff --git a/vendor/github.com/dustin/go-humanize/README.markdown b/vendor/github.com/dustin/go-humanize/README.markdown index 91b4ae564..7d0b16b34 100644 --- a/vendor/github.com/dustin/go-humanize/README.markdown +++ b/vendor/github.com/dustin/go-humanize/README.markdown @@ -5,7 +5,7 @@ Just a few functions for helping humanize times and sizes. `go get` it as `github.com/dustin/go-humanize`, import it as `"github.com/dustin/go-humanize"`, use it as `humanize`. -See [godoc](https://godoc.org/github.com/dustin/go-humanize) for +See [godoc](https://pkg.go.dev/github.com/dustin/go-humanize) for complete documentation. ## Sizes diff --git a/vendor/github.com/dustin/go-humanize/bigbytes.go b/vendor/github.com/dustin/go-humanize/bigbytes.go index 1a2bf6172..3b015fd59 100644 --- a/vendor/github.com/dustin/go-humanize/bigbytes.go +++ b/vendor/github.com/dustin/go-humanize/bigbytes.go @@ -28,6 +28,10 @@ var ( BigZiByte = (&big.Int{}).Mul(BigEiByte, bigIECExp) // BigYiByte is 1,024 z bytes in bit.Ints BigYiByte = (&big.Int{}).Mul(BigZiByte, bigIECExp) + // BigRiByte is 1,024 y bytes in bit.Ints + BigRiByte = (&big.Int{}).Mul(BigYiByte, bigIECExp) + // BigQiByte is 1,024 r bytes in bit.Ints + BigQiByte = (&big.Int{}).Mul(BigRiByte, bigIECExp) ) var ( @@ -51,6 +55,10 @@ var ( BigZByte = (&big.Int{}).Mul(BigEByte, bigSIExp) // BigYByte is 1,000 SI z bytes in big.Ints BigYByte = (&big.Int{}).Mul(BigZByte, bigSIExp) + // BigRByte is 1,000 SI y bytes in big.Ints + BigRByte = (&big.Int{}).Mul(BigYByte, bigSIExp) + // BigQByte is 1,000 SI r bytes in big.Ints + BigQByte = (&big.Int{}).Mul(BigRByte, bigSIExp) ) var bigBytesSizeTable = map[string]*big.Int{ @@ -71,6 +79,10 @@ var bigBytesSizeTable = map[string]*big.Int{ "zb": BigZByte, "yib": BigYiByte, "yb": BigYByte, + "rib": BigRiByte, + "rb": BigRByte, + "qib": BigQiByte, + "qb": BigQByte, // Without suffix "": BigByte, "ki": BigKiByte, @@ -89,6 +101,10 @@ var bigBytesSizeTable = map[string]*big.Int{ "zi": BigZiByte, "y": BigYByte, "yi": BigYiByte, + "r": BigRByte, + "ri": BigRiByte, + "q": BigQByte, + "qi": BigQiByte, } var ten = big.NewInt(10) @@ -115,7 +131,7 @@ func humanateBigBytes(s, base *big.Int, sizes []string) string { // // BigBytes(82854982) -> 83 MB func BigBytes(s *big.Int) string { - sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"} + sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB", "RB", "QB"} return humanateBigBytes(s, bigSIExp, sizes) } @@ -125,7 +141,7 @@ func BigBytes(s *big.Int) string { // // BigIBytes(82854982) -> 79 MiB func BigIBytes(s *big.Int) string { - sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"} + sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB", "RiB", "QiB"} return humanateBigBytes(s, bigIECExp, sizes) } diff --git a/vendor/github.com/dustin/go-humanize/commaf.go b/vendor/github.com/dustin/go-humanize/commaf.go index 620690dec..2bc83a03c 100644 --- a/vendor/github.com/dustin/go-humanize/commaf.go +++ b/vendor/github.com/dustin/go-humanize/commaf.go @@ -1,3 +1,4 @@ +//go:build go1.6 // +build go1.6 package humanize diff --git a/vendor/github.com/dustin/go-humanize/ftoa.go b/vendor/github.com/dustin/go-humanize/ftoa.go index 1c62b640d..bce923f37 100644 --- a/vendor/github.com/dustin/go-humanize/ftoa.go +++ b/vendor/github.com/dustin/go-humanize/ftoa.go @@ -6,6 +6,9 @@ import ( ) func stripTrailingZeros(s string) string { + if !strings.ContainsRune(s, '.') { + return s + } offset := len(s) - 1 for offset > 0 { if s[offset] == '.' { diff --git a/vendor/github.com/dustin/go-humanize/number.go b/vendor/github.com/dustin/go-humanize/number.go index dec618659..6470d0d47 100644 --- a/vendor/github.com/dustin/go-humanize/number.go +++ b/vendor/github.com/dustin/go-humanize/number.go @@ -73,7 +73,7 @@ func FormatFloat(format string, n float64) string { if n > math.MaxFloat64 { return "Infinity" } - if n < -math.MaxFloat64 { + if n < (0.0 - math.MaxFloat64) { return "-Infinity" } diff --git a/vendor/github.com/dustin/go-humanize/si.go b/vendor/github.com/dustin/go-humanize/si.go index ae659e0e4..8b8501984 100644 --- a/vendor/github.com/dustin/go-humanize/si.go +++ b/vendor/github.com/dustin/go-humanize/si.go @@ -8,6 +8,8 @@ import ( ) var siPrefixTable = map[float64]string{ + -30: "q", // quecto + -27: "r", // ronto -24: "y", // yocto -21: "z", // zepto -18: "a", // atto @@ -25,6 +27,8 @@ var siPrefixTable = map[float64]string{ 18: "E", // exa 21: "Z", // zetta 24: "Y", // yotta + 27: "R", // ronna + 30: "Q", // quetta } var revSIPrefixTable = revfmap(siPrefixTable) diff --git a/vendor/github.com/go-logr/logr/funcr/funcr.go b/vendor/github.com/go-logr/logr/funcr/funcr.go new file mode 100644 index 000000000..7accdb0c4 --- /dev/null +++ b/vendor/github.com/go-logr/logr/funcr/funcr.go @@ -0,0 +1,787 @@ +/* +Copyright 2021 The logr Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package funcr implements formatting of structured log messages and +// optionally captures the call site and timestamp. +// +// The simplest way to use it is via its implementation of a +// github.com/go-logr/logr.LogSink with output through an arbitrary +// "write" function. See New and NewJSON for details. +// +// Custom LogSinks +// +// For users who need more control, a funcr.Formatter can be embedded inside +// your own custom LogSink implementation. This is useful when the LogSink +// needs to implement additional methods, for example. +// +// Formatting +// +// This will respect logr.Marshaler, fmt.Stringer, and error interfaces for +// values which are being logged. When rendering a struct, funcr will use Go's +// standard JSON tags (all except "string"). +package funcr + +import ( + "bytes" + "encoding" + "fmt" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "time" + + "github.com/go-logr/logr" +) + +// New returns a logr.Logger which is implemented by an arbitrary function. +func New(fn func(prefix, args string), opts Options) logr.Logger { + return logr.New(newSink(fn, NewFormatter(opts))) +} + +// NewJSON returns a logr.Logger which is implemented by an arbitrary function +// and produces JSON output. +func NewJSON(fn func(obj string), opts Options) logr.Logger { + fnWrapper := func(_, obj string) { + fn(obj) + } + return logr.New(newSink(fnWrapper, NewFormatterJSON(opts))) +} + +// Underlier exposes access to the underlying logging function. Since +// callers only have a logr.Logger, they have to know which +// implementation is in use, so this interface is less of an +// abstraction and more of a way to test type conversion. +type Underlier interface { + GetUnderlying() func(prefix, args string) +} + +func newSink(fn func(prefix, args string), formatter Formatter) logr.LogSink { + l := &fnlogger{ + Formatter: formatter, + write: fn, + } + // For skipping fnlogger.Info and fnlogger.Error. + l.Formatter.AddCallDepth(1) + return l +} + +// Options carries parameters which influence the way logs are generated. +type Options struct { + // LogCaller tells funcr to add a "caller" key to some or all log lines. + // This has some overhead, so some users might not want it. + LogCaller MessageClass + + // LogCallerFunc tells funcr to also log the calling function name. This + // has no effect if caller logging is not enabled (see Options.LogCaller). + LogCallerFunc bool + + // LogTimestamp tells funcr to add a "ts" key to log lines. This has some + // overhead, so some users might not want it. + LogTimestamp bool + + // TimestampFormat tells funcr how to render timestamps when LogTimestamp + // is enabled. If not specified, a default format will be used. For more + // details, see docs for Go's time.Layout. + TimestampFormat string + + // Verbosity tells funcr which V logs to produce. Higher values enable + // more logs. Info logs at or below this level will be written, while logs + // above this level will be discarded. + Verbosity int + + // RenderBuiltinsHook allows users to mutate the list of key-value pairs + // while a log line is being rendered. The kvList argument follows logr + // conventions - each pair of slice elements is comprised of a string key + // and an arbitrary value (verified and sanitized before calling this + // hook). The value returned must follow the same conventions. This hook + // can be used to audit or modify logged data. For example, you might want + // to prefix all of funcr's built-in keys with some string. This hook is + // only called for built-in (provided by funcr itself) key-value pairs. + // Equivalent hooks are offered for key-value pairs saved via + // logr.Logger.WithValues or Formatter.AddValues (see RenderValuesHook) and + // for user-provided pairs (see RenderArgsHook). + RenderBuiltinsHook func(kvList []interface{}) []interface{} + + // RenderValuesHook is the same as RenderBuiltinsHook, except that it is + // only called for key-value pairs saved via logr.Logger.WithValues. See + // RenderBuiltinsHook for more details. + RenderValuesHook func(kvList []interface{}) []interface{} + + // RenderArgsHook is the same as RenderBuiltinsHook, except that it is only + // called for key-value pairs passed directly to Info and Error. See + // RenderBuiltinsHook for more details. + RenderArgsHook func(kvList []interface{}) []interface{} + + // MaxLogDepth tells funcr how many levels of nested fields (e.g. a struct + // that contains a struct, etc.) it may log. Every time it finds a struct, + // slice, array, or map the depth is increased by one. When the maximum is + // reached, the value will be converted to a string indicating that the max + // depth has been exceeded. If this field is not specified, a default + // value will be used. + MaxLogDepth int +} + +// MessageClass indicates which category or categories of messages to consider. +type MessageClass int + +const ( + // None ignores all message classes. + None MessageClass = iota + // All considers all message classes. + All + // Info only considers info messages. + Info + // Error only considers error messages. + Error +) + +// fnlogger inherits some of its LogSink implementation from Formatter +// and just needs to add some glue code. +type fnlogger struct { + Formatter + write func(prefix, args string) +} + +func (l fnlogger) WithName(name string) logr.LogSink { + l.Formatter.AddName(name) + return &l +} + +func (l fnlogger) WithValues(kvList ...interface{}) logr.LogSink { + l.Formatter.AddValues(kvList) + return &l +} + +func (l fnlogger) WithCallDepth(depth int) logr.LogSink { + l.Formatter.AddCallDepth(depth) + return &l +} + +func (l fnlogger) Info(level int, msg string, kvList ...interface{}) { + prefix, args := l.FormatInfo(level, msg, kvList) + l.write(prefix, args) +} + +func (l fnlogger) Error(err error, msg string, kvList ...interface{}) { + prefix, args := l.FormatError(err, msg, kvList) + l.write(prefix, args) +} + +func (l fnlogger) GetUnderlying() func(prefix, args string) { + return l.write +} + +// Assert conformance to the interfaces. +var _ logr.LogSink = &fnlogger{} +var _ logr.CallDepthLogSink = &fnlogger{} +var _ Underlier = &fnlogger{} + +// NewFormatter constructs a Formatter which emits a JSON-like key=value format. +func NewFormatter(opts Options) Formatter { + return newFormatter(opts, outputKeyValue) +} + +// NewFormatterJSON constructs a Formatter which emits strict JSON. +func NewFormatterJSON(opts Options) Formatter { + return newFormatter(opts, outputJSON) +} + +// Defaults for Options. +const defaultTimestampFormat = "2006-01-02 15:04:05.000000" +const defaultMaxLogDepth = 16 + +func newFormatter(opts Options, outfmt outputFormat) Formatter { + if opts.TimestampFormat == "" { + opts.TimestampFormat = defaultTimestampFormat + } + if opts.MaxLogDepth == 0 { + opts.MaxLogDepth = defaultMaxLogDepth + } + f := Formatter{ + outputFormat: outfmt, + prefix: "", + values: nil, + depth: 0, + opts: opts, + } + return f +} + +// Formatter is an opaque struct which can be embedded in a LogSink +// implementation. It should be constructed with NewFormatter. Some of +// its methods directly implement logr.LogSink. +type Formatter struct { + outputFormat outputFormat + prefix string + values []interface{} + valuesStr string + depth int + opts Options +} + +// outputFormat indicates which outputFormat to use. +type outputFormat int + +const ( + // outputKeyValue emits a JSON-like key=value format, but not strict JSON. + outputKeyValue outputFormat = iota + // outputJSON emits strict JSON. + outputJSON +) + +// PseudoStruct is a list of key-value pairs that gets logged as a struct. +type PseudoStruct []interface{} + +// render produces a log line, ready to use. +func (f Formatter) render(builtins, args []interface{}) string { + // Empirically bytes.Buffer is faster than strings.Builder for this. + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + if f.outputFormat == outputJSON { + buf.WriteByte('{') + } + vals := builtins + if hook := f.opts.RenderBuiltinsHook; hook != nil { + vals = hook(f.sanitize(vals)) + } + f.flatten(buf, vals, false, false) // keys are ours, no need to escape + continuing := len(builtins) > 0 + if len(f.valuesStr) > 0 { + if continuing { + if f.outputFormat == outputJSON { + buf.WriteByte(',') + } else { + buf.WriteByte(' ') + } + } + continuing = true + buf.WriteString(f.valuesStr) + } + vals = args + if hook := f.opts.RenderArgsHook; hook != nil { + vals = hook(f.sanitize(vals)) + } + f.flatten(buf, vals, continuing, true) // escape user-provided keys + if f.outputFormat == outputJSON { + buf.WriteByte('}') + } + return buf.String() +} + +// flatten renders a list of key-value pairs into a buffer. If continuing is +// true, it assumes that the buffer has previous values and will emit a +// separator (which depends on the output format) before the first pair it +// writes. If escapeKeys is true, the keys are assumed to have +// non-JSON-compatible characters in them and must be evaluated for escapes. +// +// This function returns a potentially modified version of kvList, which +// ensures that there is a value for every key (adding a value if needed) and +// that each key is a string (substituting a key if needed). +func (f Formatter) flatten(buf *bytes.Buffer, kvList []interface{}, continuing bool, escapeKeys bool) []interface{} { + // This logic overlaps with sanitize() but saves one type-cast per key, + // which can be measurable. + if len(kvList)%2 != 0 { + kvList = append(kvList, noValue) + } + for i := 0; i < len(kvList); i += 2 { + k, ok := kvList[i].(string) + if !ok { + k = f.nonStringKey(kvList[i]) + kvList[i] = k + } + v := kvList[i+1] + + if i > 0 || continuing { + if f.outputFormat == outputJSON { + buf.WriteByte(',') + } else { + // In theory the format could be something we don't understand. In + // practice, we control it, so it won't be. + buf.WriteByte(' ') + } + } + + if escapeKeys { + buf.WriteString(prettyString(k)) + } else { + // this is faster + buf.WriteByte('"') + buf.WriteString(k) + buf.WriteByte('"') + } + if f.outputFormat == outputJSON { + buf.WriteByte(':') + } else { + buf.WriteByte('=') + } + buf.WriteString(f.pretty(v)) + } + return kvList +} + +func (f Formatter) pretty(value interface{}) string { + return f.prettyWithFlags(value, 0, 0) +} + +const ( + flagRawStruct = 0x1 // do not print braces on structs +) + +// TODO: This is not fast. Most of the overhead goes here. +func (f Formatter) prettyWithFlags(value interface{}, flags uint32, depth int) string { + if depth > f.opts.MaxLogDepth { + return `""` + } + + // Handle types that take full control of logging. + if v, ok := value.(logr.Marshaler); ok { + // Replace the value with what the type wants to get logged. + // That then gets handled below via reflection. + value = invokeMarshaler(v) + } + + // Handle types that want to format themselves. + switch v := value.(type) { + case fmt.Stringer: + value = invokeStringer(v) + case error: + value = invokeError(v) + } + + // Handling the most common types without reflect is a small perf win. + switch v := value.(type) { + case bool: + return strconv.FormatBool(v) + case string: + return prettyString(v) + case int: + return strconv.FormatInt(int64(v), 10) + case int8: + return strconv.FormatInt(int64(v), 10) + case int16: + return strconv.FormatInt(int64(v), 10) + case int32: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(int64(v), 10) + case uint: + return strconv.FormatUint(uint64(v), 10) + case uint8: + return strconv.FormatUint(uint64(v), 10) + case uint16: + return strconv.FormatUint(uint64(v), 10) + case uint32: + return strconv.FormatUint(uint64(v), 10) + case uint64: + return strconv.FormatUint(v, 10) + case uintptr: + return strconv.FormatUint(uint64(v), 10) + case float32: + return strconv.FormatFloat(float64(v), 'f', -1, 32) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + case complex64: + return `"` + strconv.FormatComplex(complex128(v), 'f', -1, 64) + `"` + case complex128: + return `"` + strconv.FormatComplex(v, 'f', -1, 128) + `"` + case PseudoStruct: + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + v = f.sanitize(v) + if flags&flagRawStruct == 0 { + buf.WriteByte('{') + } + for i := 0; i < len(v); i += 2 { + if i > 0 { + buf.WriteByte(',') + } + k, _ := v[i].(string) // sanitize() above means no need to check success + // arbitrary keys might need escaping + buf.WriteString(prettyString(k)) + buf.WriteByte(':') + buf.WriteString(f.prettyWithFlags(v[i+1], 0, depth+1)) + } + if flags&flagRawStruct == 0 { + buf.WriteByte('}') + } + return buf.String() + } + + buf := bytes.NewBuffer(make([]byte, 0, 256)) + t := reflect.TypeOf(value) + if t == nil { + return "null" + } + v := reflect.ValueOf(value) + switch t.Kind() { + case reflect.Bool: + return strconv.FormatBool(v.Bool()) + case reflect.String: + return prettyString(v.String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.FormatInt(int64(v.Int()), 10) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return strconv.FormatUint(uint64(v.Uint()), 10) + case reflect.Float32: + return strconv.FormatFloat(float64(v.Float()), 'f', -1, 32) + case reflect.Float64: + return strconv.FormatFloat(v.Float(), 'f', -1, 64) + case reflect.Complex64: + return `"` + strconv.FormatComplex(complex128(v.Complex()), 'f', -1, 64) + `"` + case reflect.Complex128: + return `"` + strconv.FormatComplex(v.Complex(), 'f', -1, 128) + `"` + case reflect.Struct: + if flags&flagRawStruct == 0 { + buf.WriteByte('{') + } + for i := 0; i < t.NumField(); i++ { + fld := t.Field(i) + if fld.PkgPath != "" { + // reflect says this field is only defined for non-exported fields. + continue + } + if !v.Field(i).CanInterface() { + // reflect isn't clear exactly what this means, but we can't use it. + continue + } + name := "" + omitempty := false + if tag, found := fld.Tag.Lookup("json"); found { + if tag == "-" { + continue + } + if comma := strings.Index(tag, ","); comma != -1 { + if n := tag[:comma]; n != "" { + name = n + } + rest := tag[comma:] + if strings.Contains(rest, ",omitempty,") || strings.HasSuffix(rest, ",omitempty") { + omitempty = true + } + } else { + name = tag + } + } + if omitempty && isEmpty(v.Field(i)) { + continue + } + if i > 0 { + buf.WriteByte(',') + } + if fld.Anonymous && fld.Type.Kind() == reflect.Struct && name == "" { + buf.WriteString(f.prettyWithFlags(v.Field(i).Interface(), flags|flagRawStruct, depth+1)) + continue + } + if name == "" { + name = fld.Name + } + // field names can't contain characters which need escaping + buf.WriteByte('"') + buf.WriteString(name) + buf.WriteByte('"') + buf.WriteByte(':') + buf.WriteString(f.prettyWithFlags(v.Field(i).Interface(), 0, depth+1)) + } + if flags&flagRawStruct == 0 { + buf.WriteByte('}') + } + return buf.String() + case reflect.Slice, reflect.Array: + buf.WriteByte('[') + for i := 0; i < v.Len(); i++ { + if i > 0 { + buf.WriteByte(',') + } + e := v.Index(i) + buf.WriteString(f.prettyWithFlags(e.Interface(), 0, depth+1)) + } + buf.WriteByte(']') + return buf.String() + case reflect.Map: + buf.WriteByte('{') + // This does not sort the map keys, for best perf. + it := v.MapRange() + i := 0 + for it.Next() { + if i > 0 { + buf.WriteByte(',') + } + // If a map key supports TextMarshaler, use it. + keystr := "" + if m, ok := it.Key().Interface().(encoding.TextMarshaler); ok { + txt, err := m.MarshalText() + if err != nil { + keystr = fmt.Sprintf("", err.Error()) + } else { + keystr = string(txt) + } + keystr = prettyString(keystr) + } else { + // prettyWithFlags will produce already-escaped values + keystr = f.prettyWithFlags(it.Key().Interface(), 0, depth+1) + if t.Key().Kind() != reflect.String { + // JSON only does string keys. Unlike Go's standard JSON, we'll + // convert just about anything to a string. + keystr = prettyString(keystr) + } + } + buf.WriteString(keystr) + buf.WriteByte(':') + buf.WriteString(f.prettyWithFlags(it.Value().Interface(), 0, depth+1)) + i++ + } + buf.WriteByte('}') + return buf.String() + case reflect.Ptr, reflect.Interface: + if v.IsNil() { + return "null" + } + return f.prettyWithFlags(v.Elem().Interface(), 0, depth) + } + return fmt.Sprintf(`""`, t.Kind().String()) +} + +func prettyString(s string) string { + // Avoid escaping (which does allocations) if we can. + if needsEscape(s) { + return strconv.Quote(s) + } + b := bytes.NewBuffer(make([]byte, 0, 1024)) + b.WriteByte('"') + b.WriteString(s) + b.WriteByte('"') + return b.String() +} + +// needsEscape determines whether the input string needs to be escaped or not, +// without doing any allocations. +func needsEscape(s string) bool { + for _, r := range s { + if !strconv.IsPrint(r) || r == '\\' || r == '"' { + return true + } + } + return false +} + +func isEmpty(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Complex64, reflect.Complex128: + return v.Complex() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} + +func invokeMarshaler(m logr.Marshaler) (ret interface{}) { + defer func() { + if r := recover(); r != nil { + ret = fmt.Sprintf("", r) + } + }() + return m.MarshalLog() +} + +func invokeStringer(s fmt.Stringer) (ret string) { + defer func() { + if r := recover(); r != nil { + ret = fmt.Sprintf("", r) + } + }() + return s.String() +} + +func invokeError(e error) (ret string) { + defer func() { + if r := recover(); r != nil { + ret = fmt.Sprintf("", r) + } + }() + return e.Error() +} + +// Caller represents the original call site for a log line, after considering +// logr.Logger.WithCallDepth and logr.Logger.WithCallStackHelper. The File and +// Line fields will always be provided, while the Func field is optional. +// Users can set the render hook fields in Options to examine logged key-value +// pairs, one of which will be {"caller", Caller} if the Options.LogCaller +// field is enabled for the given MessageClass. +type Caller struct { + // File is the basename of the file for this call site. + File string `json:"file"` + // Line is the line number in the file for this call site. + Line int `json:"line"` + // Func is the function name for this call site, or empty if + // Options.LogCallerFunc is not enabled. + Func string `json:"function,omitempty"` +} + +func (f Formatter) caller() Caller { + // +1 for this frame, +1 for Info/Error. + pc, file, line, ok := runtime.Caller(f.depth + 2) + if !ok { + return Caller{"", 0, ""} + } + fn := "" + if f.opts.LogCallerFunc { + if fp := runtime.FuncForPC(pc); fp != nil { + fn = fp.Name() + } + } + + return Caller{filepath.Base(file), line, fn} +} + +const noValue = "" + +func (f Formatter) nonStringKey(v interface{}) string { + return fmt.Sprintf("", f.snippet(v)) +} + +// snippet produces a short snippet string of an arbitrary value. +func (f Formatter) snippet(v interface{}) string { + const snipLen = 16 + + snip := f.pretty(v) + if len(snip) > snipLen { + snip = snip[:snipLen] + } + return snip +} + +// sanitize ensures that a list of key-value pairs has a value for every key +// (adding a value if needed) and that each key is a string (substituting a key +// if needed). +func (f Formatter) sanitize(kvList []interface{}) []interface{} { + if len(kvList)%2 != 0 { + kvList = append(kvList, noValue) + } + for i := 0; i < len(kvList); i += 2 { + _, ok := kvList[i].(string) + if !ok { + kvList[i] = f.nonStringKey(kvList[i]) + } + } + return kvList +} + +// Init configures this Formatter from runtime info, such as the call depth +// imposed by logr itself. +// Note that this receiver is a pointer, so depth can be saved. +func (f *Formatter) Init(info logr.RuntimeInfo) { + f.depth += info.CallDepth +} + +// Enabled checks whether an info message at the given level should be logged. +func (f Formatter) Enabled(level int) bool { + return level <= f.opts.Verbosity +} + +// GetDepth returns the current depth of this Formatter. This is useful for +// implementations which do their own caller attribution. +func (f Formatter) GetDepth() int { + return f.depth +} + +// FormatInfo renders an Info log message into strings. The prefix will be +// empty when no names were set (via AddNames), or when the output is +// configured for JSON. +func (f Formatter) FormatInfo(level int, msg string, kvList []interface{}) (prefix, argsStr string) { + args := make([]interface{}, 0, 64) // using a constant here impacts perf + prefix = f.prefix + if f.outputFormat == outputJSON { + args = append(args, "logger", prefix) + prefix = "" + } + if f.opts.LogTimestamp { + args = append(args, "ts", time.Now().Format(f.opts.TimestampFormat)) + } + if policy := f.opts.LogCaller; policy == All || policy == Info { + args = append(args, "caller", f.caller()) + } + args = append(args, "level", level, "msg", msg) + return prefix, f.render(args, kvList) +} + +// FormatError renders an Error log message into strings. The prefix will be +// empty when no names were set (via AddNames), or when the output is +// configured for JSON. +func (f Formatter) FormatError(err error, msg string, kvList []interface{}) (prefix, argsStr string) { + args := make([]interface{}, 0, 64) // using a constant here impacts perf + prefix = f.prefix + if f.outputFormat == outputJSON { + args = append(args, "logger", prefix) + prefix = "" + } + if f.opts.LogTimestamp { + args = append(args, "ts", time.Now().Format(f.opts.TimestampFormat)) + } + if policy := f.opts.LogCaller; policy == All || policy == Error { + args = append(args, "caller", f.caller()) + } + args = append(args, "msg", msg) + var loggableErr interface{} + if err != nil { + loggableErr = err.Error() + } + args = append(args, "error", loggableErr) + return f.prefix, f.render(args, kvList) +} + +// AddName appends the specified name. funcr uses '/' characters to separate +// name elements. Callers should not pass '/' in the provided name string, but +// this library does not actually enforce that. +func (f *Formatter) AddName(name string) { + if len(f.prefix) > 0 { + f.prefix += "/" + } + f.prefix += name +} + +// AddValues adds key-value pairs to the set of saved values to be logged with +// each log line. +func (f *Formatter) AddValues(kvList []interface{}) { + // Three slice args forces a copy. + n := len(f.values) + f.values = append(f.values[:n:n], kvList...) + + vals := f.values + if hook := f.opts.RenderValuesHook; hook != nil { + vals = hook(f.sanitize(vals)) + } + + // Pre-render values, so we don't have to do it on each Info/Error call. + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + f.flatten(buf, vals, false, true) // escape user-provided keys + f.valuesStr = buf.String() +} + +// AddCallDepth increases the number of stack-frames to skip when attributing +// the log line to a file and line. +func (f *Formatter) AddCallDepth(depth int) { + f.depth += depth +} diff --git a/vendor/github.com/go-task/slim-sprig/.editorconfig b/vendor/github.com/go-task/slim-sprig/.editorconfig new file mode 100644 index 000000000..b0c95367e --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/.editorconfig @@ -0,0 +1,14 @@ +# editorconfig.org + +root = true + +[*] +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = tab +indent_size = 8 + +[*.{md,yml,yaml,json}] +indent_style = space +indent_size = 2 diff --git a/vendor/github.com/go-task/slim-sprig/.gitattributes b/vendor/github.com/go-task/slim-sprig/.gitattributes new file mode 100644 index 000000000..176a458f9 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/.gitattributes @@ -0,0 +1 @@ +* text=auto diff --git a/vendor/github.com/go-task/slim-sprig/.gitignore b/vendor/github.com/go-task/slim-sprig/.gitignore new file mode 100644 index 000000000..5e3002f88 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/.gitignore @@ -0,0 +1,2 @@ +vendor/ +/.glide diff --git a/vendor/github.com/go-task/slim-sprig/CHANGELOG.md b/vendor/github.com/go-task/slim-sprig/CHANGELOG.md new file mode 100644 index 000000000..61d8ebffc --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/CHANGELOG.md @@ -0,0 +1,364 @@ +# Changelog + +## Release 3.2.0 (2020-12-14) + +### Added + +- #211: Added randInt function (thanks @kochurovro) +- #223: Added fromJson and mustFromJson functions (thanks @mholt) +- #242: Added a bcrypt function (thanks @robbiet480) +- #253: Added randBytes function (thanks @MikaelSmith) +- #254: Added dig function for dicts (thanks @nyarly) +- #257: Added regexQuoteMeta for quoting regex metadata (thanks @rheaton) +- #261: Added filepath functions osBase, osDir, osExt, osClean, osIsAbs (thanks @zugl) +- #268: Added and and all functions for testing conditions (thanks @phuslu) +- #181: Added float64 arithmetic addf, add1f, subf, divf, mulf, maxf, and minf + (thanks @andrewmostello) +- #265: Added chunk function to split array into smaller arrays (thanks @karelbilek) +- #270: Extend certificate functions to handle non-RSA keys + add support for + ed25519 keys (thanks @misberner) + +### Changed + +- Removed testing and support for Go 1.12. ed25519 support requires Go 1.13 or newer +- Using semver 3.1.1 and mergo 0.3.11 + +### Fixed + +- #249: Fix htmlDateInZone example (thanks @spawnia) + +NOTE: The dependency github.com/imdario/mergo reverted the breaking change in +0.3.9 via 0.3.10 release. + +## Release 3.1.0 (2020-04-16) + +NOTE: The dependency github.com/imdario/mergo made a behavior change in 0.3.9 +that impacts sprig functionality. Do not use sprig with a version newer than 0.3.8. + +### Added + +- #225: Added support for generating htpasswd hash (thanks @rustycl0ck) +- #224: Added duration filter (thanks @frebib) +- #205: Added `seq` function (thanks @thadc23) + +### Changed + +- #203: Unlambda functions with correct signature (thanks @muesli) +- #236: Updated the license formatting for GitHub display purposes +- #238: Updated package dependency versions. Note, mergo not updated to 0.3.9 + as it causes a breaking change for sprig. That issue is tracked at + https://github.com/imdario/mergo/issues/139 + +### Fixed + +- #229: Fix `seq` example in docs (thanks @kalmant) + +## Release 3.0.2 (2019-12-13) + +### Fixed + +- #220: Updating to semver v3.0.3 to fix issue with <= ranges +- #218: fix typo elyptical->elliptic in ecdsa key description (thanks @laverya) + +## Release 3.0.1 (2019-12-08) + +### Fixed + +- #212: Updated semver fixing broken constraint checking with ^0.0 + +## Release 3.0.0 (2019-10-02) + +### Added + +- #187: Added durationRound function (thanks @yjp20) +- #189: Added numerous template functions that return errors rather than panic (thanks @nrvnrvn) +- #193: Added toRawJson support (thanks @Dean-Coakley) +- #197: Added get support to dicts (thanks @Dean-Coakley) + +### Changed + +- #186: Moving dependency management to Go modules +- #186: Updated semver to v3. This has changes in the way ^ is handled +- #194: Updated documentation on merging and how it copies. Added example using deepCopy +- #196: trunc now supports negative values (thanks @Dean-Coakley) + +## Release 2.22.0 (2019-10-02) + +### Added + +- #173: Added getHostByName function to resolve dns names to ips (thanks @fcgravalos) +- #195: Added deepCopy function for use with dicts + +### Changed + +- Updated merge and mergeOverwrite documentation to explain copying and how to + use deepCopy with it + +## Release 2.21.0 (2019-09-18) + +### Added + +- #122: Added encryptAES/decryptAES functions (thanks @n0madic) +- #128: Added toDecimal support (thanks @Dean-Coakley) +- #169: Added list contcat (thanks @astorath) +- #174: Added deepEqual function (thanks @bonifaido) +- #170: Added url parse and join functions (thanks @astorath) + +### Changed + +- #171: Updated glide config for Google UUID to v1 and to add ranges to semver and testify + +### Fixed + +- #172: Fix semver wildcard example (thanks @piepmatz) +- #175: Fix dateInZone doc example (thanks @s3than) + +## Release 2.20.0 (2019-06-18) + +### Added + +- #164: Adding function to get unix epoch for a time (@mattfarina) +- #166: Adding tests for date_in_zone (@mattfarina) + +### Changed + +- #144: Fix function comments based on best practices from Effective Go (@CodeLingoTeam) +- #150: Handles pointer type for time.Time in "htmlDate" (@mapreal19) +- #161, #157, #160, #153, #158, #156, #155, #159, #152 documentation updates (@badeadan) + +### Fixed + +## Release 2.19.0 (2019-03-02) + +IMPORTANT: This release reverts a change from 2.18.0 + +In the previous release (2.18), we prematurely merged a partial change to the crypto functions that led to creating two sets of crypto functions (I blame @technosophos -- since that's me). This release rolls back that change, and does what was originally intended: It alters the existing crypto functions to use secure random. + +We debated whether this classifies as a change worthy of major revision, but given the proximity to the last release, we have decided that treating 2.18 as a faulty release is the correct course of action. We apologize for any inconvenience. + +### Changed + +- Fix substr panic 35fb796 (Alexey igrychev) +- Remove extra period 1eb7729 (Matthew Lorimor) +- Make random string functions use crypto by default 6ceff26 (Matthew Lorimor) +- README edits/fixes/suggestions 08fe136 (Lauri Apple) + + +## Release 2.18.0 (2019-02-12) + +### Added + +- Added mergeOverwrite function +- cryptographic functions that use secure random (see fe1de12) + +### Changed + +- Improve documentation of regexMatch function, resolves #139 90b89ce (Jan Tagscherer) +- Handle has for nil list 9c10885 (Daniel Cohen) +- Document behaviour of mergeOverwrite fe0dbe9 (Lukas Rieder) +- doc: adds missing documentation. 4b871e6 (Fernandez Ludovic) +- Replace outdated goutils imports 01893d2 (Matthew Lorimor) +- Surface crypto secure random strings from goutils fe1de12 (Matthew Lorimor) +- Handle untyped nil values as paramters to string functions 2b2ec8f (Morten Torkildsen) + +### Fixed + +- Fix dict merge issue and provide mergeOverwrite .dst .src1 to overwrite from src -> dst 4c59c12 (Lukas Rieder) +- Fix substr var names and comments d581f80 (Dean Coakley) +- Fix substr documentation 2737203 (Dean Coakley) + +## Release 2.17.1 (2019-01-03) + +### Fixed + +The 2.17.0 release did not have a version pinned for xstrings, which caused compilation failures when xstrings < 1.2 was used. This adds the correct version string to glide.yaml. + +## Release 2.17.0 (2019-01-03) + +### Added + +- adds alder32sum function and test 6908fc2 (marshallford) +- Added kebabcase function ca331a1 (Ilyes512) + +### Changed + +- Update goutils to 1.1.0 4e1125d (Matt Butcher) + +### Fixed + +- Fix 'has' documentation e3f2a85 (dean-coakley) +- docs(dict): fix typo in pick example dc424f9 (Dustin Specker) +- fixes spelling errors... not sure how that happened 4cf188a (marshallford) + +## Release 2.16.0 (2018-08-13) + +### Added + +- add splitn function fccb0b0 (Helgi Þorbjörnsson) +- Add slice func df28ca7 (gongdo) +- Generate serial number a3bdffd (Cody Coons) +- Extract values of dict with values function df39312 (Lawrence Jones) + +### Changed + +- Modify panic message for list.slice ae38335 (gongdo) +- Minor improvement in code quality - Removed an unreachable piece of code at defaults.go#L26:6 - Resolve formatting issues. 5834241 (Abhishek Kashyap) +- Remove duplicated documentation 1d97af1 (Matthew Fisher) +- Test on go 1.11 49df809 (Helgi Þormar Þorbjörnsson) + +### Fixed + +- Fix file permissions c5f40b5 (gongdo) +- Fix example for buildCustomCert 7779e0d (Tin Lam) + +## Release 2.15.0 (2018-04-02) + +### Added + +- #68 and #69: Add json helpers to docs (thanks @arunvelsriram) +- #66: Add ternary function (thanks @binoculars) +- #67: Allow keys function to take multiple dicts (thanks @binoculars) +- #89: Added sha1sum to crypto function (thanks @benkeil) +- #81: Allow customizing Root CA that used by genSignedCert (thanks @chenzhiwei) +- #92: Add travis testing for go 1.10 +- #93: Adding appveyor config for windows testing + +### Changed + +- #90: Updating to more recent dependencies +- #73: replace satori/go.uuid with google/uuid (thanks @petterw) + +### Fixed + +- #76: Fixed documentation typos (thanks @Thiht) +- Fixed rounding issue on the `ago` function. Note, the removes support for Go 1.8 and older + +## Release 2.14.1 (2017-12-01) + +### Fixed + +- #60: Fix typo in function name documentation (thanks @neil-ca-moore) +- #61: Removing line with {{ due to blocking github pages genertion +- #64: Update the list functions to handle int, string, and other slices for compatibility + +## Release 2.14.0 (2017-10-06) + +This new version of Sprig adds a set of functions for generating and working with SSL certificates. + +- `genCA` generates an SSL Certificate Authority +- `genSelfSignedCert` generates an SSL self-signed certificate +- `genSignedCert` generates an SSL certificate and key based on a given CA + +## Release 2.13.0 (2017-09-18) + +This release adds new functions, including: + +- `regexMatch`, `regexFindAll`, `regexFind`, `regexReplaceAll`, `regexReplaceAllLiteral`, and `regexSplit` to work with regular expressions +- `floor`, `ceil`, and `round` math functions +- `toDate` converts a string to a date +- `nindent` is just like `indent` but also prepends a new line +- `ago` returns the time from `time.Now` + +### Added + +- #40: Added basic regex functionality (thanks @alanquillin) +- #41: Added ceil floor and round functions (thanks @alanquillin) +- #48: Added toDate function (thanks @andreynering) +- #50: Added nindent function (thanks @binoculars) +- #46: Added ago function (thanks @slayer) + +### Changed + +- #51: Updated godocs to include new string functions (thanks @curtisallen) +- #49: Added ability to merge multiple dicts (thanks @binoculars) + +## Release 2.12.0 (2017-05-17) + +- `snakecase`, `camelcase`, and `shuffle` are three new string functions +- `fail` allows you to bail out of a template render when conditions are not met + +## Release 2.11.0 (2017-05-02) + +- Added `toJson` and `toPrettyJson` +- Added `merge` +- Refactored documentation + +## Release 2.10.0 (2017-03-15) + +- Added `semver` and `semverCompare` for Semantic Versions +- `list` replaces `tuple` +- Fixed issue with `join` +- Added `first`, `last`, `intial`, `rest`, `prepend`, `append`, `toString`, `toStrings`, `sortAlpha`, `reverse`, `coalesce`, `pluck`, `pick`, `compact`, `keys`, `omit`, `uniq`, `has`, `without` + +## Release 2.9.0 (2017-02-23) + +- Added `splitList` to split a list +- Added crypto functions of `genPrivateKey` and `derivePassword` + +## Release 2.8.0 (2016-12-21) + +- Added access to several path functions (`base`, `dir`, `clean`, `ext`, and `abs`) +- Added functions for _mutating_ dictionaries (`set`, `unset`, `hasKey`) + +## Release 2.7.0 (2016-12-01) + +- Added `sha256sum` to generate a hash of an input +- Added functions to convert a numeric or string to `int`, `int64`, `float64` + +## Release 2.6.0 (2016-10-03) + +- Added a `uuidv4` template function for generating UUIDs inside of a template. + +## Release 2.5.0 (2016-08-19) + +- New `trimSuffix`, `trimPrefix`, `hasSuffix`, and `hasPrefix` functions +- New aliases have been added for a few functions that didn't follow the naming conventions (`trimAll` and `abbrevBoth`) +- `trimall` and `abbrevboth` (notice the case) are deprecated and will be removed in 3.0.0 + +## Release 2.4.0 (2016-08-16) + +- Adds two functions: `until` and `untilStep` + +## Release 2.3.0 (2016-06-21) + +- cat: Concatenate strings with whitespace separators. +- replace: Replace parts of a string: `replace " " "-" "Me First"` renders "Me-First" +- plural: Format plurals: `len "foo" | plural "one foo" "many foos"` renders "many foos" +- indent: Indent blocks of text in a way that is sensitive to "\n" characters. + +## Release 2.2.0 (2016-04-21) + +- Added a `genPrivateKey` function (Thanks @bacongobbler) + +## Release 2.1.0 (2016-03-30) + +- `default` now prints the default value when it does not receive a value down the pipeline. It is much safer now to do `{{.Foo | default "bar"}}`. +- Added accessors for "hermetic" functions. These return only functions that, when given the same input, produce the same output. + +## Release 2.0.0 (2016-03-29) + +Because we switched from `int` to `int64` as the return value for all integer math functions, the library's major version number has been incremented. + +- `min` complements `max` (formerly `biggest`) +- `empty` indicates that a value is the empty value for its type +- `tuple` creates a tuple inside of a template: `{{$t := tuple "a", "b" "c"}}` +- `dict` creates a dictionary inside of a template `{{$d := dict "key1" "val1" "key2" "val2"}}` +- Date formatters have been added for HTML dates (as used in `date` input fields) +- Integer math functions can convert from a number of types, including `string` (via `strconv.ParseInt`). + +## Release 1.2.0 (2016-02-01) + +- Added quote and squote +- Added b32enc and b32dec +- add now takes varargs +- biggest now takes varargs + +## Release 1.1.0 (2015-12-29) + +- Added #4: Added contains function. strings.Contains, but with the arguments + switched to simplify common pipelines. (thanks krancour) +- Added Travis-CI testing support + +## Release 1.0.0 (2015-12-23) + +- Initial release diff --git a/vendor/github.com/sigurn/utils/LICENSE.md b/vendor/github.com/go-task/slim-sprig/LICENSE.txt similarity index 87% rename from vendor/github.com/sigurn/utils/LICENSE.md rename to vendor/github.com/go-task/slim-sprig/LICENSE.txt index 229e8d147..f311b1eaa 100644 --- a/vendor/github.com/sigurn/utils/LICENSE.md +++ b/vendor/github.com/go-task/slim-sprig/LICENSE.txt @@ -1,6 +1,4 @@ -The MIT License (MIT) - -Copyright (c) 2015 sigurn +Copyright (C) 2013-2020 Masterminds Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,13 +7,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/go-task/slim-sprig/README.md b/vendor/github.com/go-task/slim-sprig/README.md new file mode 100644 index 000000000..72579471f --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/README.md @@ -0,0 +1,73 @@ +# Slim-Sprig: Template functions for Go templates [![GoDoc](https://godoc.org/github.com/go-task/slim-sprig?status.svg)](https://godoc.org/github.com/go-task/slim-sprig) [![Go Report Card](https://goreportcard.com/badge/github.com/go-task/slim-sprig)](https://goreportcard.com/report/github.com/go-task/slim-sprig) + +Slim-Sprig is a fork of [Sprig](https://github.com/Masterminds/sprig), but with +all functions that depend on external (non standard library) or crypto packages +removed. +The reason for this is to make this library more lightweight. Most of these +functions (specially crypto ones) are not needed on most apps, but costs a lot +in terms of binary size and compilation time. + +## Usage + +**Template developers**: Please use Slim-Sprig's [function documentation](https://go-task.github.io/slim-sprig/) for +detailed instructions and code snippets for the >100 template functions available. + +**Go developers**: If you'd like to include Slim-Sprig as a library in your program, +our API documentation is available [at GoDoc.org](http://godoc.org/github.com/go-task/slim-sprig). + +For standard usage, read on. + +### Load the Slim-Sprig library + +To load the Slim-Sprig `FuncMap`: + +```go + +import ( + "html/template" + + "github.com/go-task/slim-sprig" +) + +// This example illustrates that the FuncMap *must* be set before the +// templates themselves are loaded. +tpl := template.Must( + template.New("base").Funcs(sprig.FuncMap()).ParseGlob("*.html") +) +``` + +### Calling the functions inside of templates + +By convention, all functions are lowercase. This seems to follow the Go +idiom for template functions (as opposed to template methods, which are +TitleCase). For example, this: + +``` +{{ "hello!" | upper | repeat 5 }} +``` + +produces this: + +``` +HELLO!HELLO!HELLO!HELLO!HELLO! +``` + +## Principles Driving Our Function Selection + +We followed these principles to decide which functions to add and how to implement them: + +- Use template functions to build layout. The following + types of operations are within the domain of template functions: + - Formatting + - Layout + - Simple type conversions + - Utilities that assist in handling common formatting and layout needs (e.g. arithmetic) +- Template functions should not return errors unless there is no way to print + a sensible value. For example, converting a string to an integer should not + produce an error if conversion fails. Instead, it should display a default + value. +- Simple math is necessary for grid layouts, pagers, and so on. Complex math + (anything other than arithmetic) should be done outside of templates. +- Template functions only deal with the data passed into them. They never retrieve + data from a source. +- Finally, do not override core Go template functions. diff --git a/vendor/github.com/go-task/slim-sprig/Taskfile.yml b/vendor/github.com/go-task/slim-sprig/Taskfile.yml new file mode 100644 index 000000000..cdcfd223b --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/Taskfile.yml @@ -0,0 +1,12 @@ +# https://taskfile.dev + +version: '2' + +tasks: + default: + cmds: + - task: test + + test: + cmds: + - go test -v . diff --git a/vendor/github.com/go-task/slim-sprig/crypto.go b/vendor/github.com/go-task/slim-sprig/crypto.go new file mode 100644 index 000000000..d06e516d4 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/crypto.go @@ -0,0 +1,24 @@ +package sprig + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "fmt" + "hash/adler32" +) + +func sha256sum(input string) string { + hash := sha256.Sum256([]byte(input)) + return hex.EncodeToString(hash[:]) +} + +func sha1sum(input string) string { + hash := sha1.Sum([]byte(input)) + return hex.EncodeToString(hash[:]) +} + +func adler32sum(input string) string { + hash := adler32.Checksum([]byte(input)) + return fmt.Sprintf("%d", hash) +} diff --git a/vendor/github.com/go-task/slim-sprig/date.go b/vendor/github.com/go-task/slim-sprig/date.go new file mode 100644 index 000000000..ed022ddac --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/date.go @@ -0,0 +1,152 @@ +package sprig + +import ( + "strconv" + "time" +) + +// Given a format and a date, format the date string. +// +// Date can be a `time.Time` or an `int, int32, int64`. +// In the later case, it is treated as seconds since UNIX +// epoch. +func date(fmt string, date interface{}) string { + return dateInZone(fmt, date, "Local") +} + +func htmlDate(date interface{}) string { + return dateInZone("2006-01-02", date, "Local") +} + +func htmlDateInZone(date interface{}, zone string) string { + return dateInZone("2006-01-02", date, zone) +} + +func dateInZone(fmt string, date interface{}, zone string) string { + var t time.Time + switch date := date.(type) { + default: + t = time.Now() + case time.Time: + t = date + case *time.Time: + t = *date + case int64: + t = time.Unix(date, 0) + case int: + t = time.Unix(int64(date), 0) + case int32: + t = time.Unix(int64(date), 0) + } + + loc, err := time.LoadLocation(zone) + if err != nil { + loc, _ = time.LoadLocation("UTC") + } + + return t.In(loc).Format(fmt) +} + +func dateModify(fmt string, date time.Time) time.Time { + d, err := time.ParseDuration(fmt) + if err != nil { + return date + } + return date.Add(d) +} + +func mustDateModify(fmt string, date time.Time) (time.Time, error) { + d, err := time.ParseDuration(fmt) + if err != nil { + return time.Time{}, err + } + return date.Add(d), nil +} + +func dateAgo(date interface{}) string { + var t time.Time + + switch date := date.(type) { + default: + t = time.Now() + case time.Time: + t = date + case int64: + t = time.Unix(date, 0) + case int: + t = time.Unix(int64(date), 0) + } + // Drop resolution to seconds + duration := time.Since(t).Round(time.Second) + return duration.String() +} + +func duration(sec interface{}) string { + var n int64 + switch value := sec.(type) { + default: + n = 0 + case string: + n, _ = strconv.ParseInt(value, 10, 64) + case int64: + n = value + } + return (time.Duration(n) * time.Second).String() +} + +func durationRound(duration interface{}) string { + var d time.Duration + switch duration := duration.(type) { + default: + d = 0 + case string: + d, _ = time.ParseDuration(duration) + case int64: + d = time.Duration(duration) + case time.Time: + d = time.Since(duration) + } + + u := uint64(d) + neg := d < 0 + if neg { + u = -u + } + + var ( + year = uint64(time.Hour) * 24 * 365 + month = uint64(time.Hour) * 24 * 30 + day = uint64(time.Hour) * 24 + hour = uint64(time.Hour) + minute = uint64(time.Minute) + second = uint64(time.Second) + ) + switch { + case u > year: + return strconv.FormatUint(u/year, 10) + "y" + case u > month: + return strconv.FormatUint(u/month, 10) + "mo" + case u > day: + return strconv.FormatUint(u/day, 10) + "d" + case u > hour: + return strconv.FormatUint(u/hour, 10) + "h" + case u > minute: + return strconv.FormatUint(u/minute, 10) + "m" + case u > second: + return strconv.FormatUint(u/second, 10) + "s" + } + return "0s" +} + +func toDate(fmt, str string) time.Time { + t, _ := time.ParseInLocation(fmt, str, time.Local) + return t +} + +func mustToDate(fmt, str string) (time.Time, error) { + return time.ParseInLocation(fmt, str, time.Local) +} + +func unixEpoch(date time.Time) string { + return strconv.FormatInt(date.Unix(), 10) +} diff --git a/vendor/github.com/go-task/slim-sprig/defaults.go b/vendor/github.com/go-task/slim-sprig/defaults.go new file mode 100644 index 000000000..b9f979666 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/defaults.go @@ -0,0 +1,163 @@ +package sprig + +import ( + "bytes" + "encoding/json" + "math/rand" + "reflect" + "strings" + "time" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// dfault checks whether `given` is set, and returns default if not set. +// +// This returns `d` if `given` appears not to be set, and `given` otherwise. +// +// For numeric types 0 is unset. +// For strings, maps, arrays, and slices, len() = 0 is considered unset. +// For bool, false is unset. +// Structs are never considered unset. +// +// For everything else, including pointers, a nil value is unset. +func dfault(d interface{}, given ...interface{}) interface{} { + + if empty(given) || empty(given[0]) { + return d + } + return given[0] +} + +// empty returns true if the given value has the zero value for its type. +func empty(given interface{}) bool { + g := reflect.ValueOf(given) + if !g.IsValid() { + return true + } + + // Basically adapted from text/template.isTrue + switch g.Kind() { + default: + return g.IsNil() + case reflect.Array, reflect.Slice, reflect.Map, reflect.String: + return g.Len() == 0 + case reflect.Bool: + return !g.Bool() + case reflect.Complex64, reflect.Complex128: + return g.Complex() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return g.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return g.Uint() == 0 + case reflect.Float32, reflect.Float64: + return g.Float() == 0 + case reflect.Struct: + return false + } +} + +// coalesce returns the first non-empty value. +func coalesce(v ...interface{}) interface{} { + for _, val := range v { + if !empty(val) { + return val + } + } + return nil +} + +// all returns true if empty(x) is false for all values x in the list. +// If the list is empty, return true. +func all(v ...interface{}) bool { + for _, val := range v { + if empty(val) { + return false + } + } + return true +} + +// any returns true if empty(x) is false for any x in the list. +// If the list is empty, return false. +func any(v ...interface{}) bool { + for _, val := range v { + if !empty(val) { + return true + } + } + return false +} + +// fromJson decodes JSON into a structured value, ignoring errors. +func fromJson(v string) interface{} { + output, _ := mustFromJson(v) + return output +} + +// mustFromJson decodes JSON into a structured value, returning errors. +func mustFromJson(v string) (interface{}, error) { + var output interface{} + err := json.Unmarshal([]byte(v), &output) + return output, err +} + +// toJson encodes an item into a JSON string +func toJson(v interface{}) string { + output, _ := json.Marshal(v) + return string(output) +} + +func mustToJson(v interface{}) (string, error) { + output, err := json.Marshal(v) + if err != nil { + return "", err + } + return string(output), nil +} + +// toPrettyJson encodes an item into a pretty (indented) JSON string +func toPrettyJson(v interface{}) string { + output, _ := json.MarshalIndent(v, "", " ") + return string(output) +} + +func mustToPrettyJson(v interface{}) (string, error) { + output, err := json.MarshalIndent(v, "", " ") + if err != nil { + return "", err + } + return string(output), nil +} + +// toRawJson encodes an item into a JSON string with no escaping of HTML characters. +func toRawJson(v interface{}) string { + output, err := mustToRawJson(v) + if err != nil { + panic(err) + } + return string(output) +} + +// mustToRawJson encodes an item into a JSON string with no escaping of HTML characters. +func mustToRawJson(v interface{}) (string, error) { + buf := new(bytes.Buffer) + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) + err := enc.Encode(&v) + if err != nil { + return "", err + } + return strings.TrimSuffix(buf.String(), "\n"), nil +} + +// ternary returns the first value if the last value is true, otherwise returns the second value. +func ternary(vt interface{}, vf interface{}, v bool) interface{} { + if v { + return vt + } + + return vf +} diff --git a/vendor/github.com/go-task/slim-sprig/dict.go b/vendor/github.com/go-task/slim-sprig/dict.go new file mode 100644 index 000000000..77ebc61b1 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/dict.go @@ -0,0 +1,118 @@ +package sprig + +func get(d map[string]interface{}, key string) interface{} { + if val, ok := d[key]; ok { + return val + } + return "" +} + +func set(d map[string]interface{}, key string, value interface{}) map[string]interface{} { + d[key] = value + return d +} + +func unset(d map[string]interface{}, key string) map[string]interface{} { + delete(d, key) + return d +} + +func hasKey(d map[string]interface{}, key string) bool { + _, ok := d[key] + return ok +} + +func pluck(key string, d ...map[string]interface{}) []interface{} { + res := []interface{}{} + for _, dict := range d { + if val, ok := dict[key]; ok { + res = append(res, val) + } + } + return res +} + +func keys(dicts ...map[string]interface{}) []string { + k := []string{} + for _, dict := range dicts { + for key := range dict { + k = append(k, key) + } + } + return k +} + +func pick(dict map[string]interface{}, keys ...string) map[string]interface{} { + res := map[string]interface{}{} + for _, k := range keys { + if v, ok := dict[k]; ok { + res[k] = v + } + } + return res +} + +func omit(dict map[string]interface{}, keys ...string) map[string]interface{} { + res := map[string]interface{}{} + + omit := make(map[string]bool, len(keys)) + for _, k := range keys { + omit[k] = true + } + + for k, v := range dict { + if _, ok := omit[k]; !ok { + res[k] = v + } + } + return res +} + +func dict(v ...interface{}) map[string]interface{} { + dict := map[string]interface{}{} + lenv := len(v) + for i := 0; i < lenv; i += 2 { + key := strval(v[i]) + if i+1 >= lenv { + dict[key] = "" + continue + } + dict[key] = v[i+1] + } + return dict +} + +func values(dict map[string]interface{}) []interface{} { + values := []interface{}{} + for _, value := range dict { + values = append(values, value) + } + + return values +} + +func dig(ps ...interface{}) (interface{}, error) { + if len(ps) < 3 { + panic("dig needs at least three arguments") + } + dict := ps[len(ps)-1].(map[string]interface{}) + def := ps[len(ps)-2] + ks := make([]string, len(ps)-2) + for i := 0; i < len(ks); i++ { + ks[i] = ps[i].(string) + } + + return digFromDict(dict, def, ks) +} + +func digFromDict(dict map[string]interface{}, d interface{}, ks []string) (interface{}, error) { + k, ns := ks[0], ks[1:len(ks)] + step, has := dict[k] + if !has { + return d, nil + } + if len(ns) == 0 { + return step, nil + } + return digFromDict(step.(map[string]interface{}), d, ns) +} diff --git a/vendor/github.com/go-task/slim-sprig/doc.go b/vendor/github.com/go-task/slim-sprig/doc.go new file mode 100644 index 000000000..aabb9d448 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/doc.go @@ -0,0 +1,19 @@ +/* +Package sprig provides template functions for Go. + +This package contains a number of utility functions for working with data +inside of Go `html/template` and `text/template` files. + +To add these functions, use the `template.Funcs()` method: + + t := templates.New("foo").Funcs(sprig.FuncMap()) + +Note that you should add the function map before you parse any template files. + + In several cases, Sprig reverses the order of arguments from the way they + appear in the standard library. This is to make it easier to pipe + arguments into functions. + +See http://masterminds.github.io/sprig/ for more detailed documentation on each of the available functions. +*/ +package sprig diff --git a/vendor/github.com/go-task/slim-sprig/functions.go b/vendor/github.com/go-task/slim-sprig/functions.go new file mode 100644 index 000000000..5ea74f899 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/functions.go @@ -0,0 +1,317 @@ +package sprig + +import ( + "errors" + "html/template" + "math/rand" + "os" + "path" + "path/filepath" + "reflect" + "strconv" + "strings" + ttemplate "text/template" + "time" +) + +// FuncMap produces the function map. +// +// Use this to pass the functions into the template engine: +// +// tpl := template.New("foo").Funcs(sprig.FuncMap())) +// +func FuncMap() template.FuncMap { + return HtmlFuncMap() +} + +// HermeticTxtFuncMap returns a 'text/template'.FuncMap with only repeatable functions. +func HermeticTxtFuncMap() ttemplate.FuncMap { + r := TxtFuncMap() + for _, name := range nonhermeticFunctions { + delete(r, name) + } + return r +} + +// HermeticHtmlFuncMap returns an 'html/template'.Funcmap with only repeatable functions. +func HermeticHtmlFuncMap() template.FuncMap { + r := HtmlFuncMap() + for _, name := range nonhermeticFunctions { + delete(r, name) + } + return r +} + +// TxtFuncMap returns a 'text/template'.FuncMap +func TxtFuncMap() ttemplate.FuncMap { + return ttemplate.FuncMap(GenericFuncMap()) +} + +// HtmlFuncMap returns an 'html/template'.Funcmap +func HtmlFuncMap() template.FuncMap { + return template.FuncMap(GenericFuncMap()) +} + +// GenericFuncMap returns a copy of the basic function map as a map[string]interface{}. +func GenericFuncMap() map[string]interface{} { + gfm := make(map[string]interface{}, len(genericMap)) + for k, v := range genericMap { + gfm[k] = v + } + return gfm +} + +// These functions are not guaranteed to evaluate to the same result for given input, because they +// refer to the environment or global state. +var nonhermeticFunctions = []string{ + // Date functions + "date", + "date_in_zone", + "date_modify", + "now", + "htmlDate", + "htmlDateInZone", + "dateInZone", + "dateModify", + + // Strings + "randAlphaNum", + "randAlpha", + "randAscii", + "randNumeric", + "randBytes", + "uuidv4", + + // OS + "env", + "expandenv", + + // Network + "getHostByName", +} + +var genericMap = map[string]interface{}{ + "hello": func() string { return "Hello!" }, + + // Date functions + "ago": dateAgo, + "date": date, + "date_in_zone": dateInZone, + "date_modify": dateModify, + "dateInZone": dateInZone, + "dateModify": dateModify, + "duration": duration, + "durationRound": durationRound, + "htmlDate": htmlDate, + "htmlDateInZone": htmlDateInZone, + "must_date_modify": mustDateModify, + "mustDateModify": mustDateModify, + "mustToDate": mustToDate, + "now": time.Now, + "toDate": toDate, + "unixEpoch": unixEpoch, + + // Strings + "trunc": trunc, + "trim": strings.TrimSpace, + "upper": strings.ToUpper, + "lower": strings.ToLower, + "title": strings.Title, + "substr": substring, + // Switch order so that "foo" | repeat 5 + "repeat": func(count int, str string) string { return strings.Repeat(str, count) }, + // Deprecated: Use trimAll. + "trimall": func(a, b string) string { return strings.Trim(b, a) }, + // Switch order so that "$foo" | trimall "$" + "trimAll": func(a, b string) string { return strings.Trim(b, a) }, + "trimSuffix": func(a, b string) string { return strings.TrimSuffix(b, a) }, + "trimPrefix": func(a, b string) string { return strings.TrimPrefix(b, a) }, + // Switch order so that "foobar" | contains "foo" + "contains": func(substr string, str string) bool { return strings.Contains(str, substr) }, + "hasPrefix": func(substr string, str string) bool { return strings.HasPrefix(str, substr) }, + "hasSuffix": func(substr string, str string) bool { return strings.HasSuffix(str, substr) }, + "quote": quote, + "squote": squote, + "cat": cat, + "indent": indent, + "nindent": nindent, + "replace": replace, + "plural": plural, + "sha1sum": sha1sum, + "sha256sum": sha256sum, + "adler32sum": adler32sum, + "toString": strval, + + // Wrap Atoi to stop errors. + "atoi": func(a string) int { i, _ := strconv.Atoi(a); return i }, + "int64": toInt64, + "int": toInt, + "float64": toFloat64, + "seq": seq, + "toDecimal": toDecimal, + + //"gt": func(a, b int) bool {return a > b}, + //"gte": func(a, b int) bool {return a >= b}, + //"lt": func(a, b int) bool {return a < b}, + //"lte": func(a, b int) bool {return a <= b}, + + // split "/" foo/bar returns map[int]string{0: foo, 1: bar} + "split": split, + "splitList": func(sep, orig string) []string { return strings.Split(orig, sep) }, + // splitn "/" foo/bar/fuu returns map[int]string{0: foo, 1: bar/fuu} + "splitn": splitn, + "toStrings": strslice, + + "until": until, + "untilStep": untilStep, + + // VERY basic arithmetic. + "add1": func(i interface{}) int64 { return toInt64(i) + 1 }, + "add": func(i ...interface{}) int64 { + var a int64 = 0 + for _, b := range i { + a += toInt64(b) + } + return a + }, + "sub": func(a, b interface{}) int64 { return toInt64(a) - toInt64(b) }, + "div": func(a, b interface{}) int64 { return toInt64(a) / toInt64(b) }, + "mod": func(a, b interface{}) int64 { return toInt64(a) % toInt64(b) }, + "mul": func(a interface{}, v ...interface{}) int64 { + val := toInt64(a) + for _, b := range v { + val = val * toInt64(b) + } + return val + }, + "randInt": func(min, max int) int { return rand.Intn(max-min) + min }, + "biggest": max, + "max": max, + "min": min, + "maxf": maxf, + "minf": minf, + "ceil": ceil, + "floor": floor, + "round": round, + + // string slices. Note that we reverse the order b/c that's better + // for template processing. + "join": join, + "sortAlpha": sortAlpha, + + // Defaults + "default": dfault, + "empty": empty, + "coalesce": coalesce, + "all": all, + "any": any, + "compact": compact, + "mustCompact": mustCompact, + "fromJson": fromJson, + "toJson": toJson, + "toPrettyJson": toPrettyJson, + "toRawJson": toRawJson, + "mustFromJson": mustFromJson, + "mustToJson": mustToJson, + "mustToPrettyJson": mustToPrettyJson, + "mustToRawJson": mustToRawJson, + "ternary": ternary, + + // Reflection + "typeOf": typeOf, + "typeIs": typeIs, + "typeIsLike": typeIsLike, + "kindOf": kindOf, + "kindIs": kindIs, + "deepEqual": reflect.DeepEqual, + + // OS: + "env": os.Getenv, + "expandenv": os.ExpandEnv, + + // Network: + "getHostByName": getHostByName, + + // Paths: + "base": path.Base, + "dir": path.Dir, + "clean": path.Clean, + "ext": path.Ext, + "isAbs": path.IsAbs, + + // Filepaths: + "osBase": filepath.Base, + "osClean": filepath.Clean, + "osDir": filepath.Dir, + "osExt": filepath.Ext, + "osIsAbs": filepath.IsAbs, + + // Encoding: + "b64enc": base64encode, + "b64dec": base64decode, + "b32enc": base32encode, + "b32dec": base32decode, + + // Data Structures: + "tuple": list, // FIXME: with the addition of append/prepend these are no longer immutable. + "list": list, + "dict": dict, + "get": get, + "set": set, + "unset": unset, + "hasKey": hasKey, + "pluck": pluck, + "keys": keys, + "pick": pick, + "omit": omit, + "values": values, + + "append": push, "push": push, + "mustAppend": mustPush, "mustPush": mustPush, + "prepend": prepend, + "mustPrepend": mustPrepend, + "first": first, + "mustFirst": mustFirst, + "rest": rest, + "mustRest": mustRest, + "last": last, + "mustLast": mustLast, + "initial": initial, + "mustInitial": mustInitial, + "reverse": reverse, + "mustReverse": mustReverse, + "uniq": uniq, + "mustUniq": mustUniq, + "without": without, + "mustWithout": mustWithout, + "has": has, + "mustHas": mustHas, + "slice": slice, + "mustSlice": mustSlice, + "concat": concat, + "dig": dig, + "chunk": chunk, + "mustChunk": mustChunk, + + // Flow Control: + "fail": func(msg string) (string, error) { return "", errors.New(msg) }, + + // Regex + "regexMatch": regexMatch, + "mustRegexMatch": mustRegexMatch, + "regexFindAll": regexFindAll, + "mustRegexFindAll": mustRegexFindAll, + "regexFind": regexFind, + "mustRegexFind": mustRegexFind, + "regexReplaceAll": regexReplaceAll, + "mustRegexReplaceAll": mustRegexReplaceAll, + "regexReplaceAllLiteral": regexReplaceAllLiteral, + "mustRegexReplaceAllLiteral": mustRegexReplaceAllLiteral, + "regexSplit": regexSplit, + "mustRegexSplit": mustRegexSplit, + "regexQuoteMeta": regexQuoteMeta, + + // URLs: + "urlParse": urlParse, + "urlJoin": urlJoin, +} diff --git a/vendor/github.com/go-task/slim-sprig/list.go b/vendor/github.com/go-task/slim-sprig/list.go new file mode 100644 index 000000000..ca0fbb789 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/list.go @@ -0,0 +1,464 @@ +package sprig + +import ( + "fmt" + "math" + "reflect" + "sort" +) + +// Reflection is used in these functions so that slices and arrays of strings, +// ints, and other types not implementing []interface{} can be worked with. +// For example, this is useful if you need to work on the output of regexs. + +func list(v ...interface{}) []interface{} { + return v +} + +func push(list interface{}, v interface{}) []interface{} { + l, err := mustPush(list, v) + if err != nil { + panic(err) + } + + return l +} + +func mustPush(list interface{}, v interface{}) ([]interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + nl := make([]interface{}, l) + for i := 0; i < l; i++ { + nl[i] = l2.Index(i).Interface() + } + + return append(nl, v), nil + + default: + return nil, fmt.Errorf("Cannot push on type %s", tp) + } +} + +func prepend(list interface{}, v interface{}) []interface{} { + l, err := mustPrepend(list, v) + if err != nil { + panic(err) + } + + return l +} + +func mustPrepend(list interface{}, v interface{}) ([]interface{}, error) { + //return append([]interface{}{v}, list...) + + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + nl := make([]interface{}, l) + for i := 0; i < l; i++ { + nl[i] = l2.Index(i).Interface() + } + + return append([]interface{}{v}, nl...), nil + + default: + return nil, fmt.Errorf("Cannot prepend on type %s", tp) + } +} + +func chunk(size int, list interface{}) [][]interface{} { + l, err := mustChunk(size, list) + if err != nil { + panic(err) + } + + return l +} + +func mustChunk(size int, list interface{}) ([][]interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + + cs := int(math.Floor(float64(l-1)/float64(size)) + 1) + nl := make([][]interface{}, cs) + + for i := 0; i < cs; i++ { + clen := size + if i == cs-1 { + clen = int(math.Floor(math.Mod(float64(l), float64(size)))) + if clen == 0 { + clen = size + } + } + + nl[i] = make([]interface{}, clen) + + for j := 0; j < clen; j++ { + ix := i*size + j + nl[i][j] = l2.Index(ix).Interface() + } + } + + return nl, nil + + default: + return nil, fmt.Errorf("Cannot chunk type %s", tp) + } +} + +func last(list interface{}) interface{} { + l, err := mustLast(list) + if err != nil { + panic(err) + } + + return l +} + +func mustLast(list interface{}) (interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil, nil + } + + return l2.Index(l - 1).Interface(), nil + default: + return nil, fmt.Errorf("Cannot find last on type %s", tp) + } +} + +func first(list interface{}) interface{} { + l, err := mustFirst(list) + if err != nil { + panic(err) + } + + return l +} + +func mustFirst(list interface{}) (interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil, nil + } + + return l2.Index(0).Interface(), nil + default: + return nil, fmt.Errorf("Cannot find first on type %s", tp) + } +} + +func rest(list interface{}) []interface{} { + l, err := mustRest(list) + if err != nil { + panic(err) + } + + return l +} + +func mustRest(list interface{}) ([]interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil, nil + } + + nl := make([]interface{}, l-1) + for i := 1; i < l; i++ { + nl[i-1] = l2.Index(i).Interface() + } + + return nl, nil + default: + return nil, fmt.Errorf("Cannot find rest on type %s", tp) + } +} + +func initial(list interface{}) []interface{} { + l, err := mustInitial(list) + if err != nil { + panic(err) + } + + return l +} + +func mustInitial(list interface{}) ([]interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil, nil + } + + nl := make([]interface{}, l-1) + for i := 0; i < l-1; i++ { + nl[i] = l2.Index(i).Interface() + } + + return nl, nil + default: + return nil, fmt.Errorf("Cannot find initial on type %s", tp) + } +} + +func sortAlpha(list interface{}) []string { + k := reflect.Indirect(reflect.ValueOf(list)).Kind() + switch k { + case reflect.Slice, reflect.Array: + a := strslice(list) + s := sort.StringSlice(a) + s.Sort() + return s + } + return []string{strval(list)} +} + +func reverse(v interface{}) []interface{} { + l, err := mustReverse(v) + if err != nil { + panic(err) + } + + return l +} + +func mustReverse(v interface{}) ([]interface{}, error) { + tp := reflect.TypeOf(v).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(v) + + l := l2.Len() + // We do not sort in place because the incoming array should not be altered. + nl := make([]interface{}, l) + for i := 0; i < l; i++ { + nl[l-i-1] = l2.Index(i).Interface() + } + + return nl, nil + default: + return nil, fmt.Errorf("Cannot find reverse on type %s", tp) + } +} + +func compact(list interface{}) []interface{} { + l, err := mustCompact(list) + if err != nil { + panic(err) + } + + return l +} + +func mustCompact(list interface{}) ([]interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + nl := []interface{}{} + var item interface{} + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if !empty(item) { + nl = append(nl, item) + } + } + + return nl, nil + default: + return nil, fmt.Errorf("Cannot compact on type %s", tp) + } +} + +func uniq(list interface{}) []interface{} { + l, err := mustUniq(list) + if err != nil { + panic(err) + } + + return l +} + +func mustUniq(list interface{}) ([]interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + dest := []interface{}{} + var item interface{} + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if !inList(dest, item) { + dest = append(dest, item) + } + } + + return dest, nil + default: + return nil, fmt.Errorf("Cannot find uniq on type %s", tp) + } +} + +func inList(haystack []interface{}, needle interface{}) bool { + for _, h := range haystack { + if reflect.DeepEqual(needle, h) { + return true + } + } + return false +} + +func without(list interface{}, omit ...interface{}) []interface{} { + l, err := mustWithout(list, omit...) + if err != nil { + panic(err) + } + + return l +} + +func mustWithout(list interface{}, omit ...interface{}) ([]interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + res := []interface{}{} + var item interface{} + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if !inList(omit, item) { + res = append(res, item) + } + } + + return res, nil + default: + return nil, fmt.Errorf("Cannot find without on type %s", tp) + } +} + +func has(needle interface{}, haystack interface{}) bool { + l, err := mustHas(needle, haystack) + if err != nil { + panic(err) + } + + return l +} + +func mustHas(needle interface{}, haystack interface{}) (bool, error) { + if haystack == nil { + return false, nil + } + tp := reflect.TypeOf(haystack).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(haystack) + var item interface{} + l := l2.Len() + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if reflect.DeepEqual(needle, item) { + return true, nil + } + } + + return false, nil + default: + return false, fmt.Errorf("Cannot find has on type %s", tp) + } +} + +// $list := [1, 2, 3, 4, 5] +// slice $list -> list[0:5] = list[:] +// slice $list 0 3 -> list[0:3] = list[:3] +// slice $list 3 5 -> list[3:5] +// slice $list 3 -> list[3:5] = list[3:] +func slice(list interface{}, indices ...interface{}) interface{} { + l, err := mustSlice(list, indices...) + if err != nil { + panic(err) + } + + return l +} + +func mustSlice(list interface{}, indices ...interface{}) (interface{}, error) { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil, nil + } + + var start, end int + if len(indices) > 0 { + start = toInt(indices[0]) + } + if len(indices) < 2 { + end = l + } else { + end = toInt(indices[1]) + } + + return l2.Slice(start, end).Interface(), nil + default: + return nil, fmt.Errorf("list should be type of slice or array but %s", tp) + } +} + +func concat(lists ...interface{}) interface{} { + var res []interface{} + for _, list := range lists { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + for i := 0; i < l2.Len(); i++ { + res = append(res, l2.Index(i).Interface()) + } + default: + panic(fmt.Sprintf("Cannot concat type %s as list", tp)) + } + } + return res +} diff --git a/vendor/github.com/go-task/slim-sprig/network.go b/vendor/github.com/go-task/slim-sprig/network.go new file mode 100644 index 000000000..108d78a94 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/network.go @@ -0,0 +1,12 @@ +package sprig + +import ( + "math/rand" + "net" +) + +func getHostByName(name string) string { + addrs, _ := net.LookupHost(name) + //TODO: add error handing when release v3 comes out + return addrs[rand.Intn(len(addrs))] +} diff --git a/vendor/github.com/go-task/slim-sprig/numeric.go b/vendor/github.com/go-task/slim-sprig/numeric.go new file mode 100644 index 000000000..98cbb37a1 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/numeric.go @@ -0,0 +1,228 @@ +package sprig + +import ( + "fmt" + "math" + "reflect" + "strconv" + "strings" +) + +// toFloat64 converts 64-bit floats +func toFloat64(v interface{}) float64 { + if str, ok := v.(string); ok { + iv, err := strconv.ParseFloat(str, 64) + if err != nil { + return 0 + } + return iv + } + + val := reflect.Indirect(reflect.ValueOf(v)) + switch val.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + return float64(val.Int()) + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + return float64(val.Uint()) + case reflect.Uint, reflect.Uint64: + return float64(val.Uint()) + case reflect.Float32, reflect.Float64: + return val.Float() + case reflect.Bool: + if val.Bool() { + return 1 + } + return 0 + default: + return 0 + } +} + +func toInt(v interface{}) int { + //It's not optimal. Bud I don't want duplicate toInt64 code. + return int(toInt64(v)) +} + +// toInt64 converts integer types to 64-bit integers +func toInt64(v interface{}) int64 { + if str, ok := v.(string); ok { + iv, err := strconv.ParseInt(str, 10, 64) + if err != nil { + return 0 + } + return iv + } + + val := reflect.Indirect(reflect.ValueOf(v)) + switch val.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + return val.Int() + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + return int64(val.Uint()) + case reflect.Uint, reflect.Uint64: + tv := val.Uint() + if tv <= math.MaxInt64 { + return int64(tv) + } + // TODO: What is the sensible thing to do here? + return math.MaxInt64 + case reflect.Float32, reflect.Float64: + return int64(val.Float()) + case reflect.Bool: + if val.Bool() { + return 1 + } + return 0 + default: + return 0 + } +} + +func max(a interface{}, i ...interface{}) int64 { + aa := toInt64(a) + for _, b := range i { + bb := toInt64(b) + if bb > aa { + aa = bb + } + } + return aa +} + +func maxf(a interface{}, i ...interface{}) float64 { + aa := toFloat64(a) + for _, b := range i { + bb := toFloat64(b) + aa = math.Max(aa, bb) + } + return aa +} + +func min(a interface{}, i ...interface{}) int64 { + aa := toInt64(a) + for _, b := range i { + bb := toInt64(b) + if bb < aa { + aa = bb + } + } + return aa +} + +func minf(a interface{}, i ...interface{}) float64 { + aa := toFloat64(a) + for _, b := range i { + bb := toFloat64(b) + aa = math.Min(aa, bb) + } + return aa +} + +func until(count int) []int { + step := 1 + if count < 0 { + step = -1 + } + return untilStep(0, count, step) +} + +func untilStep(start, stop, step int) []int { + v := []int{} + + if stop < start { + if step >= 0 { + return v + } + for i := start; i > stop; i += step { + v = append(v, i) + } + return v + } + + if step <= 0 { + return v + } + for i := start; i < stop; i += step { + v = append(v, i) + } + return v +} + +func floor(a interface{}) float64 { + aa := toFloat64(a) + return math.Floor(aa) +} + +func ceil(a interface{}) float64 { + aa := toFloat64(a) + return math.Ceil(aa) +} + +func round(a interface{}, p int, rOpt ...float64) float64 { + roundOn := .5 + if len(rOpt) > 0 { + roundOn = rOpt[0] + } + val := toFloat64(a) + places := toFloat64(p) + + var round float64 + pow := math.Pow(10, places) + digit := pow * val + _, div := math.Modf(digit) + if div >= roundOn { + round = math.Ceil(digit) + } else { + round = math.Floor(digit) + } + return round / pow +} + +// converts unix octal to decimal +func toDecimal(v interface{}) int64 { + result, err := strconv.ParseInt(fmt.Sprint(v), 8, 64) + if err != nil { + return 0 + } + return result +} + +func seq(params ...int) string { + increment := 1 + switch len(params) { + case 0: + return "" + case 1: + start := 1 + end := params[0] + if end < start { + increment = -1 + } + return intArrayToString(untilStep(start, end+increment, increment), " ") + case 3: + start := params[0] + end := params[2] + step := params[1] + if end < start { + increment = -1 + if step > 0 { + return "" + } + } + return intArrayToString(untilStep(start, end+increment, step), " ") + case 2: + start := params[0] + end := params[1] + step := 1 + if end < start { + step = -1 + } + return intArrayToString(untilStep(start, end+step, step), " ") + default: + return "" + } +} + +func intArrayToString(slice []int, delimeter string) string { + return strings.Trim(strings.Join(strings.Fields(fmt.Sprint(slice)), delimeter), "[]") +} diff --git a/vendor/github.com/go-task/slim-sprig/reflect.go b/vendor/github.com/go-task/slim-sprig/reflect.go new file mode 100644 index 000000000..8a65c132f --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/reflect.go @@ -0,0 +1,28 @@ +package sprig + +import ( + "fmt" + "reflect" +) + +// typeIs returns true if the src is the type named in target. +func typeIs(target string, src interface{}) bool { + return target == typeOf(src) +} + +func typeIsLike(target string, src interface{}) bool { + t := typeOf(src) + return target == t || "*"+target == t +} + +func typeOf(src interface{}) string { + return fmt.Sprintf("%T", src) +} + +func kindIs(target string, src interface{}) bool { + return target == kindOf(src) +} + +func kindOf(src interface{}) string { + return reflect.ValueOf(src).Kind().String() +} diff --git a/vendor/github.com/go-task/slim-sprig/regex.go b/vendor/github.com/go-task/slim-sprig/regex.go new file mode 100644 index 000000000..fab551018 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/regex.go @@ -0,0 +1,83 @@ +package sprig + +import ( + "regexp" +) + +func regexMatch(regex string, s string) bool { + match, _ := regexp.MatchString(regex, s) + return match +} + +func mustRegexMatch(regex string, s string) (bool, error) { + return regexp.MatchString(regex, s) +} + +func regexFindAll(regex string, s string, n int) []string { + r := regexp.MustCompile(regex) + return r.FindAllString(s, n) +} + +func mustRegexFindAll(regex string, s string, n int) ([]string, error) { + r, err := regexp.Compile(regex) + if err != nil { + return []string{}, err + } + return r.FindAllString(s, n), nil +} + +func regexFind(regex string, s string) string { + r := regexp.MustCompile(regex) + return r.FindString(s) +} + +func mustRegexFind(regex string, s string) (string, error) { + r, err := regexp.Compile(regex) + if err != nil { + return "", err + } + return r.FindString(s), nil +} + +func regexReplaceAll(regex string, s string, repl string) string { + r := regexp.MustCompile(regex) + return r.ReplaceAllString(s, repl) +} + +func mustRegexReplaceAll(regex string, s string, repl string) (string, error) { + r, err := regexp.Compile(regex) + if err != nil { + return "", err + } + return r.ReplaceAllString(s, repl), nil +} + +func regexReplaceAllLiteral(regex string, s string, repl string) string { + r := regexp.MustCompile(regex) + return r.ReplaceAllLiteralString(s, repl) +} + +func mustRegexReplaceAllLiteral(regex string, s string, repl string) (string, error) { + r, err := regexp.Compile(regex) + if err != nil { + return "", err + } + return r.ReplaceAllLiteralString(s, repl), nil +} + +func regexSplit(regex string, s string, n int) []string { + r := regexp.MustCompile(regex) + return r.Split(s, n) +} + +func mustRegexSplit(regex string, s string, n int) ([]string, error) { + r, err := regexp.Compile(regex) + if err != nil { + return []string{}, err + } + return r.Split(s, n), nil +} + +func regexQuoteMeta(s string) string { + return regexp.QuoteMeta(s) +} diff --git a/vendor/github.com/go-task/slim-sprig/strings.go b/vendor/github.com/go-task/slim-sprig/strings.go new file mode 100644 index 000000000..3c62d6b6f --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/strings.go @@ -0,0 +1,189 @@ +package sprig + +import ( + "encoding/base32" + "encoding/base64" + "fmt" + "reflect" + "strconv" + "strings" +) + +func base64encode(v string) string { + return base64.StdEncoding.EncodeToString([]byte(v)) +} + +func base64decode(v string) string { + data, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return err.Error() + } + return string(data) +} + +func base32encode(v string) string { + return base32.StdEncoding.EncodeToString([]byte(v)) +} + +func base32decode(v string) string { + data, err := base32.StdEncoding.DecodeString(v) + if err != nil { + return err.Error() + } + return string(data) +} + +func quote(str ...interface{}) string { + out := make([]string, 0, len(str)) + for _, s := range str { + if s != nil { + out = append(out, fmt.Sprintf("%q", strval(s))) + } + } + return strings.Join(out, " ") +} + +func squote(str ...interface{}) string { + out := make([]string, 0, len(str)) + for _, s := range str { + if s != nil { + out = append(out, fmt.Sprintf("'%v'", s)) + } + } + return strings.Join(out, " ") +} + +func cat(v ...interface{}) string { + v = removeNilElements(v) + r := strings.TrimSpace(strings.Repeat("%v ", len(v))) + return fmt.Sprintf(r, v...) +} + +func indent(spaces int, v string) string { + pad := strings.Repeat(" ", spaces) + return pad + strings.Replace(v, "\n", "\n"+pad, -1) +} + +func nindent(spaces int, v string) string { + return "\n" + indent(spaces, v) +} + +func replace(old, new, src string) string { + return strings.Replace(src, old, new, -1) +} + +func plural(one, many string, count int) string { + if count == 1 { + return one + } + return many +} + +func strslice(v interface{}) []string { + switch v := v.(type) { + case []string: + return v + case []interface{}: + b := make([]string, 0, len(v)) + for _, s := range v { + if s != nil { + b = append(b, strval(s)) + } + } + return b + default: + val := reflect.ValueOf(v) + switch val.Kind() { + case reflect.Array, reflect.Slice: + l := val.Len() + b := make([]string, 0, l) + for i := 0; i < l; i++ { + value := val.Index(i).Interface() + if value != nil { + b = append(b, strval(value)) + } + } + return b + default: + if v == nil { + return []string{} + } + + return []string{strval(v)} + } + } +} + +func removeNilElements(v []interface{}) []interface{} { + newSlice := make([]interface{}, 0, len(v)) + for _, i := range v { + if i != nil { + newSlice = append(newSlice, i) + } + } + return newSlice +} + +func strval(v interface{}) string { + switch v := v.(type) { + case string: + return v + case []byte: + return string(v) + case error: + return v.Error() + case fmt.Stringer: + return v.String() + default: + return fmt.Sprintf("%v", v) + } +} + +func trunc(c int, s string) string { + if c < 0 && len(s)+c > 0 { + return s[len(s)+c:] + } + if c >= 0 && len(s) > c { + return s[:c] + } + return s +} + +func join(sep string, v interface{}) string { + return strings.Join(strslice(v), sep) +} + +func split(sep, orig string) map[string]string { + parts := strings.Split(orig, sep) + res := make(map[string]string, len(parts)) + for i, v := range parts { + res["_"+strconv.Itoa(i)] = v + } + return res +} + +func splitn(sep string, n int, orig string) map[string]string { + parts := strings.SplitN(orig, sep, n) + res := make(map[string]string, len(parts)) + for i, v := range parts { + res["_"+strconv.Itoa(i)] = v + } + return res +} + +// substring creates a substring of the given string. +// +// If start is < 0, this calls string[:end]. +// +// If start is >= 0 and end < 0 or end bigger than s length, this calls string[start:] +// +// Otherwise, this calls string[start, end]. +func substring(start, end int, s string) string { + if start < 0 { + return s[:end] + } + if end < 0 || end > len(s) { + return s[start:] + } + return s[start:end] +} diff --git a/vendor/github.com/go-task/slim-sprig/url.go b/vendor/github.com/go-task/slim-sprig/url.go new file mode 100644 index 000000000..b8e120e19 --- /dev/null +++ b/vendor/github.com/go-task/slim-sprig/url.go @@ -0,0 +1,66 @@ +package sprig + +import ( + "fmt" + "net/url" + "reflect" +) + +func dictGetOrEmpty(dict map[string]interface{}, key string) string { + value, ok := dict[key] + if !ok { + return "" + } + tp := reflect.TypeOf(value).Kind() + if tp != reflect.String { + panic(fmt.Sprintf("unable to parse %s key, must be of type string, but %s found", key, tp.String())) + } + return reflect.ValueOf(value).String() +} + +// parses given URL to return dict object +func urlParse(v string) map[string]interface{} { + dict := map[string]interface{}{} + parsedURL, err := url.Parse(v) + if err != nil { + panic(fmt.Sprintf("unable to parse url: %s", err)) + } + dict["scheme"] = parsedURL.Scheme + dict["host"] = parsedURL.Host + dict["hostname"] = parsedURL.Hostname() + dict["path"] = parsedURL.Path + dict["query"] = parsedURL.RawQuery + dict["opaque"] = parsedURL.Opaque + dict["fragment"] = parsedURL.Fragment + if parsedURL.User != nil { + dict["userinfo"] = parsedURL.User.String() + } else { + dict["userinfo"] = "" + } + + return dict +} + +// join given dict to URL string +func urlJoin(d map[string]interface{}) string { + resURL := url.URL{ + Scheme: dictGetOrEmpty(d, "scheme"), + Host: dictGetOrEmpty(d, "host"), + Path: dictGetOrEmpty(d, "path"), + RawQuery: dictGetOrEmpty(d, "query"), + Opaque: dictGetOrEmpty(d, "opaque"), + Fragment: dictGetOrEmpty(d, "fragment"), + } + userinfo := dictGetOrEmpty(d, "userinfo") + var user *url.Userinfo + if userinfo != "" { + tempURL, err := url.Parse(fmt.Sprintf("proto://%s@host", userinfo)) + if err != nil { + panic(fmt.Sprintf("unable to parse userinfo in dict: %s", err)) + } + user = tempURL.User + } + + resURL.User = user + return resURL.String() +} diff --git a/vendor/github.com/google/flatbuffers/go/builder.go b/vendor/github.com/google/flatbuffers/go/builder.go index d99b590bb..5d90e8ef9 100644 --- a/vendor/github.com/google/flatbuffers/go/builder.go +++ b/vendor/github.com/google/flatbuffers/go/builder.go @@ -1,5 +1,7 @@ package flatbuffers +import "sort" + // Builder is a state machine for creating FlatBuffer objects. // Use a Builder to construct object(s) starting from leaf nodes. // @@ -315,6 +317,25 @@ func (b *Builder) EndVector(vectorNumElems int) UOffsetT { return b.Offset() } +// CreateVectorOfTables serializes slice of table offsets into a vector. +func (b *Builder) CreateVectorOfTables(offsets []UOffsetT) UOffsetT { + b.assertNotNested() + b.StartVector(4, len(offsets), 4) + for i := len(offsets) - 1; i >= 0; i-- { + b.PrependUOffsetT(offsets[i]) + } + return b.EndVector(len(offsets)) +} + +type KeyCompare func(o1, o2 UOffsetT, buf []byte) bool + +func (b *Builder) CreateVectorOfSortedTables(offsets []UOffsetT, keyCompare KeyCompare) UOffsetT { + sort.Slice(offsets, func(i, j int) bool { + return keyCompare(offsets[i], offsets[j], b.Bytes) + }) + return b.CreateVectorOfTables(offsets) +} + // CreateSharedString Checks if the string is already written // to the buffer before calling CreateString func (b *Builder) CreateSharedString(s string) UOffsetT { diff --git a/vendor/github.com/google/flatbuffers/go/lib.go b/vendor/github.com/google/flatbuffers/go/lib.go index 9a333ff04..9333d8bd3 100644 --- a/vendor/github.com/google/flatbuffers/go/lib.go +++ b/vendor/github.com/google/flatbuffers/go/lib.go @@ -23,3 +23,8 @@ func GetSizePrefixedRootAs(buf []byte, offset UOffsetT, fb FlatBuffer) { func GetSizePrefix(buf []byte, offset UOffsetT) uint32 { return GetUint32(buf[offset:]) } + +// GetIndirectOffset retrives the relative offset in the provided buffer stored at `offset`. +func GetIndirectOffset(buf []byte, offset UOffsetT) UOffsetT { + return offset + GetUOffsetT(buf[offset:]) +} diff --git a/vendor/github.com/google/pprof/AUTHORS b/vendor/github.com/google/pprof/AUTHORS new file mode 100644 index 000000000..fd736cb1c --- /dev/null +++ b/vendor/github.com/google/pprof/AUTHORS @@ -0,0 +1,7 @@ +# This is the official list of pprof authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. +Google Inc. \ No newline at end of file diff --git a/vendor/github.com/google/pprof/CONTRIBUTORS b/vendor/github.com/google/pprof/CONTRIBUTORS new file mode 100644 index 000000000..8c8c37d2c --- /dev/null +++ b/vendor/github.com/google/pprof/CONTRIBUTORS @@ -0,0 +1,16 @@ +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# Names should be added to this file as: +# Name +Raul Silvera +Tipp Moseley +Hyoun Kyu Cho +Martin Spier +Taco de Wolff +Andrew Hunter diff --git a/vendor/github.com/google/pprof/LICENSE b/vendor/github.com/google/pprof/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/google/pprof/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/google/pprof/profile/encode.go b/vendor/github.com/google/pprof/profile/encode.go new file mode 100644 index 000000000..c8a1beb8a --- /dev/null +++ b/vendor/github.com/google/pprof/profile/encode.go @@ -0,0 +1,588 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +import ( + "errors" + "sort" + "strings" +) + +func (p *Profile) decoder() []decoder { + return profileDecoder +} + +// preEncode populates the unexported fields to be used by encode +// (with suffix X) from the corresponding exported fields. The +// exported fields are cleared up to facilitate testing. +func (p *Profile) preEncode() { + strings := make(map[string]int) + addString(strings, "") + + for _, st := range p.SampleType { + st.typeX = addString(strings, st.Type) + st.unitX = addString(strings, st.Unit) + } + + for _, s := range p.Sample { + s.labelX = nil + var keys []string + for k := range s.Label { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + vs := s.Label[k] + for _, v := range vs { + s.labelX = append(s.labelX, + label{ + keyX: addString(strings, k), + strX: addString(strings, v), + }, + ) + } + } + var numKeys []string + for k := range s.NumLabel { + numKeys = append(numKeys, k) + } + sort.Strings(numKeys) + for _, k := range numKeys { + keyX := addString(strings, k) + vs := s.NumLabel[k] + units := s.NumUnit[k] + for i, v := range vs { + var unitX int64 + if len(units) != 0 { + unitX = addString(strings, units[i]) + } + s.labelX = append(s.labelX, + label{ + keyX: keyX, + numX: v, + unitX: unitX, + }, + ) + } + } + s.locationIDX = make([]uint64, len(s.Location)) + for i, loc := range s.Location { + s.locationIDX[i] = loc.ID + } + } + + for _, m := range p.Mapping { + m.fileX = addString(strings, m.File) + m.buildIDX = addString(strings, m.BuildID) + } + + for _, l := range p.Location { + for i, ln := range l.Line { + if ln.Function != nil { + l.Line[i].functionIDX = ln.Function.ID + } else { + l.Line[i].functionIDX = 0 + } + } + if l.Mapping != nil { + l.mappingIDX = l.Mapping.ID + } else { + l.mappingIDX = 0 + } + } + for _, f := range p.Function { + f.nameX = addString(strings, f.Name) + f.systemNameX = addString(strings, f.SystemName) + f.filenameX = addString(strings, f.Filename) + } + + p.dropFramesX = addString(strings, p.DropFrames) + p.keepFramesX = addString(strings, p.KeepFrames) + + if pt := p.PeriodType; pt != nil { + pt.typeX = addString(strings, pt.Type) + pt.unitX = addString(strings, pt.Unit) + } + + p.commentX = nil + for _, c := range p.Comments { + p.commentX = append(p.commentX, addString(strings, c)) + } + + p.defaultSampleTypeX = addString(strings, p.DefaultSampleType) + + p.stringTable = make([]string, len(strings)) + for s, i := range strings { + p.stringTable[i] = s + } +} + +func (p *Profile) encode(b *buffer) { + for _, x := range p.SampleType { + encodeMessage(b, 1, x) + } + for _, x := range p.Sample { + encodeMessage(b, 2, x) + } + for _, x := range p.Mapping { + encodeMessage(b, 3, x) + } + for _, x := range p.Location { + encodeMessage(b, 4, x) + } + for _, x := range p.Function { + encodeMessage(b, 5, x) + } + encodeStrings(b, 6, p.stringTable) + encodeInt64Opt(b, 7, p.dropFramesX) + encodeInt64Opt(b, 8, p.keepFramesX) + encodeInt64Opt(b, 9, p.TimeNanos) + encodeInt64Opt(b, 10, p.DurationNanos) + if pt := p.PeriodType; pt != nil && (pt.typeX != 0 || pt.unitX != 0) { + encodeMessage(b, 11, p.PeriodType) + } + encodeInt64Opt(b, 12, p.Period) + encodeInt64s(b, 13, p.commentX) + encodeInt64(b, 14, p.defaultSampleTypeX) +} + +var profileDecoder = []decoder{ + nil, // 0 + // repeated ValueType sample_type = 1 + func(b *buffer, m message) error { + x := new(ValueType) + pp := m.(*Profile) + pp.SampleType = append(pp.SampleType, x) + return decodeMessage(b, x) + }, + // repeated Sample sample = 2 + func(b *buffer, m message) error { + x := new(Sample) + pp := m.(*Profile) + pp.Sample = append(pp.Sample, x) + return decodeMessage(b, x) + }, + // repeated Mapping mapping = 3 + func(b *buffer, m message) error { + x := new(Mapping) + pp := m.(*Profile) + pp.Mapping = append(pp.Mapping, x) + return decodeMessage(b, x) + }, + // repeated Location location = 4 + func(b *buffer, m message) error { + x := new(Location) + x.Line = b.tmpLines[:0] // Use shared space temporarily + pp := m.(*Profile) + pp.Location = append(pp.Location, x) + err := decodeMessage(b, x) + b.tmpLines = x.Line[:0] + // Copy to shrink size and detach from shared space. + x.Line = append([]Line(nil), x.Line...) + return err + }, + // repeated Function function = 5 + func(b *buffer, m message) error { + x := new(Function) + pp := m.(*Profile) + pp.Function = append(pp.Function, x) + return decodeMessage(b, x) + }, + // repeated string string_table = 6 + func(b *buffer, m message) error { + err := decodeStrings(b, &m.(*Profile).stringTable) + if err != nil { + return err + } + if m.(*Profile).stringTable[0] != "" { + return errors.New("string_table[0] must be ''") + } + return nil + }, + // int64 drop_frames = 7 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).dropFramesX) }, + // int64 keep_frames = 8 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).keepFramesX) }, + // int64 time_nanos = 9 + func(b *buffer, m message) error { + if m.(*Profile).TimeNanos != 0 { + return errConcatProfile + } + return decodeInt64(b, &m.(*Profile).TimeNanos) + }, + // int64 duration_nanos = 10 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).DurationNanos) }, + // ValueType period_type = 11 + func(b *buffer, m message) error { + x := new(ValueType) + pp := m.(*Profile) + pp.PeriodType = x + return decodeMessage(b, x) + }, + // int64 period = 12 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).Period) }, + // repeated int64 comment = 13 + func(b *buffer, m message) error { return decodeInt64s(b, &m.(*Profile).commentX) }, + // int64 defaultSampleType = 14 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).defaultSampleTypeX) }, +} + +// postDecode takes the unexported fields populated by decode (with +// suffix X) and populates the corresponding exported fields. +// The unexported fields are cleared up to facilitate testing. +func (p *Profile) postDecode() error { + var err error + mappings := make(map[uint64]*Mapping, len(p.Mapping)) + mappingIds := make([]*Mapping, len(p.Mapping)+1) + for _, m := range p.Mapping { + m.File, err = getString(p.stringTable, &m.fileX, err) + m.BuildID, err = getString(p.stringTable, &m.buildIDX, err) + if m.ID < uint64(len(mappingIds)) { + mappingIds[m.ID] = m + } else { + mappings[m.ID] = m + } + + // If this a main linux kernel mapping with a relocation symbol suffix + // ("[kernel.kallsyms]_text"), extract said suffix. + // It is fairly hacky to handle at this level, but the alternatives appear even worse. + if strings.HasPrefix(m.File, "[kernel.kallsyms]") { + m.KernelRelocationSymbol = strings.ReplaceAll(m.File, "[kernel.kallsyms]", "") + } + + } + + functions := make(map[uint64]*Function, len(p.Function)) + functionIds := make([]*Function, len(p.Function)+1) + for _, f := range p.Function { + f.Name, err = getString(p.stringTable, &f.nameX, err) + f.SystemName, err = getString(p.stringTable, &f.systemNameX, err) + f.Filename, err = getString(p.stringTable, &f.filenameX, err) + if f.ID < uint64(len(functionIds)) { + functionIds[f.ID] = f + } else { + functions[f.ID] = f + } + } + + locations := make(map[uint64]*Location, len(p.Location)) + locationIds := make([]*Location, len(p.Location)+1) + for _, l := range p.Location { + if id := l.mappingIDX; id < uint64(len(mappingIds)) { + l.Mapping = mappingIds[id] + } else { + l.Mapping = mappings[id] + } + l.mappingIDX = 0 + for i, ln := range l.Line { + if id := ln.functionIDX; id != 0 { + l.Line[i].functionIDX = 0 + if id < uint64(len(functionIds)) { + l.Line[i].Function = functionIds[id] + } else { + l.Line[i].Function = functions[id] + } + } + } + if l.ID < uint64(len(locationIds)) { + locationIds[l.ID] = l + } else { + locations[l.ID] = l + } + } + + for _, st := range p.SampleType { + st.Type, err = getString(p.stringTable, &st.typeX, err) + st.Unit, err = getString(p.stringTable, &st.unitX, err) + } + + // Pre-allocate space for all locations. + numLocations := 0 + for _, s := range p.Sample { + numLocations += len(s.locationIDX) + } + locBuffer := make([]*Location, numLocations) + + for _, s := range p.Sample { + if len(s.labelX) > 0 { + labels := make(map[string][]string, len(s.labelX)) + numLabels := make(map[string][]int64, len(s.labelX)) + numUnits := make(map[string][]string, len(s.labelX)) + for _, l := range s.labelX { + var key, value string + key, err = getString(p.stringTable, &l.keyX, err) + if l.strX != 0 { + value, err = getString(p.stringTable, &l.strX, err) + labels[key] = append(labels[key], value) + } else if l.numX != 0 || l.unitX != 0 { + numValues := numLabels[key] + units := numUnits[key] + if l.unitX != 0 { + var unit string + unit, err = getString(p.stringTable, &l.unitX, err) + units = padStringArray(units, len(numValues)) + numUnits[key] = append(units, unit) + } + numLabels[key] = append(numLabels[key], l.numX) + } + } + if len(labels) > 0 { + s.Label = labels + } + if len(numLabels) > 0 { + s.NumLabel = numLabels + for key, units := range numUnits { + if len(units) > 0 { + numUnits[key] = padStringArray(units, len(numLabels[key])) + } + } + s.NumUnit = numUnits + } + } + + s.Location = locBuffer[:len(s.locationIDX)] + locBuffer = locBuffer[len(s.locationIDX):] + for i, lid := range s.locationIDX { + if lid < uint64(len(locationIds)) { + s.Location[i] = locationIds[lid] + } else { + s.Location[i] = locations[lid] + } + } + s.locationIDX = nil + } + + p.DropFrames, err = getString(p.stringTable, &p.dropFramesX, err) + p.KeepFrames, err = getString(p.stringTable, &p.keepFramesX, err) + + if pt := p.PeriodType; pt == nil { + p.PeriodType = &ValueType{} + } + + if pt := p.PeriodType; pt != nil { + pt.Type, err = getString(p.stringTable, &pt.typeX, err) + pt.Unit, err = getString(p.stringTable, &pt.unitX, err) + } + + for _, i := range p.commentX { + var c string + c, err = getString(p.stringTable, &i, err) + p.Comments = append(p.Comments, c) + } + + p.commentX = nil + p.DefaultSampleType, err = getString(p.stringTable, &p.defaultSampleTypeX, err) + p.stringTable = nil + return err +} + +// padStringArray pads arr with enough empty strings to make arr +// length l when arr's length is less than l. +func padStringArray(arr []string, l int) []string { + if l <= len(arr) { + return arr + } + return append(arr, make([]string, l-len(arr))...) +} + +func (p *ValueType) decoder() []decoder { + return valueTypeDecoder +} + +func (p *ValueType) encode(b *buffer) { + encodeInt64Opt(b, 1, p.typeX) + encodeInt64Opt(b, 2, p.unitX) +} + +var valueTypeDecoder = []decoder{ + nil, // 0 + // optional int64 type = 1 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).typeX) }, + // optional int64 unit = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).unitX) }, +} + +func (p *Sample) decoder() []decoder { + return sampleDecoder +} + +func (p *Sample) encode(b *buffer) { + encodeUint64s(b, 1, p.locationIDX) + encodeInt64s(b, 2, p.Value) + for _, x := range p.labelX { + encodeMessage(b, 3, x) + } +} + +var sampleDecoder = []decoder{ + nil, // 0 + // repeated uint64 location = 1 + func(b *buffer, m message) error { return decodeUint64s(b, &m.(*Sample).locationIDX) }, + // repeated int64 value = 2 + func(b *buffer, m message) error { return decodeInt64s(b, &m.(*Sample).Value) }, + // repeated Label label = 3 + func(b *buffer, m message) error { + s := m.(*Sample) + n := len(s.labelX) + s.labelX = append(s.labelX, label{}) + return decodeMessage(b, &s.labelX[n]) + }, +} + +func (p label) decoder() []decoder { + return labelDecoder +} + +func (p label) encode(b *buffer) { + encodeInt64Opt(b, 1, p.keyX) + encodeInt64Opt(b, 2, p.strX) + encodeInt64Opt(b, 3, p.numX) + encodeInt64Opt(b, 4, p.unitX) +} + +var labelDecoder = []decoder{ + nil, // 0 + // optional int64 key = 1 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*label).keyX) }, + // optional int64 str = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*label).strX) }, + // optional int64 num = 3 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*label).numX) }, + // optional int64 num = 4 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*label).unitX) }, +} + +func (p *Mapping) decoder() []decoder { + return mappingDecoder +} + +func (p *Mapping) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeUint64Opt(b, 2, p.Start) + encodeUint64Opt(b, 3, p.Limit) + encodeUint64Opt(b, 4, p.Offset) + encodeInt64Opt(b, 5, p.fileX) + encodeInt64Opt(b, 6, p.buildIDX) + encodeBoolOpt(b, 7, p.HasFunctions) + encodeBoolOpt(b, 8, p.HasFilenames) + encodeBoolOpt(b, 9, p.HasLineNumbers) + encodeBoolOpt(b, 10, p.HasInlineFrames) +} + +var mappingDecoder = []decoder{ + nil, // 0 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).ID) }, // optional uint64 id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Start) }, // optional uint64 memory_offset = 2 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Limit) }, // optional uint64 memory_limit = 3 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Offset) }, // optional uint64 file_offset = 4 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).fileX) }, // optional int64 filename = 5 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).buildIDX) }, // optional int64 build_id = 6 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFunctions) }, // optional bool has_functions = 7 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFilenames) }, // optional bool has_filenames = 8 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasLineNumbers) }, // optional bool has_line_numbers = 9 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasInlineFrames) }, // optional bool has_inline_frames = 10 +} + +func (p *Location) decoder() []decoder { + return locationDecoder +} + +func (p *Location) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeUint64Opt(b, 2, p.mappingIDX) + encodeUint64Opt(b, 3, p.Address) + for i := range p.Line { + encodeMessage(b, 4, &p.Line[i]) + } + encodeBoolOpt(b, 5, p.IsFolded) +} + +var locationDecoder = []decoder{ + nil, // 0 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).ID) }, // optional uint64 id = 1; + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).mappingIDX) }, // optional uint64 mapping_id = 2; + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).Address) }, // optional uint64 address = 3; + func(b *buffer, m message) error { // repeated Line line = 4 + pp := m.(*Location) + n := len(pp.Line) + pp.Line = append(pp.Line, Line{}) + return decodeMessage(b, &pp.Line[n]) + }, + func(b *buffer, m message) error { return decodeBool(b, &m.(*Location).IsFolded) }, // optional bool is_folded = 5; +} + +func (p *Line) decoder() []decoder { + return lineDecoder +} + +func (p *Line) encode(b *buffer) { + encodeUint64Opt(b, 1, p.functionIDX) + encodeInt64Opt(b, 2, p.Line) +} + +var lineDecoder = []decoder{ + nil, // 0 + // optional uint64 function_id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Line).functionIDX) }, + // optional int64 line = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Line).Line) }, +} + +func (p *Function) decoder() []decoder { + return functionDecoder +} + +func (p *Function) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeInt64Opt(b, 2, p.nameX) + encodeInt64Opt(b, 3, p.systemNameX) + encodeInt64Opt(b, 4, p.filenameX) + encodeInt64Opt(b, 5, p.StartLine) +} + +var functionDecoder = []decoder{ + nil, // 0 + // optional uint64 id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Function).ID) }, + // optional int64 function_name = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).nameX) }, + // optional int64 function_system_name = 3 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).systemNameX) }, + // repeated int64 filename = 4 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).filenameX) }, + // optional int64 start_line = 5 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).StartLine) }, +} + +func addString(strings map[string]int, s string) int64 { + i, ok := strings[s] + if !ok { + i = len(strings) + strings[s] = i + } + return int64(i) +} + +func getString(strings []string, strng *int64, err error) (string, error) { + if err != nil { + return "", err + } + s := int(*strng) + if s < 0 || s >= len(strings) { + return "", errMalformed + } + *strng = 0 + return strings[s], nil +} diff --git a/vendor/github.com/google/pprof/profile/filter.go b/vendor/github.com/google/pprof/profile/filter.go new file mode 100644 index 000000000..c794b9390 --- /dev/null +++ b/vendor/github.com/google/pprof/profile/filter.go @@ -0,0 +1,274 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +// Implements methods to filter samples from profiles. + +import "regexp" + +// FilterSamplesByName filters the samples in a profile and only keeps +// samples where at least one frame matches focus but none match ignore. +// Returns true is the corresponding regexp matched at least one sample. +func (p *Profile) FilterSamplesByName(focus, ignore, hide, show *regexp.Regexp) (fm, im, hm, hnm bool) { + if focus == nil && ignore == nil && hide == nil && show == nil { + fm = true // Missing focus implies a match + return + } + focusOrIgnore := make(map[uint64]bool) + hidden := make(map[uint64]bool) + for _, l := range p.Location { + if ignore != nil && l.matchesName(ignore) { + im = true + focusOrIgnore[l.ID] = false + } else if focus == nil || l.matchesName(focus) { + fm = true + focusOrIgnore[l.ID] = true + } + + if hide != nil && l.matchesName(hide) { + hm = true + l.Line = l.unmatchedLines(hide) + if len(l.Line) == 0 { + hidden[l.ID] = true + } + } + if show != nil { + l.Line = l.matchedLines(show) + if len(l.Line) == 0 { + hidden[l.ID] = true + } else { + hnm = true + } + } + } + + s := make([]*Sample, 0, len(p.Sample)) + for _, sample := range p.Sample { + if focusedAndNotIgnored(sample.Location, focusOrIgnore) { + if len(hidden) > 0 { + var locs []*Location + for _, loc := range sample.Location { + if !hidden[loc.ID] { + locs = append(locs, loc) + } + } + if len(locs) == 0 { + // Remove sample with no locations (by not adding it to s). + continue + } + sample.Location = locs + } + s = append(s, sample) + } + } + p.Sample = s + + return +} + +// ShowFrom drops all stack frames above the highest matching frame and returns +// whether a match was found. If showFrom is nil it returns false and does not +// modify the profile. +// +// Example: consider a sample with frames [A, B, C, B], where A is the root. +// ShowFrom(nil) returns false and has frames [A, B, C, B]. +// ShowFrom(A) returns true and has frames [A, B, C, B]. +// ShowFrom(B) returns true and has frames [B, C, B]. +// ShowFrom(C) returns true and has frames [C, B]. +// ShowFrom(D) returns false and drops the sample because no frames remain. +func (p *Profile) ShowFrom(showFrom *regexp.Regexp) (matched bool) { + if showFrom == nil { + return false + } + // showFromLocs stores location IDs that matched ShowFrom. + showFromLocs := make(map[uint64]bool) + // Apply to locations. + for _, loc := range p.Location { + if filterShowFromLocation(loc, showFrom) { + showFromLocs[loc.ID] = true + matched = true + } + } + // For all samples, strip locations after the highest matching one. + s := make([]*Sample, 0, len(p.Sample)) + for _, sample := range p.Sample { + for i := len(sample.Location) - 1; i >= 0; i-- { + if showFromLocs[sample.Location[i].ID] { + sample.Location = sample.Location[:i+1] + s = append(s, sample) + break + } + } + } + p.Sample = s + return matched +} + +// filterShowFromLocation tests a showFrom regex against a location, removes +// lines after the last match and returns whether a match was found. If the +// mapping is matched, then all lines are kept. +func filterShowFromLocation(loc *Location, showFrom *regexp.Regexp) bool { + if m := loc.Mapping; m != nil && showFrom.MatchString(m.File) { + return true + } + if i := loc.lastMatchedLineIndex(showFrom); i >= 0 { + loc.Line = loc.Line[:i+1] + return true + } + return false +} + +// lastMatchedLineIndex returns the index of the last line that matches a regex, +// or -1 if no match is found. +func (loc *Location) lastMatchedLineIndex(re *regexp.Regexp) int { + for i := len(loc.Line) - 1; i >= 0; i-- { + if fn := loc.Line[i].Function; fn != nil { + if re.MatchString(fn.Name) || re.MatchString(fn.Filename) { + return i + } + } + } + return -1 +} + +// FilterTagsByName filters the tags in a profile and only keeps +// tags that match show and not hide. +func (p *Profile) FilterTagsByName(show, hide *regexp.Regexp) (sm, hm bool) { + matchRemove := func(name string) bool { + matchShow := show == nil || show.MatchString(name) + matchHide := hide != nil && hide.MatchString(name) + + if matchShow { + sm = true + } + if matchHide { + hm = true + } + return !matchShow || matchHide + } + for _, s := range p.Sample { + for lab := range s.Label { + if matchRemove(lab) { + delete(s.Label, lab) + } + } + for lab := range s.NumLabel { + if matchRemove(lab) { + delete(s.NumLabel, lab) + } + } + } + return +} + +// matchesName returns whether the location matches the regular +// expression. It checks any available function names, file names, and +// mapping object filename. +func (loc *Location) matchesName(re *regexp.Regexp) bool { + for _, ln := range loc.Line { + if fn := ln.Function; fn != nil { + if re.MatchString(fn.Name) || re.MatchString(fn.Filename) { + return true + } + } + } + if m := loc.Mapping; m != nil && re.MatchString(m.File) { + return true + } + return false +} + +// unmatchedLines returns the lines in the location that do not match +// the regular expression. +func (loc *Location) unmatchedLines(re *regexp.Regexp) []Line { + if m := loc.Mapping; m != nil && re.MatchString(m.File) { + return nil + } + var lines []Line + for _, ln := range loc.Line { + if fn := ln.Function; fn != nil { + if re.MatchString(fn.Name) || re.MatchString(fn.Filename) { + continue + } + } + lines = append(lines, ln) + } + return lines +} + +// matchedLines returns the lines in the location that match +// the regular expression. +func (loc *Location) matchedLines(re *regexp.Regexp) []Line { + if m := loc.Mapping; m != nil && re.MatchString(m.File) { + return loc.Line + } + var lines []Line + for _, ln := range loc.Line { + if fn := ln.Function; fn != nil { + if !re.MatchString(fn.Name) && !re.MatchString(fn.Filename) { + continue + } + } + lines = append(lines, ln) + } + return lines +} + +// focusedAndNotIgnored looks up a slice of ids against a map of +// focused/ignored locations. The map only contains locations that are +// explicitly focused or ignored. Returns whether there is at least +// one focused location but no ignored locations. +func focusedAndNotIgnored(locs []*Location, m map[uint64]bool) bool { + var f bool + for _, loc := range locs { + if focus, focusOrIgnore := m[loc.ID]; focusOrIgnore { + if focus { + // Found focused location. Must keep searching in case there + // is an ignored one as well. + f = true + } else { + // Found ignored location. Can return false right away. + return false + } + } + } + return f +} + +// TagMatch selects tags for filtering +type TagMatch func(s *Sample) bool + +// FilterSamplesByTag removes all samples from the profile, except +// those that match focus and do not match the ignore regular +// expression. +func (p *Profile) FilterSamplesByTag(focus, ignore TagMatch) (fm, im bool) { + samples := make([]*Sample, 0, len(p.Sample)) + for _, s := range p.Sample { + focused, ignored := true, false + if focus != nil { + focused = focus(s) + } + if ignore != nil { + ignored = ignore(s) + } + fm = fm || focused + im = im || ignored + if focused && !ignored { + samples = append(samples, s) + } + } + p.Sample = samples + return +} diff --git a/vendor/github.com/google/pprof/profile/index.go b/vendor/github.com/google/pprof/profile/index.go new file mode 100644 index 000000000..bef1d6046 --- /dev/null +++ b/vendor/github.com/google/pprof/profile/index.go @@ -0,0 +1,64 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +import ( + "fmt" + "strconv" + "strings" +) + +// SampleIndexByName returns the appropriate index for a value of sample index. +// If numeric, it returns the number, otherwise it looks up the text in the +// profile sample types. +func (p *Profile) SampleIndexByName(sampleIndex string) (int, error) { + if sampleIndex == "" { + if dst := p.DefaultSampleType; dst != "" { + for i, t := range sampleTypes(p) { + if t == dst { + return i, nil + } + } + } + // By default select the last sample value + return len(p.SampleType) - 1, nil + } + if i, err := strconv.Atoi(sampleIndex); err == nil { + if i < 0 || i >= len(p.SampleType) { + return 0, fmt.Errorf("sample_index %s is outside the range [0..%d]", sampleIndex, len(p.SampleType)-1) + } + return i, nil + } + + // Remove the inuse_ prefix to support legacy pprof options + // "inuse_space" and "inuse_objects" for profiles containing types + // "space" and "objects". + noInuse := strings.TrimPrefix(sampleIndex, "inuse_") + for i, t := range p.SampleType { + if t.Type == sampleIndex || t.Type == noInuse { + return i, nil + } + } + + return 0, fmt.Errorf("sample_index %q must be one of: %v", sampleIndex, sampleTypes(p)) +} + +func sampleTypes(p *Profile) []string { + types := make([]string, len(p.SampleType)) + for i, t := range p.SampleType { + types[i] = t.Type + } + return types +} diff --git a/vendor/github.com/google/pprof/profile/legacy_java_profile.go b/vendor/github.com/google/pprof/profile/legacy_java_profile.go new file mode 100644 index 000000000..91f45e53c --- /dev/null +++ b/vendor/github.com/google/pprof/profile/legacy_java_profile.go @@ -0,0 +1,315 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file implements parsers to convert java legacy profiles into +// the profile.proto format. + +package profile + +import ( + "bytes" + "fmt" + "io" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +var ( + attributeRx = regexp.MustCompile(`([\w ]+)=([\w ]+)`) + javaSampleRx = regexp.MustCompile(` *(\d+) +(\d+) +@ +([ x0-9a-f]*)`) + javaLocationRx = regexp.MustCompile(`^\s*0x([[:xdigit:]]+)\s+(.*)\s*$`) + javaLocationFileLineRx = regexp.MustCompile(`^(.*)\s+\((.+):(-?[[:digit:]]+)\)$`) + javaLocationPathRx = regexp.MustCompile(`^(.*)\s+\((.*)\)$`) +) + +// javaCPUProfile returns a new Profile from profilez data. +// b is the profile bytes after the header, period is the profiling +// period, and parse is a function to parse 8-byte chunks from the +// profile in its native endianness. +func javaCPUProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) (*Profile, error) { + p := &Profile{ + Period: period * 1000, + PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"}, + SampleType: []*ValueType{{Type: "samples", Unit: "count"}, {Type: "cpu", Unit: "nanoseconds"}}, + } + var err error + var locs map[uint64]*Location + if b, locs, err = parseCPUSamples(b, parse, false, p); err != nil { + return nil, err + } + + if err = parseJavaLocations(b, locs, p); err != nil { + return nil, err + } + + // Strip out addresses for better merge. + if err = p.Aggregate(true, true, true, true, false); err != nil { + return nil, err + } + + return p, nil +} + +// parseJavaProfile returns a new profile from heapz or contentionz +// data. b is the profile bytes after the header. +func parseJavaProfile(b []byte) (*Profile, error) { + h := bytes.SplitAfterN(b, []byte("\n"), 2) + if len(h) < 2 { + return nil, errUnrecognized + } + + p := &Profile{ + PeriodType: &ValueType{}, + } + header := string(bytes.TrimSpace(h[0])) + + var err error + var pType string + switch header { + case "--- heapz 1 ---": + pType = "heap" + case "--- contentionz 1 ---": + pType = "contention" + default: + return nil, errUnrecognized + } + + if b, err = parseJavaHeader(pType, h[1], p); err != nil { + return nil, err + } + var locs map[uint64]*Location + if b, locs, err = parseJavaSamples(pType, b, p); err != nil { + return nil, err + } + if err = parseJavaLocations(b, locs, p); err != nil { + return nil, err + } + + // Strip out addresses for better merge. + if err = p.Aggregate(true, true, true, true, false); err != nil { + return nil, err + } + + return p, nil +} + +// parseJavaHeader parses the attribute section on a java profile and +// populates a profile. Returns the remainder of the buffer after all +// attributes. +func parseJavaHeader(pType string, b []byte, p *Profile) ([]byte, error) { + nextNewLine := bytes.IndexByte(b, byte('\n')) + for nextNewLine != -1 { + line := string(bytes.TrimSpace(b[0:nextNewLine])) + if line != "" { + h := attributeRx.FindStringSubmatch(line) + if h == nil { + // Not a valid attribute, exit. + return b, nil + } + + attribute, value := strings.TrimSpace(h[1]), strings.TrimSpace(h[2]) + var err error + switch pType + "/" + attribute { + case "heap/format", "cpu/format", "contention/format": + if value != "java" { + return nil, errUnrecognized + } + case "heap/resolution": + p.SampleType = []*ValueType{ + {Type: "inuse_objects", Unit: "count"}, + {Type: "inuse_space", Unit: value}, + } + case "contention/resolution": + p.SampleType = []*ValueType{ + {Type: "contentions", Unit: "count"}, + {Type: "delay", Unit: value}, + } + case "contention/sampling period": + p.PeriodType = &ValueType{ + Type: "contentions", Unit: "count", + } + if p.Period, err = strconv.ParseInt(value, 0, 64); err != nil { + return nil, fmt.Errorf("failed to parse attribute %s: %v", line, err) + } + case "contention/ms since reset": + millis, err := strconv.ParseInt(value, 0, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse attribute %s: %v", line, err) + } + p.DurationNanos = millis * 1000 * 1000 + default: + return nil, errUnrecognized + } + } + // Grab next line. + b = b[nextNewLine+1:] + nextNewLine = bytes.IndexByte(b, byte('\n')) + } + return b, nil +} + +// parseJavaSamples parses the samples from a java profile and +// populates the Samples in a profile. Returns the remainder of the +// buffer after the samples. +func parseJavaSamples(pType string, b []byte, p *Profile) ([]byte, map[uint64]*Location, error) { + nextNewLine := bytes.IndexByte(b, byte('\n')) + locs := make(map[uint64]*Location) + for nextNewLine != -1 { + line := string(bytes.TrimSpace(b[0:nextNewLine])) + if line != "" { + sample := javaSampleRx.FindStringSubmatch(line) + if sample == nil { + // Not a valid sample, exit. + return b, locs, nil + } + + // Java profiles have data/fields inverted compared to other + // profile types. + var err error + value1, value2, value3 := sample[2], sample[1], sample[3] + addrs, err := parseHexAddresses(value3) + if err != nil { + return nil, nil, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + var sloc []*Location + for _, addr := range addrs { + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + s := &Sample{ + Value: make([]int64, 2), + Location: sloc, + } + + if s.Value[0], err = strconv.ParseInt(value1, 0, 64); err != nil { + return nil, nil, fmt.Errorf("parsing sample %s: %v", line, err) + } + if s.Value[1], err = strconv.ParseInt(value2, 0, 64); err != nil { + return nil, nil, fmt.Errorf("parsing sample %s: %v", line, err) + } + + switch pType { + case "heap": + const javaHeapzSamplingRate = 524288 // 512K + if s.Value[0] == 0 { + return nil, nil, fmt.Errorf("parsing sample %s: second value must be non-zero", line) + } + s.NumLabel = map[string][]int64{"bytes": {s.Value[1] / s.Value[0]}} + s.Value[0], s.Value[1] = scaleHeapSample(s.Value[0], s.Value[1], javaHeapzSamplingRate) + case "contention": + if period := p.Period; period != 0 { + s.Value[0] = s.Value[0] * p.Period + s.Value[1] = s.Value[1] * p.Period + } + } + p.Sample = append(p.Sample, s) + } + // Grab next line. + b = b[nextNewLine+1:] + nextNewLine = bytes.IndexByte(b, byte('\n')) + } + return b, locs, nil +} + +// parseJavaLocations parses the location information in a java +// profile and populates the Locations in a profile. It uses the +// location addresses from the profile as both the ID of each +// location. +func parseJavaLocations(b []byte, locs map[uint64]*Location, p *Profile) error { + r := bytes.NewBuffer(b) + fns := make(map[string]*Function) + for { + line, err := r.ReadString('\n') + if err != nil { + if err != io.EOF { + return err + } + if line == "" { + break + } + } + + if line = strings.TrimSpace(line); line == "" { + continue + } + + jloc := javaLocationRx.FindStringSubmatch(line) + if len(jloc) != 3 { + continue + } + addr, err := strconv.ParseUint(jloc[1], 16, 64) + if err != nil { + return fmt.Errorf("parsing sample %s: %v", line, err) + } + loc := locs[addr] + if loc == nil { + // Unused/unseen + continue + } + var lineFunc, lineFile string + var lineNo int64 + + if fileLine := javaLocationFileLineRx.FindStringSubmatch(jloc[2]); len(fileLine) == 4 { + // Found a line of the form: "function (file:line)" + lineFunc, lineFile = fileLine[1], fileLine[2] + if n, err := strconv.ParseInt(fileLine[3], 10, 64); err == nil && n > 0 { + lineNo = n + } + } else if filePath := javaLocationPathRx.FindStringSubmatch(jloc[2]); len(filePath) == 3 { + // If there's not a file:line, it's a shared library path. + // The path isn't interesting, so just give the .so. + lineFunc, lineFile = filePath[1], filepath.Base(filePath[2]) + } else if strings.Contains(jloc[2], "generated stub/JIT") { + lineFunc = "STUB" + } else { + // Treat whole line as the function name. This is used by the + // java agent for internal states such as "GC" or "VM". + lineFunc = jloc[2] + } + fn := fns[lineFunc] + + if fn == nil { + fn = &Function{ + Name: lineFunc, + SystemName: lineFunc, + Filename: lineFile, + } + fns[lineFunc] = fn + p.Function = append(p.Function, fn) + } + loc.Line = []Line{ + { + Function: fn, + Line: lineNo, + }, + } + loc.Address = 0 + } + + p.remapLocationIDs() + p.remapFunctionIDs() + p.remapMappingIDs() + + return nil +} diff --git a/vendor/github.com/google/pprof/profile/legacy_profile.go b/vendor/github.com/google/pprof/profile/legacy_profile.go new file mode 100644 index 000000000..8d07fd6c2 --- /dev/null +++ b/vendor/github.com/google/pprof/profile/legacy_profile.go @@ -0,0 +1,1228 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file implements parsers to convert legacy profiles into the +// profile.proto format. + +package profile + +import ( + "bufio" + "bytes" + "fmt" + "io" + "math" + "regexp" + "strconv" + "strings" +) + +var ( + countStartRE = regexp.MustCompile(`\A(\S+) profile: total \d+\z`) + countRE = regexp.MustCompile(`\A(\d+) @(( 0x[0-9a-f]+)+)\z`) + + heapHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] *@ *(heap[_a-z0-9]*)/?(\d*)`) + heapSampleRE = regexp.MustCompile(`(-?\d+): *(-?\d+) *\[ *(\d+): *(\d+) *] @([ x0-9a-f]*)`) + + contentionSampleRE = regexp.MustCompile(`(\d+) *(\d+) @([ x0-9a-f]*)`) + + hexNumberRE = regexp.MustCompile(`0x[0-9a-f]+`) + + growthHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ growthz?`) + + fragmentationHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ fragmentationz?`) + + threadzStartRE = regexp.MustCompile(`--- threadz \d+ ---`) + threadStartRE = regexp.MustCompile(`--- Thread ([[:xdigit:]]+) \(name: (.*)/(\d+)\) stack: ---`) + + // Regular expressions to parse process mappings. Support the format used by Linux /proc/.../maps and other tools. + // Recommended format: + // Start End object file name offset(optional) linker build id + // 0x40000-0x80000 /path/to/binary (@FF00) abc123456 + spaceDigits = `\s+[[:digit:]]+` + hexPair = `\s+[[:xdigit:]]+:[[:xdigit:]]+` + oSpace = `\s*` + // Capturing expressions. + cHex = `(?:0x)?([[:xdigit:]]+)` + cHexRange = `\s*` + cHex + `[\s-]?` + oSpace + cHex + `:?` + cSpaceString = `(?:\s+(\S+))?` + cSpaceHex = `(?:\s+([[:xdigit:]]+))?` + cSpaceAtOffset = `(?:\s+\(@([[:xdigit:]]+)\))?` + cPerm = `(?:\s+([-rwxp]+))?` + + procMapsRE = regexp.MustCompile(`^` + cHexRange + cPerm + cSpaceHex + hexPair + spaceDigits + cSpaceString) + briefMapsRE = regexp.MustCompile(`^` + cHexRange + cPerm + cSpaceString + cSpaceAtOffset + cSpaceHex) + + // Regular expression to parse log data, of the form: + // ... file:line] msg... + logInfoRE = regexp.MustCompile(`^[^\[\]]+:[0-9]+]\s`) +) + +func isSpaceOrComment(line string) bool { + trimmed := strings.TrimSpace(line) + return len(trimmed) == 0 || trimmed[0] == '#' +} + +// parseGoCount parses a Go count profile (e.g., threadcreate or +// goroutine) and returns a new Profile. +func parseGoCount(b []byte) (*Profile, error) { + s := bufio.NewScanner(bytes.NewBuffer(b)) + // Skip comments at the beginning of the file. + for s.Scan() && isSpaceOrComment(s.Text()) { + } + if err := s.Err(); err != nil { + return nil, err + } + m := countStartRE.FindStringSubmatch(s.Text()) + if m == nil { + return nil, errUnrecognized + } + profileType := m[1] + p := &Profile{ + PeriodType: &ValueType{Type: profileType, Unit: "count"}, + Period: 1, + SampleType: []*ValueType{{Type: profileType, Unit: "count"}}, + } + locations := make(map[uint64]*Location) + for s.Scan() { + line := s.Text() + if isSpaceOrComment(line) { + continue + } + if strings.HasPrefix(line, "---") { + break + } + m := countRE.FindStringSubmatch(line) + if m == nil { + return nil, errMalformed + } + n, err := strconv.ParseInt(m[1], 0, 64) + if err != nil { + return nil, errMalformed + } + fields := strings.Fields(m[2]) + locs := make([]*Location, 0, len(fields)) + for _, stk := range fields { + addr, err := strconv.ParseUint(stk, 0, 64) + if err != nil { + return nil, errMalformed + } + // Adjust all frames by -1 to land on top of the call instruction. + addr-- + loc := locations[addr] + if loc == nil { + loc = &Location{ + Address: addr, + } + locations[addr] = loc + p.Location = append(p.Location, loc) + } + locs = append(locs, loc) + } + p.Sample = append(p.Sample, &Sample{ + Location: locs, + Value: []int64{n}, + }) + } + if err := s.Err(); err != nil { + return nil, err + } + + if err := parseAdditionalSections(s, p); err != nil { + return nil, err + } + return p, nil +} + +// remapLocationIDs ensures there is a location for each address +// referenced by a sample, and remaps the samples to point to the new +// location ids. +func (p *Profile) remapLocationIDs() { + seen := make(map[*Location]bool, len(p.Location)) + var locs []*Location + + for _, s := range p.Sample { + for _, l := range s.Location { + if seen[l] { + continue + } + l.ID = uint64(len(locs) + 1) + locs = append(locs, l) + seen[l] = true + } + } + p.Location = locs +} + +func (p *Profile) remapFunctionIDs() { + seen := make(map[*Function]bool, len(p.Function)) + var fns []*Function + + for _, l := range p.Location { + for _, ln := range l.Line { + fn := ln.Function + if fn == nil || seen[fn] { + continue + } + fn.ID = uint64(len(fns) + 1) + fns = append(fns, fn) + seen[fn] = true + } + } + p.Function = fns +} + +// remapMappingIDs matches location addresses with existing mappings +// and updates them appropriately. This is O(N*M), if this ever shows +// up as a bottleneck, evaluate sorting the mappings and doing a +// binary search, which would make it O(N*log(M)). +func (p *Profile) remapMappingIDs() { + // Some profile handlers will incorrectly set regions for the main + // executable if its section is remapped. Fix them through heuristics. + + if len(p.Mapping) > 0 { + // Remove the initial mapping if named '/anon_hugepage' and has a + // consecutive adjacent mapping. + if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") { + if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start { + p.Mapping = p.Mapping[1:] + } + } + } + + // Subtract the offset from the start of the main mapping if it + // ends up at a recognizable start address. + if len(p.Mapping) > 0 { + const expectedStart = 0x400000 + if m := p.Mapping[0]; m.Start-m.Offset == expectedStart { + m.Start = expectedStart + m.Offset = 0 + } + } + + // Associate each location with an address to the corresponding + // mapping. Create fake mapping if a suitable one isn't found. + var fake *Mapping +nextLocation: + for _, l := range p.Location { + a := l.Address + if l.Mapping != nil || a == 0 { + continue + } + for _, m := range p.Mapping { + if m.Start <= a && a < m.Limit { + l.Mapping = m + continue nextLocation + } + } + // Work around legacy handlers failing to encode the first + // part of mappings split into adjacent ranges. + for _, m := range p.Mapping { + if m.Offset != 0 && m.Start-m.Offset <= a && a < m.Start { + m.Start -= m.Offset + m.Offset = 0 + l.Mapping = m + continue nextLocation + } + } + // If there is still no mapping, create a fake one. + // This is important for the Go legacy handler, which produced + // no mappings. + if fake == nil { + fake = &Mapping{ + ID: 1, + Limit: ^uint64(0), + } + p.Mapping = append(p.Mapping, fake) + } + l.Mapping = fake + } + + // Reset all mapping IDs. + for i, m := range p.Mapping { + m.ID = uint64(i + 1) + } +} + +var cpuInts = []func([]byte) (uint64, []byte){ + get32l, + get32b, + get64l, + get64b, +} + +func get32l(b []byte) (uint64, []byte) { + if len(b) < 4 { + return 0, nil + } + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24, b[4:] +} + +func get32b(b []byte) (uint64, []byte) { + if len(b) < 4 { + return 0, nil + } + return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24, b[4:] +} + +func get64l(b []byte) (uint64, []byte) { + if len(b) < 8 { + return 0, nil + } + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56, b[8:] +} + +func get64b(b []byte) (uint64, []byte) { + if len(b) < 8 { + return 0, nil + } + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56, b[8:] +} + +// parseCPU parses a profilez legacy profile and returns a newly +// populated Profile. +// +// The general format for profilez samples is a sequence of words in +// binary format. The first words are a header with the following data: +// +// 1st word -- 0 +// 2nd word -- 3 +// 3rd word -- 0 if a c++ application, 1 if a java application. +// 4th word -- Sampling period (in microseconds). +// 5th word -- Padding. +func parseCPU(b []byte) (*Profile, error) { + var parse func([]byte) (uint64, []byte) + var n1, n2, n3, n4, n5 uint64 + for _, parse = range cpuInts { + var tmp []byte + n1, tmp = parse(b) + n2, tmp = parse(tmp) + n3, tmp = parse(tmp) + n4, tmp = parse(tmp) + n5, tmp = parse(tmp) + + if tmp != nil && n1 == 0 && n2 == 3 && n3 == 0 && n4 > 0 && n5 == 0 { + b = tmp + return cpuProfile(b, int64(n4), parse) + } + if tmp != nil && n1 == 0 && n2 == 3 && n3 == 1 && n4 > 0 && n5 == 0 { + b = tmp + return javaCPUProfile(b, int64(n4), parse) + } + } + return nil, errUnrecognized +} + +// cpuProfile returns a new Profile from C++ profilez data. +// b is the profile bytes after the header, period is the profiling +// period, and parse is a function to parse 8-byte chunks from the +// profile in its native endianness. +func cpuProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) (*Profile, error) { + p := &Profile{ + Period: period * 1000, + PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"}, + SampleType: []*ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + }, + } + var err error + if b, _, err = parseCPUSamples(b, parse, true, p); err != nil { + return nil, err + } + + // If *most* samples have the same second-to-the-bottom frame, it + // strongly suggests that it is an uninteresting artifact of + // measurement -- a stack frame pushed by the signal handler. The + // bottom frame is always correct as it is picked up from the signal + // structure, not the stack. Check if this is the case and if so, + // remove. + + // Remove up to two frames. + maxiter := 2 + // Allow one different sample for this many samples with the same + // second-to-last frame. + similarSamples := 32 + margin := len(p.Sample) / similarSamples + + for iter := 0; iter < maxiter; iter++ { + addr1 := make(map[uint64]int) + for _, s := range p.Sample { + if len(s.Location) > 1 { + a := s.Location[1].Address + addr1[a] = addr1[a] + 1 + } + } + + for id1, count := range addr1 { + if count >= len(p.Sample)-margin { + // Found uninteresting frame, strip it out from all samples + for _, s := range p.Sample { + if len(s.Location) > 1 && s.Location[1].Address == id1 { + s.Location = append(s.Location[:1], s.Location[2:]...) + } + } + break + } + } + } + + if err := p.ParseMemoryMap(bytes.NewBuffer(b)); err != nil { + return nil, err + } + + cleanupDuplicateLocations(p) + return p, nil +} + +func cleanupDuplicateLocations(p *Profile) { + // The profile handler may duplicate the leaf frame, because it gets + // its address both from stack unwinding and from the signal + // context. Detect this and delete the duplicate, which has been + // adjusted by -1. The leaf address should not be adjusted as it is + // not a call. + for _, s := range p.Sample { + if len(s.Location) > 1 && s.Location[0].Address == s.Location[1].Address+1 { + s.Location = append(s.Location[:1], s.Location[2:]...) + } + } +} + +// parseCPUSamples parses a collection of profilez samples from a +// profile. +// +// profilez samples are a repeated sequence of stack frames of the +// form: +// +// 1st word -- The number of times this stack was encountered. +// 2nd word -- The size of the stack (StackSize). +// 3rd word -- The first address on the stack. +// ... +// StackSize + 2 -- The last address on the stack +// +// The last stack trace is of the form: +// +// 1st word -- 0 +// 2nd word -- 1 +// 3rd word -- 0 +// +// Addresses from stack traces may point to the next instruction after +// each call. Optionally adjust by -1 to land somewhere on the actual +// call (except for the leaf, which is not a call). +func parseCPUSamples(b []byte, parse func(b []byte) (uint64, []byte), adjust bool, p *Profile) ([]byte, map[uint64]*Location, error) { + locs := make(map[uint64]*Location) + for len(b) > 0 { + var count, nstk uint64 + count, b = parse(b) + nstk, b = parse(b) + if b == nil || nstk > uint64(len(b)/4) { + return nil, nil, errUnrecognized + } + var sloc []*Location + addrs := make([]uint64, nstk) + for i := 0; i < int(nstk); i++ { + addrs[i], b = parse(b) + } + + if count == 0 && nstk == 1 && addrs[0] == 0 { + // End of data marker + break + } + for i, addr := range addrs { + if adjust && i > 0 { + addr-- + } + loc := locs[addr] + if loc == nil { + loc = &Location{ + Address: addr, + } + locs[addr] = loc + p.Location = append(p.Location, loc) + } + sloc = append(sloc, loc) + } + p.Sample = append(p.Sample, + &Sample{ + Value: []int64{int64(count), int64(count) * p.Period}, + Location: sloc, + }) + } + // Reached the end without finding the EOD marker. + return b, locs, nil +} + +// parseHeap parses a heapz legacy or a growthz profile and +// returns a newly populated Profile. +func parseHeap(b []byte) (p *Profile, err error) { + s := bufio.NewScanner(bytes.NewBuffer(b)) + if !s.Scan() { + if err := s.Err(); err != nil { + return nil, err + } + return nil, errUnrecognized + } + p = &Profile{} + + sampling := "" + hasAlloc := false + + line := s.Text() + p.PeriodType = &ValueType{Type: "space", Unit: "bytes"} + if header := heapHeaderRE.FindStringSubmatch(line); header != nil { + sampling, p.Period, hasAlloc, err = parseHeapHeader(line) + if err != nil { + return nil, err + } + } else if header = growthHeaderRE.FindStringSubmatch(line); header != nil { + p.Period = 1 + } else if header = fragmentationHeaderRE.FindStringSubmatch(line); header != nil { + p.Period = 1 + } else { + return nil, errUnrecognized + } + + if hasAlloc { + // Put alloc before inuse so that default pprof selection + // will prefer inuse_space. + p.SampleType = []*ValueType{ + {Type: "alloc_objects", Unit: "count"}, + {Type: "alloc_space", Unit: "bytes"}, + {Type: "inuse_objects", Unit: "count"}, + {Type: "inuse_space", Unit: "bytes"}, + } + } else { + p.SampleType = []*ValueType{ + {Type: "objects", Unit: "count"}, + {Type: "space", Unit: "bytes"}, + } + } + + locs := make(map[uint64]*Location) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + + if isSpaceOrComment(line) { + continue + } + + if isMemoryMapSentinel(line) { + break + } + + value, blocksize, addrs, err := parseHeapSample(line, p.Period, sampling, hasAlloc) + if err != nil { + return nil, err + } + + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + p.Sample = append(p.Sample, &Sample{ + Value: value, + Location: sloc, + NumLabel: map[string][]int64{"bytes": {blocksize}}, + }) + } + if err := s.Err(); err != nil { + return nil, err + } + if err := parseAdditionalSections(s, p); err != nil { + return nil, err + } + return p, nil +} + +func parseHeapHeader(line string) (sampling string, period int64, hasAlloc bool, err error) { + header := heapHeaderRE.FindStringSubmatch(line) + if header == nil { + return "", 0, false, errUnrecognized + } + + if len(header[6]) > 0 { + if period, err = strconv.ParseInt(header[6], 10, 64); err != nil { + return "", 0, false, errUnrecognized + } + } + + if (header[3] != header[1] && header[3] != "0") || (header[4] != header[2] && header[4] != "0") { + hasAlloc = true + } + + switch header[5] { + case "heapz_v2", "heap_v2": + return "v2", period, hasAlloc, nil + case "heapprofile": + return "", 1, hasAlloc, nil + case "heap": + return "v2", period / 2, hasAlloc, nil + default: + return "", 0, false, errUnrecognized + } +} + +// parseHeapSample parses a single row from a heap profile into a new Sample. +func parseHeapSample(line string, rate int64, sampling string, includeAlloc bool) (value []int64, blocksize int64, addrs []uint64, err error) { + sampleData := heapSampleRE.FindStringSubmatch(line) + if len(sampleData) != 6 { + return nil, 0, nil, fmt.Errorf("unexpected number of sample values: got %d, want 6", len(sampleData)) + } + + // This is a local-scoped helper function to avoid needing to pass + // around rate, sampling and many return parameters. + addValues := func(countString, sizeString string, label string) error { + count, err := strconv.ParseInt(countString, 10, 64) + if err != nil { + return fmt.Errorf("malformed sample: %s: %v", line, err) + } + size, err := strconv.ParseInt(sizeString, 10, 64) + if err != nil { + return fmt.Errorf("malformed sample: %s: %v", line, err) + } + if count == 0 && size != 0 { + return fmt.Errorf("%s count was 0 but %s bytes was %d", label, label, size) + } + if count != 0 { + blocksize = size / count + if sampling == "v2" { + count, size = scaleHeapSample(count, size, rate) + } + } + value = append(value, count, size) + return nil + } + + if includeAlloc { + if err := addValues(sampleData[3], sampleData[4], "allocation"); err != nil { + return nil, 0, nil, err + } + } + + if err := addValues(sampleData[1], sampleData[2], "inuse"); err != nil { + return nil, 0, nil, err + } + + addrs, err = parseHexAddresses(sampleData[5]) + if err != nil { + return nil, 0, nil, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + return value, blocksize, addrs, nil +} + +// parseHexAddresses extracts hex numbers from a string, attempts to convert +// each to an unsigned 64-bit number and returns the resulting numbers as a +// slice, or an error if the string contains hex numbers which are too large to +// handle (which means a malformed profile). +func parseHexAddresses(s string) ([]uint64, error) { + hexStrings := hexNumberRE.FindAllString(s, -1) + var addrs []uint64 + for _, s := range hexStrings { + if addr, err := strconv.ParseUint(s, 0, 64); err == nil { + addrs = append(addrs, addr) + } else { + return nil, fmt.Errorf("failed to parse as hex 64-bit number: %s", s) + } + } + return addrs, nil +} + +// scaleHeapSample adjusts the data from a heapz Sample to +// account for its probability of appearing in the collected +// data. heapz profiles are a sampling of the memory allocations +// requests in a program. We estimate the unsampled value by dividing +// each collected sample by its probability of appearing in the +// profile. heapz v2 profiles rely on a poisson process to determine +// which samples to collect, based on the desired average collection +// rate R. The probability of a sample of size S to appear in that +// profile is 1-exp(-S/R). +func scaleHeapSample(count, size, rate int64) (int64, int64) { + if count == 0 || size == 0 { + return 0, 0 + } + + if rate <= 1 { + // if rate==1 all samples were collected so no adjustment is needed. + // if rate<1 treat as unknown and skip scaling. + return count, size + } + + avgSize := float64(size) / float64(count) + scale := 1 / (1 - math.Exp(-avgSize/float64(rate))) + + return int64(float64(count) * scale), int64(float64(size) * scale) +} + +// parseContention parses a mutex or contention profile. There are 2 cases: +// "--- contentionz " for legacy C++ profiles (and backwards compatibility) +// "--- mutex:" or "--- contention:" for profiles generated by the Go runtime. +func parseContention(b []byte) (*Profile, error) { + s := bufio.NewScanner(bytes.NewBuffer(b)) + if !s.Scan() { + if err := s.Err(); err != nil { + return nil, err + } + return nil, errUnrecognized + } + + switch l := s.Text(); { + case strings.HasPrefix(l, "--- contentionz "): + case strings.HasPrefix(l, "--- mutex:"): + case strings.HasPrefix(l, "--- contention:"): + default: + return nil, errUnrecognized + } + + p := &Profile{ + PeriodType: &ValueType{Type: "contentions", Unit: "count"}, + Period: 1, + SampleType: []*ValueType{ + {Type: "contentions", Unit: "count"}, + {Type: "delay", Unit: "nanoseconds"}, + }, + } + + var cpuHz int64 + // Parse text of the form "attribute = value" before the samples. + const delimiter = "=" + for s.Scan() { + line := s.Text() + if line = strings.TrimSpace(line); isSpaceOrComment(line) { + continue + } + if strings.HasPrefix(line, "---") { + break + } + attr := strings.SplitN(line, delimiter, 2) + if len(attr) != 2 { + break + } + key, val := strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1]) + var err error + switch key { + case "cycles/second": + if cpuHz, err = strconv.ParseInt(val, 0, 64); err != nil { + return nil, errUnrecognized + } + case "sampling period": + if p.Period, err = strconv.ParseInt(val, 0, 64); err != nil { + return nil, errUnrecognized + } + case "ms since reset": + ms, err := strconv.ParseInt(val, 0, 64) + if err != nil { + return nil, errUnrecognized + } + p.DurationNanos = ms * 1000 * 1000 + case "format": + // CPP contentionz profiles don't have format. + return nil, errUnrecognized + case "resolution": + // CPP contentionz profiles don't have resolution. + return nil, errUnrecognized + case "discarded samples": + default: + return nil, errUnrecognized + } + } + if err := s.Err(); err != nil { + return nil, err + } + + locs := make(map[uint64]*Location) + for { + line := strings.TrimSpace(s.Text()) + if strings.HasPrefix(line, "---") { + break + } + if !isSpaceOrComment(line) { + value, addrs, err := parseContentionSample(line, p.Period, cpuHz) + if err != nil { + return nil, err + } + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + p.Sample = append(p.Sample, &Sample{ + Value: value, + Location: sloc, + }) + } + if !s.Scan() { + break + } + } + if err := s.Err(); err != nil { + return nil, err + } + + if err := parseAdditionalSections(s, p); err != nil { + return nil, err + } + + return p, nil +} + +// parseContentionSample parses a single row from a contention profile +// into a new Sample. +func parseContentionSample(line string, period, cpuHz int64) (value []int64, addrs []uint64, err error) { + sampleData := contentionSampleRE.FindStringSubmatch(line) + if sampleData == nil { + return nil, nil, errUnrecognized + } + + v1, err := strconv.ParseInt(sampleData[1], 10, 64) + if err != nil { + return nil, nil, fmt.Errorf("malformed sample: %s: %v", line, err) + } + v2, err := strconv.ParseInt(sampleData[2], 10, 64) + if err != nil { + return nil, nil, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + // Unsample values if period and cpuHz are available. + // - Delays are scaled to cycles and then to nanoseconds. + // - Contentions are scaled to cycles. + if period > 0 { + if cpuHz > 0 { + cpuGHz := float64(cpuHz) / 1e9 + v1 = int64(float64(v1) * float64(period) / cpuGHz) + } + v2 = v2 * period + } + + value = []int64{v2, v1} + addrs, err = parseHexAddresses(sampleData[3]) + if err != nil { + return nil, nil, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + return value, addrs, nil +} + +// parseThread parses a Threadz profile and returns a new Profile. +func parseThread(b []byte) (*Profile, error) { + s := bufio.NewScanner(bytes.NewBuffer(b)) + // Skip past comments and empty lines seeking a real header. + for s.Scan() && isSpaceOrComment(s.Text()) { + } + + line := s.Text() + if m := threadzStartRE.FindStringSubmatch(line); m != nil { + // Advance over initial comments until first stack trace. + for s.Scan() { + if line = s.Text(); isMemoryMapSentinel(line) || strings.HasPrefix(line, "-") { + break + } + } + } else if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { + return nil, errUnrecognized + } + + p := &Profile{ + SampleType: []*ValueType{{Type: "thread", Unit: "count"}}, + PeriodType: &ValueType{Type: "thread", Unit: "count"}, + Period: 1, + } + + locs := make(map[uint64]*Location) + // Recognize each thread and populate profile samples. + for !isMemoryMapSentinel(line) { + if strings.HasPrefix(line, "---- no stack trace for") { + break + } + if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { + return nil, errUnrecognized + } + + var addrs []uint64 + var err error + line, addrs, err = parseThreadSample(s) + if err != nil { + return nil, err + } + if len(addrs) == 0 { + // We got a --same as previous threads--. Bump counters. + if len(p.Sample) > 0 { + s := p.Sample[len(p.Sample)-1] + s.Value[0]++ + } + continue + } + + var sloc []*Location + for i, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call + // (except for the leaf, which is not a call). + if i > 0 { + addr-- + } + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + p.Sample = append(p.Sample, &Sample{ + Value: []int64{1}, + Location: sloc, + }) + } + + if err := parseAdditionalSections(s, p); err != nil { + return nil, err + } + + cleanupDuplicateLocations(p) + return p, nil +} + +// parseThreadSample parses a symbolized or unsymbolized stack trace. +// Returns the first line after the traceback, the sample (or nil if +// it hits a 'same-as-previous' marker) and an error. +func parseThreadSample(s *bufio.Scanner) (nextl string, addrs []uint64, err error) { + var line string + sameAsPrevious := false + for s.Scan() { + line = strings.TrimSpace(s.Text()) + if line == "" { + continue + } + + if strings.HasPrefix(line, "---") { + break + } + if strings.Contains(line, "same as previous thread") { + sameAsPrevious = true + continue + } + + curAddrs, err := parseHexAddresses(line) + if err != nil { + return "", nil, fmt.Errorf("malformed sample: %s: %v", line, err) + } + addrs = append(addrs, curAddrs...) + } + if err := s.Err(); err != nil { + return "", nil, err + } + if sameAsPrevious { + return line, nil, nil + } + return line, addrs, nil +} + +// parseAdditionalSections parses any additional sections in the +// profile, ignoring any unrecognized sections. +func parseAdditionalSections(s *bufio.Scanner, p *Profile) error { + for !isMemoryMapSentinel(s.Text()) && s.Scan() { + } + if err := s.Err(); err != nil { + return err + } + return p.ParseMemoryMapFromScanner(s) +} + +// ParseProcMaps parses a memory map in the format of /proc/self/maps. +// ParseMemoryMap should be called after setting on a profile to +// associate locations to the corresponding mapping based on their +// address. +func ParseProcMaps(rd io.Reader) ([]*Mapping, error) { + s := bufio.NewScanner(rd) + return parseProcMapsFromScanner(s) +} + +func parseProcMapsFromScanner(s *bufio.Scanner) ([]*Mapping, error) { + var mapping []*Mapping + + var attrs []string + const delimiter = "=" + r := strings.NewReplacer() + for s.Scan() { + line := r.Replace(removeLoggingInfo(s.Text())) + m, err := parseMappingEntry(line) + if err != nil { + if err == errUnrecognized { + // Recognize assignments of the form: attr=value, and replace + // $attr with value on subsequent mappings. + if attr := strings.SplitN(line, delimiter, 2); len(attr) == 2 { + attrs = append(attrs, "$"+strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1])) + r = strings.NewReplacer(attrs...) + } + // Ignore any unrecognized entries + continue + } + return nil, err + } + if m == nil { + continue + } + mapping = append(mapping, m) + } + if err := s.Err(); err != nil { + return nil, err + } + return mapping, nil +} + +// removeLoggingInfo detects and removes log prefix entries generated +// by the glog package. If no logging prefix is detected, the string +// is returned unmodified. +func removeLoggingInfo(line string) string { + if match := logInfoRE.FindStringIndex(line); match != nil { + return line[match[1]:] + } + return line +} + +// ParseMemoryMap parses a memory map in the format of +// /proc/self/maps, and overrides the mappings in the current profile. +// It renumbers the samples and locations in the profile correspondingly. +func (p *Profile) ParseMemoryMap(rd io.Reader) error { + return p.ParseMemoryMapFromScanner(bufio.NewScanner(rd)) +} + +// ParseMemoryMapFromScanner parses a memory map in the format of +// /proc/self/maps or a variety of legacy format, and overrides the +// mappings in the current profile. It renumbers the samples and +// locations in the profile correspondingly. +func (p *Profile) ParseMemoryMapFromScanner(s *bufio.Scanner) error { + mapping, err := parseProcMapsFromScanner(s) + if err != nil { + return err + } + p.Mapping = append(p.Mapping, mapping...) + p.massageMappings() + p.remapLocationIDs() + p.remapFunctionIDs() + p.remapMappingIDs() + return nil +} + +func parseMappingEntry(l string) (*Mapping, error) { + var start, end, perm, file, offset, buildID string + if me := procMapsRE.FindStringSubmatch(l); len(me) == 6 { + start, end, perm, offset, file = me[1], me[2], me[3], me[4], me[5] + } else if me := briefMapsRE.FindStringSubmatch(l); len(me) == 7 { + start, end, perm, file, offset, buildID = me[1], me[2], me[3], me[4], me[5], me[6] + } else { + return nil, errUnrecognized + } + + var err error + mapping := &Mapping{ + File: file, + BuildID: buildID, + } + if perm != "" && !strings.Contains(perm, "x") { + // Skip non-executable entries. + return nil, nil + } + if mapping.Start, err = strconv.ParseUint(start, 16, 64); err != nil { + return nil, errUnrecognized + } + if mapping.Limit, err = strconv.ParseUint(end, 16, 64); err != nil { + return nil, errUnrecognized + } + if offset != "" { + if mapping.Offset, err = strconv.ParseUint(offset, 16, 64); err != nil { + return nil, errUnrecognized + } + } + return mapping, nil +} + +var memoryMapSentinels = []string{ + "--- Memory map: ---", + "MAPPED_LIBRARIES:", +} + +// isMemoryMapSentinel returns true if the string contains one of the +// known sentinels for memory map information. +func isMemoryMapSentinel(line string) bool { + for _, s := range memoryMapSentinels { + if strings.Contains(line, s) { + return true + } + } + return false +} + +func (p *Profile) addLegacyFrameInfo() { + switch { + case isProfileType(p, heapzSampleTypes): + p.DropFrames, p.KeepFrames = allocRxStr, allocSkipRxStr + case isProfileType(p, contentionzSampleTypes): + p.DropFrames, p.KeepFrames = lockRxStr, "" + default: + p.DropFrames, p.KeepFrames = cpuProfilerRxStr, "" + } +} + +var heapzSampleTypes = [][]string{ + {"allocations", "size"}, // early Go pprof profiles + {"objects", "space"}, + {"inuse_objects", "inuse_space"}, + {"alloc_objects", "alloc_space"}, + {"alloc_objects", "alloc_space", "inuse_objects", "inuse_space"}, // Go pprof legacy profiles +} +var contentionzSampleTypes = [][]string{ + {"contentions", "delay"}, +} + +func isProfileType(p *Profile, types [][]string) bool { + st := p.SampleType +nextType: + for _, t := range types { + if len(st) != len(t) { + continue + } + + for i := range st { + if st[i].Type != t[i] { + continue nextType + } + } + return true + } + return false +} + +var allocRxStr = strings.Join([]string{ + // POSIX entry points. + `calloc`, + `cfree`, + `malloc`, + `free`, + `memalign`, + `do_memalign`, + `(__)?posix_memalign`, + `pvalloc`, + `valloc`, + `realloc`, + + // TC malloc. + `tcmalloc::.*`, + `tc_calloc`, + `tc_cfree`, + `tc_malloc`, + `tc_free`, + `tc_memalign`, + `tc_posix_memalign`, + `tc_pvalloc`, + `tc_valloc`, + `tc_realloc`, + `tc_new`, + `tc_delete`, + `tc_newarray`, + `tc_deletearray`, + `tc_new_nothrow`, + `tc_newarray_nothrow`, + + // Memory-allocation routines on OS X. + `malloc_zone_malloc`, + `malloc_zone_calloc`, + `malloc_zone_valloc`, + `malloc_zone_realloc`, + `malloc_zone_memalign`, + `malloc_zone_free`, + + // Go runtime + `runtime\..*`, + + // Other misc. memory allocation routines + `BaseArena::.*`, + `(::)?do_malloc_no_errno`, + `(::)?do_malloc_pages`, + `(::)?do_malloc`, + `DoSampledAllocation`, + `MallocedMemBlock::MallocedMemBlock`, + `_M_allocate`, + `__builtin_(vec_)?delete`, + `__builtin_(vec_)?new`, + `__gnu_cxx::new_allocator::allocate`, + `__libc_malloc`, + `__malloc_alloc_template::allocate`, + `allocate`, + `cpp_alloc`, + `operator new(\[\])?`, + `simple_alloc::allocate`, +}, `|`) + +var allocSkipRxStr = strings.Join([]string{ + // Preserve Go runtime frames that appear in the middle/bottom of + // the stack. + `runtime\.panic`, + `runtime\.reflectcall`, + `runtime\.call[0-9]*`, +}, `|`) + +var cpuProfilerRxStr = strings.Join([]string{ + `ProfileData::Add`, + `ProfileData::prof_handler`, + `CpuProfiler::prof_handler`, + `__pthread_sighandler`, + `__restore`, +}, `|`) + +var lockRxStr = strings.Join([]string{ + `RecordLockProfileData`, + `(base::)?RecordLockProfileData.*`, + `(base::)?SubmitMutexProfileData.*`, + `(base::)?SubmitSpinLockProfileData.*`, + `(base::Mutex::)?AwaitCommon.*`, + `(base::Mutex::)?Unlock.*`, + `(base::Mutex::)?UnlockSlow.*`, + `(base::Mutex::)?ReaderUnlock.*`, + `(base::MutexLock::)?~MutexLock.*`, + `(Mutex::)?AwaitCommon.*`, + `(Mutex::)?Unlock.*`, + `(Mutex::)?UnlockSlow.*`, + `(Mutex::)?ReaderUnlock.*`, + `(MutexLock::)?~MutexLock.*`, + `(SpinLock::)?Unlock.*`, + `(SpinLock::)?SlowUnlock.*`, + `(SpinLockHolder::)?~SpinLockHolder.*`, +}, `|`) diff --git a/vendor/github.com/google/pprof/profile/merge.go b/vendor/github.com/google/pprof/profile/merge.go new file mode 100644 index 000000000..90c5723d7 --- /dev/null +++ b/vendor/github.com/google/pprof/profile/merge.go @@ -0,0 +1,568 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +import ( + "encoding/binary" + "fmt" + "sort" + "strconv" + "strings" +) + +// Compact performs garbage collection on a profile to remove any +// unreferenced fields. This is useful to reduce the size of a profile +// after samples or locations have been removed. +func (p *Profile) Compact() *Profile { + p, _ = Merge([]*Profile{p}) + return p +} + +// Merge merges all the profiles in profs into a single Profile. +// Returns a new profile independent of the input profiles. The merged +// profile is compacted to eliminate unused samples, locations, +// functions and mappings. Profiles must have identical profile sample +// and period types or the merge will fail. profile.Period of the +// resulting profile will be the maximum of all profiles, and +// profile.TimeNanos will be the earliest nonzero one. Merges are +// associative with the caveat of the first profile having some +// specialization in how headers are combined. There may be other +// subtleties now or in the future regarding associativity. +func Merge(srcs []*Profile) (*Profile, error) { + if len(srcs) == 0 { + return nil, fmt.Errorf("no profiles to merge") + } + p, err := combineHeaders(srcs) + if err != nil { + return nil, err + } + + pm := &profileMerger{ + p: p, + samples: make(map[sampleKey]*Sample, len(srcs[0].Sample)), + locations: make(map[locationKey]*Location, len(srcs[0].Location)), + functions: make(map[functionKey]*Function, len(srcs[0].Function)), + mappings: make(map[mappingKey]*Mapping, len(srcs[0].Mapping)), + } + + for _, src := range srcs { + // Clear the profile-specific hash tables + pm.locationsByID = makeLocationIDMap(len(src.Location)) + pm.functionsByID = make(map[uint64]*Function, len(src.Function)) + pm.mappingsByID = make(map[uint64]mapInfo, len(src.Mapping)) + + if len(pm.mappings) == 0 && len(src.Mapping) > 0 { + // The Mapping list has the property that the first mapping + // represents the main binary. Take the first Mapping we see, + // otherwise the operations below will add mappings in an + // arbitrary order. + pm.mapMapping(src.Mapping[0]) + } + + for _, s := range src.Sample { + if !isZeroSample(s) { + pm.mapSample(s) + } + } + } + + for _, s := range p.Sample { + if isZeroSample(s) { + // If there are any zero samples, re-merge the profile to GC + // them. + return Merge([]*Profile{p}) + } + } + + return p, nil +} + +// Normalize normalizes the source profile by multiplying each value in profile by the +// ratio of the sum of the base profile's values of that sample type to the sum of the +// source profile's value of that sample type. +func (p *Profile) Normalize(pb *Profile) error { + + if err := p.compatible(pb); err != nil { + return err + } + + baseVals := make([]int64, len(p.SampleType)) + for _, s := range pb.Sample { + for i, v := range s.Value { + baseVals[i] += v + } + } + + srcVals := make([]int64, len(p.SampleType)) + for _, s := range p.Sample { + for i, v := range s.Value { + srcVals[i] += v + } + } + + normScale := make([]float64, len(baseVals)) + for i := range baseVals { + if srcVals[i] == 0 { + normScale[i] = 0.0 + } else { + normScale[i] = float64(baseVals[i]) / float64(srcVals[i]) + } + } + p.ScaleN(normScale) + return nil +} + +func isZeroSample(s *Sample) bool { + for _, v := range s.Value { + if v != 0 { + return false + } + } + return true +} + +type profileMerger struct { + p *Profile + + // Memoization tables within a profile. + locationsByID locationIDMap + functionsByID map[uint64]*Function + mappingsByID map[uint64]mapInfo + + // Memoization tables for profile entities. + samples map[sampleKey]*Sample + locations map[locationKey]*Location + functions map[functionKey]*Function + mappings map[mappingKey]*Mapping +} + +type mapInfo struct { + m *Mapping + offset int64 +} + +func (pm *profileMerger) mapSample(src *Sample) *Sample { + // Check memoization table + k := pm.sampleKey(src) + if ss, ok := pm.samples[k]; ok { + for i, v := range src.Value { + ss.Value[i] += v + } + return ss + } + + // Make new sample. + s := &Sample{ + Location: make([]*Location, len(src.Location)), + Value: make([]int64, len(src.Value)), + Label: make(map[string][]string, len(src.Label)), + NumLabel: make(map[string][]int64, len(src.NumLabel)), + NumUnit: make(map[string][]string, len(src.NumLabel)), + } + for i, l := range src.Location { + s.Location[i] = pm.mapLocation(l) + } + for k, v := range src.Label { + vv := make([]string, len(v)) + copy(vv, v) + s.Label[k] = vv + } + for k, v := range src.NumLabel { + u := src.NumUnit[k] + vv := make([]int64, len(v)) + uu := make([]string, len(u)) + copy(vv, v) + copy(uu, u) + s.NumLabel[k] = vv + s.NumUnit[k] = uu + } + copy(s.Value, src.Value) + pm.samples[k] = s + pm.p.Sample = append(pm.p.Sample, s) + return s +} + +func (pm *profileMerger) sampleKey(sample *Sample) sampleKey { + // Accumulate contents into a string. + var buf strings.Builder + buf.Grow(64) // Heuristic to avoid extra allocs + + // encode a number + putNumber := func(v uint64) { + var num [binary.MaxVarintLen64]byte + n := binary.PutUvarint(num[:], v) + buf.Write(num[:n]) + } + + // encode a string prefixed with its length. + putDelimitedString := func(s string) { + putNumber(uint64(len(s))) + buf.WriteString(s) + } + + for _, l := range sample.Location { + // Get the location in the merged profile, which may have a different ID. + if loc := pm.mapLocation(l); loc != nil { + putNumber(loc.ID) + } + } + putNumber(0) // Delimiter + + for _, l := range sortedKeys1(sample.Label) { + putDelimitedString(l) + values := sample.Label[l] + putNumber(uint64(len(values))) + for _, v := range values { + putDelimitedString(v) + } + } + + for _, l := range sortedKeys2(sample.NumLabel) { + putDelimitedString(l) + values := sample.NumLabel[l] + putNumber(uint64(len(values))) + for _, v := range values { + putNumber(uint64(v)) + } + units := sample.NumUnit[l] + putNumber(uint64(len(units))) + for _, v := range units { + putDelimitedString(v) + } + } + + return sampleKey(buf.String()) +} + +type sampleKey string + +// sortedKeys1 returns the sorted keys found in a string->[]string map. +// +// Note: this is currently non-generic since github pprof runs golint, +// which does not support generics. When that issue is fixed, it can +// be merged with sortedKeys2 and made into a generic function. +func sortedKeys1(m map[string][]string) []string { + if len(m) == 0 { + return nil + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// sortedKeys2 returns the sorted keys found in a string->[]int64 map. +// +// Note: this is currently non-generic since github pprof runs golint, +// which does not support generics. When that issue is fixed, it can +// be merged with sortedKeys1 and made into a generic function. +func sortedKeys2(m map[string][]int64) []string { + if len(m) == 0 { + return nil + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func (pm *profileMerger) mapLocation(src *Location) *Location { + if src == nil { + return nil + } + + if l := pm.locationsByID.get(src.ID); l != nil { + return l + } + + mi := pm.mapMapping(src.Mapping) + l := &Location{ + ID: uint64(len(pm.p.Location) + 1), + Mapping: mi.m, + Address: uint64(int64(src.Address) + mi.offset), + Line: make([]Line, len(src.Line)), + IsFolded: src.IsFolded, + } + for i, ln := range src.Line { + l.Line[i] = pm.mapLine(ln) + } + // Check memoization table. Must be done on the remapped location to + // account for the remapped mapping ID. + k := l.key() + if ll, ok := pm.locations[k]; ok { + pm.locationsByID.set(src.ID, ll) + return ll + } + pm.locationsByID.set(src.ID, l) + pm.locations[k] = l + pm.p.Location = append(pm.p.Location, l) + return l +} + +// key generates locationKey to be used as a key for maps. +func (l *Location) key() locationKey { + key := locationKey{ + addr: l.Address, + isFolded: l.IsFolded, + } + if l.Mapping != nil { + // Normalizes address to handle address space randomization. + key.addr -= l.Mapping.Start + key.mappingID = l.Mapping.ID + } + lines := make([]string, len(l.Line)*2) + for i, line := range l.Line { + if line.Function != nil { + lines[i*2] = strconv.FormatUint(line.Function.ID, 16) + } + lines[i*2+1] = strconv.FormatInt(line.Line, 16) + } + key.lines = strings.Join(lines, "|") + return key +} + +type locationKey struct { + addr, mappingID uint64 + lines string + isFolded bool +} + +func (pm *profileMerger) mapMapping(src *Mapping) mapInfo { + if src == nil { + return mapInfo{} + } + + if mi, ok := pm.mappingsByID[src.ID]; ok { + return mi + } + + // Check memoization tables. + mk := src.key() + if m, ok := pm.mappings[mk]; ok { + mi := mapInfo{m, int64(m.Start) - int64(src.Start)} + pm.mappingsByID[src.ID] = mi + return mi + } + m := &Mapping{ + ID: uint64(len(pm.p.Mapping) + 1), + Start: src.Start, + Limit: src.Limit, + Offset: src.Offset, + File: src.File, + KernelRelocationSymbol: src.KernelRelocationSymbol, + BuildID: src.BuildID, + HasFunctions: src.HasFunctions, + HasFilenames: src.HasFilenames, + HasLineNumbers: src.HasLineNumbers, + HasInlineFrames: src.HasInlineFrames, + } + pm.p.Mapping = append(pm.p.Mapping, m) + + // Update memoization tables. + pm.mappings[mk] = m + mi := mapInfo{m, 0} + pm.mappingsByID[src.ID] = mi + return mi +} + +// key generates encoded strings of Mapping to be used as a key for +// maps. +func (m *Mapping) key() mappingKey { + // Normalize addresses to handle address space randomization. + // Round up to next 4K boundary to avoid minor discrepancies. + const mapsizeRounding = 0x1000 + + size := m.Limit - m.Start + size = size + mapsizeRounding - 1 + size = size - (size % mapsizeRounding) + key := mappingKey{ + size: size, + offset: m.Offset, + } + + switch { + case m.BuildID != "": + key.buildIDOrFile = m.BuildID + case m.File != "": + key.buildIDOrFile = m.File + default: + // A mapping containing neither build ID nor file name is a fake mapping. A + // key with empty buildIDOrFile is used for fake mappings so that they are + // treated as the same mapping during merging. + } + return key +} + +type mappingKey struct { + size, offset uint64 + buildIDOrFile string +} + +func (pm *profileMerger) mapLine(src Line) Line { + ln := Line{ + Function: pm.mapFunction(src.Function), + Line: src.Line, + } + return ln +} + +func (pm *profileMerger) mapFunction(src *Function) *Function { + if src == nil { + return nil + } + if f, ok := pm.functionsByID[src.ID]; ok { + return f + } + k := src.key() + if f, ok := pm.functions[k]; ok { + pm.functionsByID[src.ID] = f + return f + } + f := &Function{ + ID: uint64(len(pm.p.Function) + 1), + Name: src.Name, + SystemName: src.SystemName, + Filename: src.Filename, + StartLine: src.StartLine, + } + pm.functions[k] = f + pm.functionsByID[src.ID] = f + pm.p.Function = append(pm.p.Function, f) + return f +} + +// key generates a struct to be used as a key for maps. +func (f *Function) key() functionKey { + return functionKey{ + f.StartLine, + f.Name, + f.SystemName, + f.Filename, + } +} + +type functionKey struct { + startLine int64 + name, systemName, fileName string +} + +// combineHeaders checks that all profiles can be merged and returns +// their combined profile. +func combineHeaders(srcs []*Profile) (*Profile, error) { + for _, s := range srcs[1:] { + if err := srcs[0].compatible(s); err != nil { + return nil, err + } + } + + var timeNanos, durationNanos, period int64 + var comments []string + seenComments := map[string]bool{} + var defaultSampleType string + for _, s := range srcs { + if timeNanos == 0 || s.TimeNanos < timeNanos { + timeNanos = s.TimeNanos + } + durationNanos += s.DurationNanos + if period == 0 || period < s.Period { + period = s.Period + } + for _, c := range s.Comments { + if seen := seenComments[c]; !seen { + comments = append(comments, c) + seenComments[c] = true + } + } + if defaultSampleType == "" { + defaultSampleType = s.DefaultSampleType + } + } + + p := &Profile{ + SampleType: make([]*ValueType, len(srcs[0].SampleType)), + + DropFrames: srcs[0].DropFrames, + KeepFrames: srcs[0].KeepFrames, + + TimeNanos: timeNanos, + DurationNanos: durationNanos, + PeriodType: srcs[0].PeriodType, + Period: period, + + Comments: comments, + DefaultSampleType: defaultSampleType, + } + copy(p.SampleType, srcs[0].SampleType) + return p, nil +} + +// compatible determines if two profiles can be compared/merged. +// returns nil if the profiles are compatible; otherwise an error with +// details on the incompatibility. +func (p *Profile) compatible(pb *Profile) error { + if !equalValueType(p.PeriodType, pb.PeriodType) { + return fmt.Errorf("incompatible period types %v and %v", p.PeriodType, pb.PeriodType) + } + + if len(p.SampleType) != len(pb.SampleType) { + return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) + } + + for i := range p.SampleType { + if !equalValueType(p.SampleType[i], pb.SampleType[i]) { + return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) + } + } + return nil +} + +// equalValueType returns true if the two value types are semantically +// equal. It ignores the internal fields used during encode/decode. +func equalValueType(st1, st2 *ValueType) bool { + return st1.Type == st2.Type && st1.Unit == st2.Unit +} + +// locationIDMap is like a map[uint64]*Location, but provides efficiency for +// ids that are densely numbered, which is often the case. +type locationIDMap struct { + dense []*Location // indexed by id for id < len(dense) + sparse map[uint64]*Location // indexed by id for id >= len(dense) +} + +func makeLocationIDMap(n int) locationIDMap { + return locationIDMap{ + dense: make([]*Location, n), + sparse: map[uint64]*Location{}, + } +} + +func (lm locationIDMap) get(id uint64) *Location { + if id < uint64(len(lm.dense)) { + return lm.dense[int(id)] + } + return lm.sparse[id] +} + +func (lm locationIDMap) set(id uint64, loc *Location) { + if id < uint64(len(lm.dense)) { + lm.dense[id] = loc + return + } + lm.sparse[id] = loc +} diff --git a/vendor/github.com/google/pprof/profile/profile.go b/vendor/github.com/google/pprof/profile/profile.go new file mode 100644 index 000000000..4ec00fe7d --- /dev/null +++ b/vendor/github.com/google/pprof/profile/profile.go @@ -0,0 +1,813 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package profile provides a representation of profile.proto and +// methods to encode/decode profiles in this format. +package profile + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "math" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" +) + +// Profile is an in-memory representation of profile.proto. +type Profile struct { + SampleType []*ValueType + DefaultSampleType string + Sample []*Sample + Mapping []*Mapping + Location []*Location + Function []*Function + Comments []string + + DropFrames string + KeepFrames string + + TimeNanos int64 + DurationNanos int64 + PeriodType *ValueType + Period int64 + + // The following fields are modified during encoding and copying, + // so are protected by a Mutex. + encodeMu sync.Mutex + + commentX []int64 + dropFramesX int64 + keepFramesX int64 + stringTable []string + defaultSampleTypeX int64 +} + +// ValueType corresponds to Profile.ValueType +type ValueType struct { + Type string // cpu, wall, inuse_space, etc + Unit string // seconds, nanoseconds, bytes, etc + + typeX int64 + unitX int64 +} + +// Sample corresponds to Profile.Sample +type Sample struct { + Location []*Location + Value []int64 + Label map[string][]string + NumLabel map[string][]int64 + NumUnit map[string][]string + + locationIDX []uint64 + labelX []label +} + +// label corresponds to Profile.Label +type label struct { + keyX int64 + // Exactly one of the two following values must be set + strX int64 + numX int64 // Integer value for this label + // can be set if numX has value + unitX int64 +} + +// Mapping corresponds to Profile.Mapping +type Mapping struct { + ID uint64 + Start uint64 + Limit uint64 + Offset uint64 + File string + BuildID string + HasFunctions bool + HasFilenames bool + HasLineNumbers bool + HasInlineFrames bool + + fileX int64 + buildIDX int64 + + // Name of the kernel relocation symbol ("_text" or "_stext"), extracted from File. + // For linux kernel mappings generated by some tools, correct symbolization depends + // on knowing which of the two possible relocation symbols was used for `Start`. + // This is given to us as a suffix in `File` (e.g. "[kernel.kallsyms]_stext"). + // + // Note, this public field is not persisted in the proto. For the purposes of + // copying / merging / hashing profiles, it is considered subsumed by `File`. + KernelRelocationSymbol string +} + +// Location corresponds to Profile.Location +type Location struct { + ID uint64 + Mapping *Mapping + Address uint64 + Line []Line + IsFolded bool + + mappingIDX uint64 +} + +// Line corresponds to Profile.Line +type Line struct { + Function *Function + Line int64 + + functionIDX uint64 +} + +// Function corresponds to Profile.Function +type Function struct { + ID uint64 + Name string + SystemName string + Filename string + StartLine int64 + + nameX int64 + systemNameX int64 + filenameX int64 +} + +// Parse parses a profile and checks for its validity. The input +// may be a gzip-compressed encoded protobuf or one of many legacy +// profile formats which may be unsupported in the future. +func Parse(r io.Reader) (*Profile, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + return ParseData(data) +} + +// ParseData parses a profile from a buffer and checks for its +// validity. +func ParseData(data []byte) (*Profile, error) { + var p *Profile + var err error + if len(data) >= 2 && data[0] == 0x1f && data[1] == 0x8b { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err == nil { + data, err = io.ReadAll(gz) + } + if err != nil { + return nil, fmt.Errorf("decompressing profile: %v", err) + } + } + if p, err = ParseUncompressed(data); err != nil && err != errNoData && err != errConcatProfile { + p, err = parseLegacy(data) + } + + if err != nil { + return nil, fmt.Errorf("parsing profile: %v", err) + } + + if err := p.CheckValid(); err != nil { + return nil, fmt.Errorf("malformed profile: %v", err) + } + return p, nil +} + +var errUnrecognized = fmt.Errorf("unrecognized profile format") +var errMalformed = fmt.Errorf("malformed profile format") +var errNoData = fmt.Errorf("empty input file") +var errConcatProfile = fmt.Errorf("concatenated profiles detected") + +func parseLegacy(data []byte) (*Profile, error) { + parsers := []func([]byte) (*Profile, error){ + parseCPU, + parseHeap, + parseGoCount, // goroutine, threadcreate + parseThread, + parseContention, + parseJavaProfile, + } + + for _, parser := range parsers { + p, err := parser(data) + if err == nil { + p.addLegacyFrameInfo() + return p, nil + } + if err != errUnrecognized { + return nil, err + } + } + return nil, errUnrecognized +} + +// ParseUncompressed parses an uncompressed protobuf into a profile. +func ParseUncompressed(data []byte) (*Profile, error) { + if len(data) == 0 { + return nil, errNoData + } + p := &Profile{} + if err := unmarshal(data, p); err != nil { + return nil, err + } + + if err := p.postDecode(); err != nil { + return nil, err + } + + return p, nil +} + +var libRx = regexp.MustCompile(`([.]so$|[.]so[._][0-9]+)`) + +// massageMappings applies heuristic-based changes to the profile +// mappings to account for quirks of some environments. +func (p *Profile) massageMappings() { + // Merge adjacent regions with matching names, checking that the offsets match + if len(p.Mapping) > 1 { + mappings := []*Mapping{p.Mapping[0]} + for _, m := range p.Mapping[1:] { + lm := mappings[len(mappings)-1] + if adjacent(lm, m) { + lm.Limit = m.Limit + if m.File != "" { + lm.File = m.File + } + if m.BuildID != "" { + lm.BuildID = m.BuildID + } + p.updateLocationMapping(m, lm) + continue + } + mappings = append(mappings, m) + } + p.Mapping = mappings + } + + // Use heuristics to identify main binary and move it to the top of the list of mappings + for i, m := range p.Mapping { + file := strings.TrimSpace(strings.Replace(m.File, "(deleted)", "", -1)) + if len(file) == 0 { + continue + } + if len(libRx.FindStringSubmatch(file)) > 0 { + continue + } + if file[0] == '[' { + continue + } + // Swap what we guess is main to position 0. + p.Mapping[0], p.Mapping[i] = p.Mapping[i], p.Mapping[0] + break + } + + // Keep the mapping IDs neatly sorted + for i, m := range p.Mapping { + m.ID = uint64(i + 1) + } +} + +// adjacent returns whether two mapping entries represent the same +// mapping that has been split into two. Check that their addresses are adjacent, +// and if the offsets match, if they are available. +func adjacent(m1, m2 *Mapping) bool { + if m1.File != "" && m2.File != "" { + if m1.File != m2.File { + return false + } + } + if m1.BuildID != "" && m2.BuildID != "" { + if m1.BuildID != m2.BuildID { + return false + } + } + if m1.Limit != m2.Start { + return false + } + if m1.Offset != 0 && m2.Offset != 0 { + offset := m1.Offset + (m1.Limit - m1.Start) + if offset != m2.Offset { + return false + } + } + return true +} + +func (p *Profile) updateLocationMapping(from, to *Mapping) { + for _, l := range p.Location { + if l.Mapping == from { + l.Mapping = to + } + } +} + +func serialize(p *Profile) []byte { + p.encodeMu.Lock() + p.preEncode() + b := marshal(p) + p.encodeMu.Unlock() + return b +} + +// Write writes the profile as a gzip-compressed marshaled protobuf. +func (p *Profile) Write(w io.Writer) error { + zw := gzip.NewWriter(w) + defer zw.Close() + _, err := zw.Write(serialize(p)) + return err +} + +// WriteUncompressed writes the profile as a marshaled protobuf. +func (p *Profile) WriteUncompressed(w io.Writer) error { + _, err := w.Write(serialize(p)) + return err +} + +// CheckValid tests whether the profile is valid. Checks include, but are +// not limited to: +// - len(Profile.Sample[n].value) == len(Profile.value_unit) +// - Sample.id has a corresponding Profile.Location +func (p *Profile) CheckValid() error { + // Check that sample values are consistent + sampleLen := len(p.SampleType) + if sampleLen == 0 && len(p.Sample) != 0 { + return fmt.Errorf("missing sample type information") + } + for _, s := range p.Sample { + if s == nil { + return fmt.Errorf("profile has nil sample") + } + if len(s.Value) != sampleLen { + return fmt.Errorf("mismatch: sample has %d values vs. %d types", len(s.Value), len(p.SampleType)) + } + for _, l := range s.Location { + if l == nil { + return fmt.Errorf("sample has nil location") + } + } + } + + // Check that all mappings/locations/functions are in the tables + // Check that there are no duplicate ids + mappings := make(map[uint64]*Mapping, len(p.Mapping)) + for _, m := range p.Mapping { + if m == nil { + return fmt.Errorf("profile has nil mapping") + } + if m.ID == 0 { + return fmt.Errorf("found mapping with reserved ID=0") + } + if mappings[m.ID] != nil { + return fmt.Errorf("multiple mappings with same id: %d", m.ID) + } + mappings[m.ID] = m + } + functions := make(map[uint64]*Function, len(p.Function)) + for _, f := range p.Function { + if f == nil { + return fmt.Errorf("profile has nil function") + } + if f.ID == 0 { + return fmt.Errorf("found function with reserved ID=0") + } + if functions[f.ID] != nil { + return fmt.Errorf("multiple functions with same id: %d", f.ID) + } + functions[f.ID] = f + } + locations := make(map[uint64]*Location, len(p.Location)) + for _, l := range p.Location { + if l == nil { + return fmt.Errorf("profile has nil location") + } + if l.ID == 0 { + return fmt.Errorf("found location with reserved id=0") + } + if locations[l.ID] != nil { + return fmt.Errorf("multiple locations with same id: %d", l.ID) + } + locations[l.ID] = l + if m := l.Mapping; m != nil { + if m.ID == 0 || mappings[m.ID] != m { + return fmt.Errorf("inconsistent mapping %p: %d", m, m.ID) + } + } + for _, ln := range l.Line { + f := ln.Function + if f == nil { + return fmt.Errorf("location id: %d has a line with nil function", l.ID) + } + if f.ID == 0 || functions[f.ID] != f { + return fmt.Errorf("inconsistent function %p: %d", f, f.ID) + } + } + } + return nil +} + +// Aggregate merges the locations in the profile into equivalence +// classes preserving the request attributes. It also updates the +// samples to point to the merged locations. +func (p *Profile) Aggregate(inlineFrame, function, filename, linenumber, address bool) error { + for _, m := range p.Mapping { + m.HasInlineFrames = m.HasInlineFrames && inlineFrame + m.HasFunctions = m.HasFunctions && function + m.HasFilenames = m.HasFilenames && filename + m.HasLineNumbers = m.HasLineNumbers && linenumber + } + + // Aggregate functions + if !function || !filename { + for _, f := range p.Function { + if !function { + f.Name = "" + f.SystemName = "" + } + if !filename { + f.Filename = "" + } + } + } + + // Aggregate locations + if !inlineFrame || !address || !linenumber { + for _, l := range p.Location { + if !inlineFrame && len(l.Line) > 1 { + l.Line = l.Line[len(l.Line)-1:] + } + if !linenumber { + for i := range l.Line { + l.Line[i].Line = 0 + } + } + if !address { + l.Address = 0 + } + } + } + + return p.CheckValid() +} + +// NumLabelUnits returns a map of numeric label keys to the units +// associated with those keys and a map of those keys to any units +// that were encountered but not used. +// Unit for a given key is the first encountered unit for that key. If multiple +// units are encountered for values paired with a particular key, then the first +// unit encountered is used and all other units are returned in sorted order +// in map of ignored units. +// If no units are encountered for a particular key, the unit is then inferred +// based on the key. +func (p *Profile) NumLabelUnits() (map[string]string, map[string][]string) { + numLabelUnits := map[string]string{} + ignoredUnits := map[string]map[string]bool{} + encounteredKeys := map[string]bool{} + + // Determine units based on numeric tags for each sample. + for _, s := range p.Sample { + for k := range s.NumLabel { + encounteredKeys[k] = true + for _, unit := range s.NumUnit[k] { + if unit == "" { + continue + } + if wantUnit, ok := numLabelUnits[k]; !ok { + numLabelUnits[k] = unit + } else if wantUnit != unit { + if v, ok := ignoredUnits[k]; ok { + v[unit] = true + } else { + ignoredUnits[k] = map[string]bool{unit: true} + } + } + } + } + } + // Infer units for keys without any units associated with + // numeric tag values. + for key := range encounteredKeys { + unit := numLabelUnits[key] + if unit == "" { + switch key { + case "alignment", "request": + numLabelUnits[key] = "bytes" + default: + numLabelUnits[key] = key + } + } + } + + // Copy ignored units into more readable format + unitsIgnored := make(map[string][]string, len(ignoredUnits)) + for key, values := range ignoredUnits { + units := make([]string, len(values)) + i := 0 + for unit := range values { + units[i] = unit + i++ + } + sort.Strings(units) + unitsIgnored[key] = units + } + + return numLabelUnits, unitsIgnored +} + +// String dumps a text representation of a profile. Intended mainly +// for debugging purposes. +func (p *Profile) String() string { + ss := make([]string, 0, len(p.Comments)+len(p.Sample)+len(p.Mapping)+len(p.Location)) + for _, c := range p.Comments { + ss = append(ss, "Comment: "+c) + } + if pt := p.PeriodType; pt != nil { + ss = append(ss, fmt.Sprintf("PeriodType: %s %s", pt.Type, pt.Unit)) + } + ss = append(ss, fmt.Sprintf("Period: %d", p.Period)) + if p.TimeNanos != 0 { + ss = append(ss, fmt.Sprintf("Time: %v", time.Unix(0, p.TimeNanos))) + } + if p.DurationNanos != 0 { + ss = append(ss, fmt.Sprintf("Duration: %.4v", time.Duration(p.DurationNanos))) + } + + ss = append(ss, "Samples:") + var sh1 string + for _, s := range p.SampleType { + dflt := "" + if s.Type == p.DefaultSampleType { + dflt = "[dflt]" + } + sh1 = sh1 + fmt.Sprintf("%s/%s%s ", s.Type, s.Unit, dflt) + } + ss = append(ss, strings.TrimSpace(sh1)) + for _, s := range p.Sample { + ss = append(ss, s.string()) + } + + ss = append(ss, "Locations") + for _, l := range p.Location { + ss = append(ss, l.string()) + } + + ss = append(ss, "Mappings") + for _, m := range p.Mapping { + ss = append(ss, m.string()) + } + + return strings.Join(ss, "\n") + "\n" +} + +// string dumps a text representation of a mapping. Intended mainly +// for debugging purposes. +func (m *Mapping) string() string { + bits := "" + if m.HasFunctions { + bits = bits + "[FN]" + } + if m.HasFilenames { + bits = bits + "[FL]" + } + if m.HasLineNumbers { + bits = bits + "[LN]" + } + if m.HasInlineFrames { + bits = bits + "[IN]" + } + return fmt.Sprintf("%d: %#x/%#x/%#x %s %s %s", + m.ID, + m.Start, m.Limit, m.Offset, + m.File, + m.BuildID, + bits) +} + +// string dumps a text representation of a location. Intended mainly +// for debugging purposes. +func (l *Location) string() string { + ss := []string{} + locStr := fmt.Sprintf("%6d: %#x ", l.ID, l.Address) + if m := l.Mapping; m != nil { + locStr = locStr + fmt.Sprintf("M=%d ", m.ID) + } + if l.IsFolded { + locStr = locStr + "[F] " + } + if len(l.Line) == 0 { + ss = append(ss, locStr) + } + for li := range l.Line { + lnStr := "??" + if fn := l.Line[li].Function; fn != nil { + lnStr = fmt.Sprintf("%s %s:%d s=%d", + fn.Name, + fn.Filename, + l.Line[li].Line, + fn.StartLine) + if fn.Name != fn.SystemName { + lnStr = lnStr + "(" + fn.SystemName + ")" + } + } + ss = append(ss, locStr+lnStr) + // Do not print location details past the first line + locStr = " " + } + return strings.Join(ss, "\n") +} + +// string dumps a text representation of a sample. Intended mainly +// for debugging purposes. +func (s *Sample) string() string { + ss := []string{} + var sv string + for _, v := range s.Value { + sv = fmt.Sprintf("%s %10d", sv, v) + } + sv = sv + ": " + for _, l := range s.Location { + sv = sv + fmt.Sprintf("%d ", l.ID) + } + ss = append(ss, sv) + const labelHeader = " " + if len(s.Label) > 0 { + ss = append(ss, labelHeader+labelsToString(s.Label)) + } + if len(s.NumLabel) > 0 { + ss = append(ss, labelHeader+numLabelsToString(s.NumLabel, s.NumUnit)) + } + return strings.Join(ss, "\n") +} + +// labelsToString returns a string representation of a +// map representing labels. +func labelsToString(labels map[string][]string) string { + ls := []string{} + for k, v := range labels { + ls = append(ls, fmt.Sprintf("%s:%v", k, v)) + } + sort.Strings(ls) + return strings.Join(ls, " ") +} + +// numLabelsToString returns a string representation of a map +// representing numeric labels. +func numLabelsToString(numLabels map[string][]int64, numUnits map[string][]string) string { + ls := []string{} + for k, v := range numLabels { + units := numUnits[k] + var labelString string + if len(units) == len(v) { + values := make([]string, len(v)) + for i, vv := range v { + values[i] = fmt.Sprintf("%d %s", vv, units[i]) + } + labelString = fmt.Sprintf("%s:%v", k, values) + } else { + labelString = fmt.Sprintf("%s:%v", k, v) + } + ls = append(ls, labelString) + } + sort.Strings(ls) + return strings.Join(ls, " ") +} + +// SetLabel sets the specified key to the specified value for all samples in the +// profile. +func (p *Profile) SetLabel(key string, value []string) { + for _, sample := range p.Sample { + if sample.Label == nil { + sample.Label = map[string][]string{key: value} + } else { + sample.Label[key] = value + } + } +} + +// RemoveLabel removes all labels associated with the specified key for all +// samples in the profile. +func (p *Profile) RemoveLabel(key string) { + for _, sample := range p.Sample { + delete(sample.Label, key) + } +} + +// HasLabel returns true if a sample has a label with indicated key and value. +func (s *Sample) HasLabel(key, value string) bool { + for _, v := range s.Label[key] { + if v == value { + return true + } + } + return false +} + +// DiffBaseSample returns true if a sample belongs to the diff base and false +// otherwise. +func (s *Sample) DiffBaseSample() bool { + return s.HasLabel("pprof::base", "true") +} + +// Scale multiplies all sample values in a profile by a constant and keeps +// only samples that have at least one non-zero value. +func (p *Profile) Scale(ratio float64) { + if ratio == 1 { + return + } + ratios := make([]float64, len(p.SampleType)) + for i := range p.SampleType { + ratios[i] = ratio + } + p.ScaleN(ratios) +} + +// ScaleN multiplies each sample values in a sample by a different amount +// and keeps only samples that have at least one non-zero value. +func (p *Profile) ScaleN(ratios []float64) error { + if len(p.SampleType) != len(ratios) { + return fmt.Errorf("mismatched scale ratios, got %d, want %d", len(ratios), len(p.SampleType)) + } + allOnes := true + for _, r := range ratios { + if r != 1 { + allOnes = false + break + } + } + if allOnes { + return nil + } + fillIdx := 0 + for _, s := range p.Sample { + keepSample := false + for i, v := range s.Value { + if ratios[i] != 1 { + val := int64(math.Round(float64(v) * ratios[i])) + s.Value[i] = val + keepSample = keepSample || val != 0 + } + } + if keepSample { + p.Sample[fillIdx] = s + fillIdx++ + } + } + p.Sample = p.Sample[:fillIdx] + return nil +} + +// HasFunctions determines if all locations in this profile have +// symbolized function information. +func (p *Profile) HasFunctions() bool { + for _, l := range p.Location { + if l.Mapping != nil && !l.Mapping.HasFunctions { + return false + } + } + return true +} + +// HasFileLines determines if all locations in this profile have +// symbolized file and line number information. +func (p *Profile) HasFileLines() bool { + for _, l := range p.Location { + if l.Mapping != nil && (!l.Mapping.HasFilenames || !l.Mapping.HasLineNumbers) { + return false + } + } + return true +} + +// Unsymbolizable returns true if a mapping points to a binary for which +// locations can't be symbolized in principle, at least now. Examples are +// "[vdso]", [vsyscall]" and some others, see the code. +func (m *Mapping) Unsymbolizable() bool { + name := filepath.Base(m.File) + return strings.HasPrefix(name, "[") || strings.HasPrefix(name, "linux-vdso") || strings.HasPrefix(m.File, "/dev/dri/") +} + +// Copy makes a fully independent copy of a profile. +func (p *Profile) Copy() *Profile { + pp := &Profile{} + if err := unmarshal(serialize(p), pp); err != nil { + panic(err) + } + if err := pp.postDecode(); err != nil { + panic(err) + } + + return pp +} diff --git a/vendor/github.com/google/pprof/profile/proto.go b/vendor/github.com/google/pprof/profile/proto.go new file mode 100644 index 000000000..a15696ba1 --- /dev/null +++ b/vendor/github.com/google/pprof/profile/proto.go @@ -0,0 +1,367 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is a simple protocol buffer encoder and decoder. +// The format is described at +// https://developers.google.com/protocol-buffers/docs/encoding +// +// A protocol message must implement the message interface: +// decoder() []decoder +// encode(*buffer) +// +// The decode method returns a slice indexed by field number that gives the +// function to decode that field. +// The encode method encodes its receiver into the given buffer. +// +// The two methods are simple enough to be implemented by hand rather than +// by using a protocol compiler. +// +// See profile.go for examples of messages implementing this interface. +// +// There is no support for groups, message sets, or "has" bits. + +package profile + +import ( + "errors" + "fmt" +) + +type buffer struct { + field int // field tag + typ int // proto wire type code for field + u64 uint64 + data []byte + tmp [16]byte + tmpLines []Line // temporary storage used while decoding "repeated Line". +} + +type decoder func(*buffer, message) error + +type message interface { + decoder() []decoder + encode(*buffer) +} + +func marshal(m message) []byte { + var b buffer + m.encode(&b) + return b.data +} + +func encodeVarint(b *buffer, x uint64) { + for x >= 128 { + b.data = append(b.data, byte(x)|0x80) + x >>= 7 + } + b.data = append(b.data, byte(x)) +} + +func encodeLength(b *buffer, tag int, len int) { + encodeVarint(b, uint64(tag)<<3|2) + encodeVarint(b, uint64(len)) +} + +func encodeUint64(b *buffer, tag int, x uint64) { + // append varint to b.data + encodeVarint(b, uint64(tag)<<3) + encodeVarint(b, x) +} + +func encodeUint64s(b *buffer, tag int, x []uint64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + encodeVarint(b, u) + } + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + encodeUint64(b, tag, u) + } +} + +func encodeUint64Opt(b *buffer, tag int, x uint64) { + if x == 0 { + return + } + encodeUint64(b, tag, x) +} + +func encodeInt64(b *buffer, tag int, x int64) { + u := uint64(x) + encodeUint64(b, tag, u) +} + +func encodeInt64s(b *buffer, tag int, x []int64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + encodeVarint(b, uint64(u)) + } + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + encodeInt64(b, tag, u) + } +} + +func encodeInt64Opt(b *buffer, tag int, x int64) { + if x == 0 { + return + } + encodeInt64(b, tag, x) +} + +func encodeString(b *buffer, tag int, x string) { + encodeLength(b, tag, len(x)) + b.data = append(b.data, x...) +} + +func encodeStrings(b *buffer, tag int, x []string) { + for _, s := range x { + encodeString(b, tag, s) + } +} + +func encodeBool(b *buffer, tag int, x bool) { + if x { + encodeUint64(b, tag, 1) + } else { + encodeUint64(b, tag, 0) + } +} + +func encodeBoolOpt(b *buffer, tag int, x bool) { + if x { + encodeBool(b, tag, x) + } +} + +func encodeMessage(b *buffer, tag int, m message) { + n1 := len(b.data) + m.encode(b) + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) +} + +func unmarshal(data []byte, m message) (err error) { + b := buffer{data: data, typ: 2} + return decodeMessage(&b, m) +} + +func le64(p []byte) uint64 { + return uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 +} + +func le32(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 +} + +func decodeVarint(data []byte) (uint64, []byte, error) { + var u uint64 + for i := 0; ; i++ { + if i >= 10 || i >= len(data) { + return 0, nil, errors.New("bad varint") + } + u |= uint64(data[i]&0x7F) << uint(7*i) + if data[i]&0x80 == 0 { + return u, data[i+1:], nil + } + } +} + +func decodeField(b *buffer, data []byte) ([]byte, error) { + x, data, err := decodeVarint(data) + if err != nil { + return nil, err + } + b.field = int(x >> 3) + b.typ = int(x & 7) + b.data = nil + b.u64 = 0 + switch b.typ { + case 0: + b.u64, data, err = decodeVarint(data) + if err != nil { + return nil, err + } + case 1: + if len(data) < 8 { + return nil, errors.New("not enough data") + } + b.u64 = le64(data[:8]) + data = data[8:] + case 2: + var n uint64 + n, data, err = decodeVarint(data) + if err != nil { + return nil, err + } + if n > uint64(len(data)) { + return nil, errors.New("too much data") + } + b.data = data[:n] + data = data[n:] + case 5: + if len(data) < 4 { + return nil, errors.New("not enough data") + } + b.u64 = uint64(le32(data[:4])) + data = data[4:] + default: + return nil, fmt.Errorf("unknown wire type: %d", b.typ) + } + + return data, nil +} + +func checkType(b *buffer, typ int) error { + if b.typ != typ { + return errors.New("type mismatch") + } + return nil +} + +func decodeMessage(b *buffer, m message) error { + if err := checkType(b, 2); err != nil { + return err + } + dec := m.decoder() + data := b.data + for len(data) > 0 { + // pull varint field# + type + var err error + data, err = decodeField(b, data) + if err != nil { + return err + } + if b.field >= len(dec) || dec[b.field] == nil { + continue + } + if err := dec[b.field](b, m); err != nil { + return err + } + } + return nil +} + +func decodeInt64(b *buffer, x *int64) error { + if err := checkType(b, 0); err != nil { + return err + } + *x = int64(b.u64) + return nil +} + +func decodeInt64s(b *buffer, x *[]int64) error { + if b.typ == 2 { + // Packed encoding + data := b.data + for len(data) > 0 { + var u uint64 + var err error + + if u, data, err = decodeVarint(data); err != nil { + return err + } + *x = append(*x, int64(u)) + } + return nil + } + var i int64 + if err := decodeInt64(b, &i); err != nil { + return err + } + *x = append(*x, i) + return nil +} + +func decodeUint64(b *buffer, x *uint64) error { + if err := checkType(b, 0); err != nil { + return err + } + *x = b.u64 + return nil +} + +func decodeUint64s(b *buffer, x *[]uint64) error { + if b.typ == 2 { + data := b.data + // Packed encoding + for len(data) > 0 { + var u uint64 + var err error + + if u, data, err = decodeVarint(data); err != nil { + return err + } + *x = append(*x, u) + } + return nil + } + var u uint64 + if err := decodeUint64(b, &u); err != nil { + return err + } + *x = append(*x, u) + return nil +} + +func decodeString(b *buffer, x *string) error { + if err := checkType(b, 2); err != nil { + return err + } + *x = string(b.data) + return nil +} + +func decodeStrings(b *buffer, x *[]string) error { + var s string + if err := decodeString(b, &s); err != nil { + return err + } + *x = append(*x, s) + return nil +} + +func decodeBool(b *buffer, x *bool) error { + if err := checkType(b, 0); err != nil { + return err + } + if int64(b.u64) == 0 { + *x = false + } else { + *x = true + } + return nil +} diff --git a/vendor/github.com/google/pprof/profile/prune.go b/vendor/github.com/google/pprof/profile/prune.go new file mode 100644 index 000000000..b2f9fd546 --- /dev/null +++ b/vendor/github.com/google/pprof/profile/prune.go @@ -0,0 +1,194 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Implements methods to remove frames from profiles. + +package profile + +import ( + "fmt" + "regexp" + "strings" +) + +var ( + reservedNames = []string{"(anonymous namespace)", "operator()"} + bracketRx = func() *regexp.Regexp { + var quotedNames []string + for _, name := range append(reservedNames, "(") { + quotedNames = append(quotedNames, regexp.QuoteMeta(name)) + } + return regexp.MustCompile(strings.Join(quotedNames, "|")) + }() +) + +// simplifyFunc does some primitive simplification of function names. +func simplifyFunc(f string) string { + // Account for leading '.' on the PPC ELF v1 ABI. + funcName := strings.TrimPrefix(f, ".") + // Account for unsimplified names -- try to remove the argument list by trimming + // starting from the first '(', but skipping reserved names that have '('. + for _, ind := range bracketRx.FindAllStringSubmatchIndex(funcName, -1) { + foundReserved := false + for _, res := range reservedNames { + if funcName[ind[0]:ind[1]] == res { + foundReserved = true + break + } + } + if !foundReserved { + funcName = funcName[:ind[0]] + break + } + } + return funcName +} + +// Prune removes all nodes beneath a node matching dropRx, and not +// matching keepRx. If the root node of a Sample matches, the sample +// will have an empty stack. +func (p *Profile) Prune(dropRx, keepRx *regexp.Regexp) { + prune := make(map[uint64]bool) + pruneBeneath := make(map[uint64]bool) + + // simplifyFunc can be expensive, so cache results. + // Note that the same function name can be encountered many times due + // different lines and addresses in the same function. + pruneCache := map[string]bool{} // Map from function to whether or not to prune + pruneFromHere := func(s string) bool { + if r, ok := pruneCache[s]; ok { + return r + } + funcName := simplifyFunc(s) + if dropRx.MatchString(funcName) { + if keepRx == nil || !keepRx.MatchString(funcName) { + pruneCache[s] = true + return true + } + } + pruneCache[s] = false + return false + } + + for _, loc := range p.Location { + var i int + for i = len(loc.Line) - 1; i >= 0; i-- { + if fn := loc.Line[i].Function; fn != nil && fn.Name != "" { + if pruneFromHere(fn.Name) { + break + } + } + } + + if i >= 0 { + // Found matching entry to prune. + pruneBeneath[loc.ID] = true + + // Remove the matching location. + if i == len(loc.Line)-1 { + // Matched the top entry: prune the whole location. + prune[loc.ID] = true + } else { + loc.Line = loc.Line[i+1:] + } + } + } + + // Prune locs from each Sample + for _, sample := range p.Sample { + // Scan from the root to the leaves to find the prune location. + // Do not prune frames before the first user frame, to avoid + // pruning everything. + foundUser := false + for i := len(sample.Location) - 1; i >= 0; i-- { + id := sample.Location[i].ID + if !prune[id] && !pruneBeneath[id] { + foundUser = true + continue + } + if !foundUser { + continue + } + if prune[id] { + sample.Location = sample.Location[i+1:] + break + } + if pruneBeneath[id] { + sample.Location = sample.Location[i:] + break + } + } + } +} + +// RemoveUninteresting prunes and elides profiles using built-in +// tables of uninteresting function names. +func (p *Profile) RemoveUninteresting() error { + var keep, drop *regexp.Regexp + var err error + + if p.DropFrames != "" { + if drop, err = regexp.Compile("^(" + p.DropFrames + ")$"); err != nil { + return fmt.Errorf("failed to compile regexp %s: %v", p.DropFrames, err) + } + if p.KeepFrames != "" { + if keep, err = regexp.Compile("^(" + p.KeepFrames + ")$"); err != nil { + return fmt.Errorf("failed to compile regexp %s: %v", p.KeepFrames, err) + } + } + p.Prune(drop, keep) + } + return nil +} + +// PruneFrom removes all nodes beneath the lowest node matching dropRx, not including itself. +// +// Please see the example below to understand this method as well as +// the difference from Prune method. +// +// A sample contains Location of [A,B,C,B,D] where D is the top frame and there's no inline. +// +// PruneFrom(A) returns [A,B,C,B,D] because there's no node beneath A. +// Prune(A, nil) returns [B,C,B,D] by removing A itself. +// +// PruneFrom(B) returns [B,C,B,D] by removing all nodes beneath the first B when scanning from the bottom. +// Prune(B, nil) returns [D] because a matching node is found by scanning from the root. +func (p *Profile) PruneFrom(dropRx *regexp.Regexp) { + pruneBeneath := make(map[uint64]bool) + + for _, loc := range p.Location { + for i := 0; i < len(loc.Line); i++ { + if fn := loc.Line[i].Function; fn != nil && fn.Name != "" { + funcName := simplifyFunc(fn.Name) + if dropRx.MatchString(funcName) { + // Found matching entry to prune. + pruneBeneath[loc.ID] = true + loc.Line = loc.Line[i:] + break + } + } + } + } + + // Prune locs from each Sample + for _, sample := range p.Sample { + // Scan from the bottom leaf to the root to find the prune location. + for i, loc := range sample.Location { + if pruneBeneath[loc.ID] { + sample.Location = sample.Location[i:] + break + } + } + } +} diff --git a/vendor/github.com/klauspost/compress/.gitignore b/vendor/github.com/klauspost/compress/.gitignore index b35f8449b..d31b37815 100644 --- a/vendor/github.com/klauspost/compress/.gitignore +++ b/vendor/github.com/klauspost/compress/.gitignore @@ -23,3 +23,10 @@ _testmain.go *.test *.prof /s2/cmd/_s2sx/sfx-exe + +# Linux perf files +perf.data +perf.data.old + +# gdb history +.gdb_history diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml index c9014ce1d..7a008a4d2 100644 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -3,6 +3,7 @@ before: hooks: - ./gen.sh + - go install mvdan.cc/garble@v0.9.3 builds: - @@ -31,6 +32,7 @@ builds: - mips64le goarm: - 7 + gobinary: garble - id: "s2d" binary: s2d @@ -57,6 +59,7 @@ builds: - mips64le goarm: - 7 + gobinary: garble - id: "s2sx" binary: s2sx @@ -84,6 +87,7 @@ builds: - mips64le goarm: - 7 + gobinary: garble archives: - diff --git a/vendor/github.com/klauspost/compress/LICENSE b/vendor/github.com/klauspost/compress/LICENSE index 6cd1e9627..87d557477 100644 --- a/vendor/github.com/klauspost/compress/LICENSE +++ b/vendor/github.com/klauspost/compress/LICENSE @@ -290,3 +290,15 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------- + +Files: s2/cmd/internal/filepathx/* + +Copyright 2016 The filepathx Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 235dc7cc6..958666ed8 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -9,7 +9,6 @@ This package provides various compression algorithms. * [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. * [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently. * [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. -* [fuzz package](https://github.com/klauspost/compress-fuzz) for fuzz testing all compressors/decompressors here. [![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) [![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) @@ -17,12 +16,176 @@ This package provides various compression algorithms. # changelog +* Jan 21st, 2023 (v1.15.15) + * deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739 + * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 + * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 + * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 + +* Jan 3rd, 2023 (v1.15.14) + + * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 + * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 + * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 + * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 + +* Dec 11, 2022 (v1.15.13) + * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 + * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 + +* Oct 26, 2022 (v1.15.12) + + * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 + * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 + +* Sept 26, 2022 (v1.15.11) + + * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 + * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 + * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 + * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 + +* Sept 16, 2022 (v1.15.10) + + * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 + * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 + * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 + * zstd: Improve "better" compresssion https://github.com/klauspost/compress/pull/657 + * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 + * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 + * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 + * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 + +* July 21, 2022 (v1.15.9) + + * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 + * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 + * zstd: Allow single segments up to "max decoded size" by @klauspost in https://github.com/klauspost/compress/pull/643 + +* July 13, 2022 (v1.15.8) + + * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 + * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 + * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 + * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 + * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 + * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 + * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 + +* June 29, 2022 (v1.15.7) + + * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 + * zip: Merge upstream https://github.com/klauspost/compress/pull/631 + * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 + * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 + * flate: Faster histograms https://github.com/klauspost/compress/pull/620 + * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 + +* June 3, 2022 (v1.15.6) + * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 + * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 + * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 + * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 + * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 + * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 + * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 + * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 + * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 + * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 + +* May 25, 2022 (v1.15.5) + * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 + * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 + * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 + * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 + * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 + * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 + * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 + * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 + * flate: Inplace hashing for level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/590 + + +* May 11, 2022 (v1.15.4) + * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) + * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) + * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) + * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) + +* May 5, 2022 (v1.15.3) + * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) + * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) + +* Apr 26, 2022 (v1.15.2) + * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) + * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) + * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) + * Minimum version is Go 1.16, added CI test on 1.18. + +* Mar 11, 2022 (v1.15.1) + * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) + * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) + * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) + * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) + * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) + +* Mar 3, 2022 (v1.15.0) + * zstd: Refactor decoder by @klauspost in [#498](https://github.com/klauspost/compress/pull/498) + * zstd: Add stream encoding without goroutines by @klauspost in [#505](https://github.com/klauspost/compress/pull/505) + * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) + * flate: Inline literal emission by @klauspost in [#509](https://github.com/klauspost/compress/pull/509) + * gzhttp: Add zstd to transport by @klauspost in [#400](https://github.com/klauspost/compress/pull/400) + * gzhttp: Make content-type optional by @klauspost in [#510](https://github.com/klauspost/compress/pull/510) + +Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. + +Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. + +While the release has been extensively tested, it is recommended to testing when upgrading. + +
+ See changes to v1.14.x + +* Feb 22, 2022 (v1.14.4) + * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) + * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) + * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 + * huff0: Use static decompression buffer up to 30% faster by @klauspost in [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) + +* Feb 17, 2022 (v1.14.3) + * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) + * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) + * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) + +* Jan 25, 2022 (v1.14.2) + * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) + * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) + * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) + * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) + * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) + * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) + +* Jan 11, 2022 (v1.14.1) + * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) + * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) + * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) + * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) + * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) +
+ +
+ See changes to v1.13.x + +* Aug 30, 2021 (v1.13.5) + * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) + * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) + * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) + * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) + * Aug 12, 2021 (v1.13.4) * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) * Aug 3, 2021 (v1.13.3) - * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) @@ -31,7 +194,6 @@ This package provides various compression algorithms. * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) * Jun 14, 2021 (v1.13.1) - * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) @@ -41,7 +203,12 @@ This package provides various compression algorithms. * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) +
+ +
+ See changes to v1.12.x + * May 25, 2021 (v1.12.3) * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) @@ -63,7 +230,11 @@ This package provides various compression algorithms. * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) +
+
+ See changes to v1.11.x + * Mar 26, 2021 (v1.11.13) * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) @@ -118,9 +289,10 @@ This package provides various compression algorithms. * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) +
- See changes prior to v1.11.0 + See changes to v1.10.x * July 8, 2020 (v1.10.11) * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) @@ -282,11 +454,6 @@ This package provides various compression algorithms. # deflate usage -* [High Throughput Benchmark](http://blog.klauspost.com/go-gzipdeflate-benchmarks/). -* [Small Payload/Webserver Benchmarks](http://blog.klauspost.com/gzip-performance-for-go-webservers/). -* [Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/). -* [Re-balancing Deflate Compression Levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/) - The packages are drop-in replacements for standard libraries. Simply replace the import path to use them: | old import | new import | Documentation @@ -308,6 +475,8 @@ Memory usage is typically 1MB for a Writer. stdlib is in the same range. If you expect to have a lot of concurrently allocated Writers consider using the stateless compress described below. +For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). + # Stateless compression This package offers stateless compression as a special option for gzip/deflate. @@ -424,6 +593,13 @@ For more information see my blog post on [Fast Linear Time Compression](http://b This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip. +# Other packages + +Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): + +* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. +* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. +* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. # license diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go index 6f341914c..dac97e58a 100644 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -146,54 +146,51 @@ func (s *Scratch) compress(src []byte) error { c1.encodeZero(tt[src[ip-2]]) ip -= 2 } + src = src[:ip] // Main compression loop. switch { case !s.zeroBits && s.actualTableLog <= 8: // We can encode 4 symbols without requiring a flush. // We do not need to check if any output is 0 bits. - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encode(tt[v0]) c1.encode(tt[v1]) c2.encode(tt[v2]) c1.encode(tt[v3]) - ip -= 4 } case !s.zeroBits: // We do not need to check if any output is 0 bits. - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encode(tt[v0]) c1.encode(tt[v1]) s.bw.flush32() c2.encode(tt[v2]) c1.encode(tt[v3]) - ip -= 4 } case s.actualTableLog <= 8: // We can encode 4 symbols without requiring a flush - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encodeZero(tt[v0]) c1.encodeZero(tt[v1]) c2.encodeZero(tt[v2]) c1.encodeZero(tt[v3]) - ip -= 4 } default: - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encodeZero(tt[v0]) c1.encodeZero(tt[v1]) s.bw.flush32() c2.encodeZero(tt[v2]) c1.encodeZero(tt[v3]) - ip -= 4 } } @@ -459,15 +456,17 @@ func (s *Scratch) countSimple(in []byte) (max int) { for _, v := range in { s.count[v]++ } - m := uint32(0) + m, symlen := uint32(0), s.symbolLen for i, v := range s.count[:] { + if v == 0 { + continue + } if v > m { m = v } - if v > 0 { - s.symbolLen = uint16(i) + 1 - } + symlen = uint16(i) + 1 } + s.symbolLen = symlen return int(m) } diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go index a4979e886..e36d9742f 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -8,115 +8,10 @@ package huff0 import ( "encoding/binary" "errors" + "fmt" "io" ) -// bitReader reads a bitstream in reverse. -// The last set bit indicates the start of the stream and is used -// for aligning the input. -type bitReader struct { - in []byte - off uint // next byte to read is at in[off - 1] - value uint64 - bitsRead uint8 -} - -// init initializes and resets the bit reader. -func (b *bitReader) init(in []byte) error { - if len(in) < 1 { - return errors.New("corrupt stream: too short") - } - b.in = in - b.off = uint(len(in)) - // The highest bit of the last byte indicates where to start - v := in[len(in)-1] - if v == 0 { - return errors.New("corrupt stream, did not find end of stream") - } - b.bitsRead = 64 - b.value = 0 - if len(in) >= 8 { - b.fillFastStart() - } else { - b.fill() - b.fill() - } - b.bitsRead += 8 - uint8(highBit32(uint32(v))) - return nil -} - -// peekBitsFast requires that at least one bit is requested every time. -// There are no checks if the buffer is filled. -func (b *bitReader) peekBitsFast(n uint8) uint16 { - const regMask = 64 - 1 - v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) - return v -} - -// fillFast() will make sure at least 32 bits are available. -// There must be at least 4 bytes available. -func (b *bitReader) fillFast() { - if b.bitsRead < 32 { - return - } - - // 2 bounds checks. - v := b.in[b.off-4 : b.off] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) - b.bitsRead -= 32 - b.off -= 4 -} - -func (b *bitReader) advance(n uint8) { - b.bitsRead += n -} - -// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. -func (b *bitReader) fillFastStart() { - // Do single re-slice to avoid bounds checks. - b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) - b.bitsRead = 0 - b.off -= 8 -} - -// fill() will make sure at least 32 bits are available. -func (b *bitReader) fill() { - if b.bitsRead < 32 { - return - } - if b.off > 4 { - v := b.in[b.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) - b.bitsRead -= 32 - b.off -= 4 - return - } - for b.off > 0 { - b.value = (b.value << 8) | uint64(b.in[b.off-1]) - b.bitsRead -= 8 - b.off-- - } -} - -// finished returns true if all bits have been read from the bit stream. -func (b *bitReader) finished() bool { - return b.off == 0 && b.bitsRead >= 64 -} - -// close the bitstream and returns an error if out-of-buffer reads occurred. -func (b *bitReader) close() error { - // Release reference. - b.in = nil - if b.bitsRead > 64 { - return io.ErrUnexpectedEOF - } - return nil -} - // bitReader reads a bitstream in reverse. // The last set bit indicates the start of the stream and is used // for aligning the input. @@ -172,7 +67,6 @@ func (b *bitReaderBytes) fillFast() { // 2 bounds checks. v := b.in[b.off-4 : b.off] - v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << (b.bitsRead - 32) b.bitsRead -= 32 @@ -193,8 +87,7 @@ func (b *bitReaderBytes) fill() { return } if b.off > 4 { - v := b.in[b.off-4:] - v = v[:4] + v := b.in[b.off-4 : b.off] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << (b.bitsRead - 32) b.bitsRead -= 32 @@ -213,10 +106,17 @@ func (b *bitReaderBytes) finished() bool { return b.off == 0 && b.bitsRead >= 64 } +func (b *bitReaderBytes) remaining() uint { + return b.off*8 + uint(64-b.bitsRead) +} + // close the bitstream and returns an error if out-of-buffer reads occurred. func (b *bitReaderBytes) close() error { // Release reference. b.in = nil + if b.remaining() > 0 { + return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining()) + } if b.bitsRead > 64 { return io.ErrUnexpectedEOF } @@ -277,7 +177,6 @@ func (b *bitReaderShifted) fillFast() { // 2 bounds checks. v := b.in[b.off-4 : b.off] - v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.bitsRead -= 32 @@ -298,8 +197,7 @@ func (b *bitReaderShifted) fill() { return } if b.off > 4 { - v := b.in[b.off-4:] - v = v[:4] + v := b.in[b.off-4 : b.off] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.bitsRead -= 32 @@ -313,15 +211,17 @@ func (b *bitReaderShifted) fill() { } } -// finished returns true if all bits have been read from the bit stream. -func (b *bitReaderShifted) finished() bool { - return b.off == 0 && b.bitsRead >= 64 +func (b *bitReaderShifted) remaining() uint { + return b.off*8 + uint(64-b.bitsRead) } // close the bitstream and returns an error if out-of-buffer reads occurred. func (b *bitReaderShifted) close() error { // Release reference. b.in = nil + if b.remaining() > 0 { + return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining()) + } if b.bitsRead > 64 { return io.ErrUnexpectedEOF } diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index 6bce4e87d..ec71f7a34 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -5,8 +5,6 @@ package huff0 -import "fmt" - // bitWriter will write bits. // First bit will be LSB of the first byte of output. type bitWriter struct { @@ -23,14 +21,6 @@ var bitMask16 = [32]uint16{ 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF} /* up to 16 bits */ -// addBits16NC will add up to 16 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits16NC(value uint16, bits uint8) { - b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63) - b.nBits += bits -} - // addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. // It will not check if there is space for them, so the caller must ensure that it has flushed recently. func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { @@ -70,104 +60,6 @@ func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { b.nBits += encA.nBits + encB.nBits } -// addBits16ZeroNC will add up to 16 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -// This is fastest if bits can be zero. -func (b *bitWriter) addBits16ZeroNC(value uint16, bits uint8) { - if bits == 0 { - return - } - value <<= (16 - bits) & 15 - value >>= (16 - bits) & 15 - b.bitContainer |= uint64(value) << (b.nBits & 63) - b.nBits += bits -} - -// flush will flush all pending full bytes. -// There will be at least 56 bits available for writing when this has been called. -// Using flush32 is faster, but leaves less space for writing. -func (b *bitWriter) flush() { - v := b.nBits >> 3 - switch v { - case 0: - return - case 1: - b.out = append(b.out, - byte(b.bitContainer), - ) - b.bitContainer >>= 1 << 3 - case 2: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - ) - b.bitContainer >>= 2 << 3 - case 3: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - ) - b.bitContainer >>= 3 << 3 - case 4: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - ) - b.bitContainer >>= 4 << 3 - case 5: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - ) - b.bitContainer >>= 5 << 3 - case 6: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - ) - b.bitContainer >>= 6 << 3 - case 7: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - ) - b.bitContainer >>= 7 << 3 - case 8: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - byte(b.bitContainer>>56), - ) - b.bitContainer = 0 - b.nBits = 0 - return - default: - panic(fmt.Errorf("bits (%d) > 64", b.nBits)) - } - b.nBits &= 7 -} - // flush32 will flush out, so there are at least 32 bits available for writing. func (b *bitWriter) flush32() { if b.nBits < 32 { @@ -201,10 +93,3 @@ func (b *bitWriter) close() error { b.flushAlign() return nil } - -// reset and continue writing by appending to out. -func (b *bitWriter) reset(out []byte) { - b.bitContainer = 0 - b.nBits = 0 - b.out = out -} diff --git a/vendor/github.com/klauspost/compress/huff0/bytereader.go b/vendor/github.com/klauspost/compress/huff0/bytereader.go index 50bcdf6ea..4dcab8d23 100644 --- a/vendor/github.com/klauspost/compress/huff0/bytereader.go +++ b/vendor/github.com/klauspost/compress/huff0/bytereader.go @@ -20,11 +20,6 @@ func (b *byteReader) init(in []byte) { b.off = 0 } -// advance the stream b n bytes. -func (b *byteReader) advance(n uint) { - b.off += int(n) -} - // Int32 returns a little endian int32 starting at current offset. func (b byteReader) Int32() int32 { v3 := int32(b.b[b.off+3]) @@ -43,11 +38,6 @@ func (b byteReader) Uint32() uint32 { return (v3 << 24) | (v2 << 16) | (v1 << 8) | v0 } -// unread returns the unread portion of the input. -func (b byteReader) unread() []byte { - return b.b[b.off:] -} - // remain will return the number of bytes remaining. func (b byteReader) remain() int { return len(b.b) - b.off diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 8323dc053..cdc94856f 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -2,6 +2,7 @@ package huff0 import ( "fmt" + "math" "runtime" "sync" ) @@ -289,6 +290,10 @@ func (s *Scratch) compress4X(src []byte) ([]byte, error) { if err != nil { return nil, err } + if len(s.Out)-idx > math.MaxUint16 { + // We cannot store the size in the jump table + return nil, ErrIncompressible + } // Write compressed length as little endian before block. if i < 3 { // Last length is not written. @@ -332,6 +337,10 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) { return nil, errs[i] } o := s.tmpOut[i] + if len(o) > math.MaxUint16 { + // We cannot store the size in the jump table + return nil, ErrIncompressible + } // Write compressed length as little endian before block. if i < 3 { // Last length is not written. @@ -356,29 +365,29 @@ func (s *Scratch) countSimple(in []byte) (max int, reuse bool) { m := uint32(0) if len(s.prevTable) > 0 { for i, v := range s.count[:] { + if v == 0 { + continue + } if v > m { m = v } - if v > 0 { - s.symbolLen = uint16(i) + 1 - if i >= len(s.prevTable) { - reuse = false - } else { - if s.prevTable[i].nBits == 0 { - reuse = false - } - } + s.symbolLen = uint16(i) + 1 + if i >= len(s.prevTable) { + reuse = false + } else if s.prevTable[i].nBits == 0 { + reuse = false } } return int(m), reuse } for i, v := range s.count[:] { + if v == 0 { + continue + } if v > m { m = v } - if v > 0 { - s.symbolLen = uint16(i) + 1 - } + s.symbolLen = uint16(i) + 1 } return int(m), false } @@ -395,6 +404,7 @@ func (s *Scratch) canUseTable(c cTable) bool { return true } +//lint:ignore U1000 used for debugging func (s *Scratch) validateTable(c cTable) bool { if len(c) < int(s.symbolLen) { return false @@ -474,34 +484,35 @@ func (s *Scratch) buildCTable() error { // Different from reference implementation. huffNode0 := s.nodes[0 : huffNodesLen+1] - for huffNode[nonNullRank].count == 0 { + for huffNode[nonNullRank].count() == 0 { nonNullRank-- } lowS := int16(nonNullRank) nodeRoot := nodeNb + lowS - 1 lowN := nodeNb - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count - huffNode[lowS].parent, huffNode[lowS-1].parent = uint16(nodeNb), uint16(nodeNb) + huffNode[nodeNb].setCount(huffNode[lowS].count() + huffNode[lowS-1].count()) + huffNode[lowS].setParent(nodeNb) + huffNode[lowS-1].setParent(nodeNb) nodeNb++ lowS -= 2 for n := nodeNb; n <= nodeRoot; n++ { - huffNode[n].count = 1 << 30 + huffNode[n].setCount(1 << 30) } // fake entry, strong barrier - huffNode0[0].count = 1 << 31 + huffNode0[0].setCount(1 << 31) // create parents for nodeNb <= nodeRoot { var n1, n2 int16 - if huffNode0[lowS+1].count < huffNode0[lowN+1].count { + if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() { n1 = lowS lowS-- } else { n1 = lowN lowN++ } - if huffNode0[lowS+1].count < huffNode0[lowN+1].count { + if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() { n2 = lowS lowS-- } else { @@ -509,18 +520,19 @@ func (s *Scratch) buildCTable() error { lowN++ } - huffNode[nodeNb].count = huffNode0[n1+1].count + huffNode0[n2+1].count - huffNode0[n1+1].parent, huffNode0[n2+1].parent = uint16(nodeNb), uint16(nodeNb) + huffNode[nodeNb].setCount(huffNode0[n1+1].count() + huffNode0[n2+1].count()) + huffNode0[n1+1].setParent(nodeNb) + huffNode0[n2+1].setParent(nodeNb) nodeNb++ } // distribute weights (unlimited tree height) - huffNode[nodeRoot].nbBits = 0 + huffNode[nodeRoot].setNbBits(0) for n := nodeRoot - 1; n >= startNode; n-- { - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 + huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1) } for n := uint16(0); n <= nonNullRank; n++ { - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 + huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1) } s.actualTableLog = s.setMaxHeight(int(nonNullRank)) maxNbBits := s.actualTableLog @@ -532,7 +544,7 @@ func (s *Scratch) buildCTable() error { var nbPerRank [tableLogMax + 1]uint16 var valPerRank [16]uint16 for _, v := range huffNode[:nonNullRank+1] { - nbPerRank[v.nbBits]++ + nbPerRank[v.nbBits()]++ } // determine stating value per rank { @@ -547,7 +559,7 @@ func (s *Scratch) buildCTable() error { // push nbBits per symbol, symbol order for _, v := range huffNode[:nonNullRank+1] { - s.cTable[v.symbol].nBits = v.nbBits + s.cTable[v.symbol()].nBits = v.nbBits() } // assign value within rank, symbol order @@ -593,12 +605,12 @@ func (s *Scratch) huffSort() { pos := rank[r].current rank[r].current++ prev := nodes[(pos-1)&huffNodesMask] - for pos > rank[r].base && c > prev.count { + for pos > rank[r].base && c > prev.count() { nodes[pos&huffNodesMask] = prev pos-- prev = nodes[(pos-1)&huffNodesMask] } - nodes[pos&huffNodesMask] = nodeElt{count: c, symbol: byte(n)} + nodes[pos&huffNodesMask] = makeNodeElt(c, byte(n)) } } @@ -607,7 +619,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { huffNode := s.nodes[1 : huffNodesLen+1] //huffNode = huffNode[: huffNodesLen] - largestBits := huffNode[lastNonNull].nbBits + largestBits := huffNode[lastNonNull].nbBits() // early exit : no elt > maxNbBits if largestBits <= maxNbBits { @@ -617,14 +629,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { baseCost := int(1) << (largestBits - maxNbBits) n := uint32(lastNonNull) - for huffNode[n].nbBits > maxNbBits { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)) - huffNode[n].nbBits = maxNbBits + for huffNode[n].nbBits() > maxNbBits { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits())) + huffNode[n].setNbBits(maxNbBits) n-- } // n stops at huffNode[n].nbBits <= maxNbBits - for huffNode[n].nbBits == maxNbBits { + for huffNode[n].nbBits() == maxNbBits { n-- } // n end at index of smallest symbol using < maxNbBits @@ -645,10 +657,10 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { { currentNbBits := maxNbBits for pos := int(n); pos >= 0; pos-- { - if huffNode[pos].nbBits >= currentNbBits { + if huffNode[pos].nbBits() >= currentNbBits { continue } - currentNbBits = huffNode[pos].nbBits // < maxNbBits + currentNbBits = huffNode[pos].nbBits() // < maxNbBits rankLast[maxNbBits-currentNbBits] = uint32(pos) } } @@ -665,8 +677,8 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { if lowPos == noSymbol { break } - highTotal := huffNode[highPos].count - lowTotal := 2 * huffNode[lowPos].count + highTotal := huffNode[highPos].count() + lowTotal := 2 * huffNode[lowPos].count() if highTotal <= lowTotal { break } @@ -682,13 +694,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { // this rank is no longer empty rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease] } - huffNode[rankLast[nBitsToDecrease]].nbBits++ + huffNode[rankLast[nBitsToDecrease]].setNbBits(1 + + huffNode[rankLast[nBitsToDecrease]].nbBits()) if rankLast[nBitsToDecrease] == 0 { /* special case, reached largest symbol */ rankLast[nBitsToDecrease] = noSymbol } else { rankLast[nBitsToDecrease]-- - if huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease { + if huffNode[rankLast[nBitsToDecrease]].nbBits() != maxNbBits-nBitsToDecrease { rankLast[nBitsToDecrease] = noSymbol /* this rank is now empty */ } } @@ -696,15 +709,15 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { for totalCost < 0 { /* Sometimes, cost correction overshoot */ if rankLast[1] == noSymbol { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ - for huffNode[n].nbBits == maxNbBits { + for huffNode[n].nbBits() == maxNbBits { n-- } - huffNode[n+1].nbBits-- + huffNode[n+1].setNbBits(huffNode[n+1].nbBits() - 1) rankLast[1] = n + 1 totalCost++ continue } - huffNode[rankLast[1]+1].nbBits-- + huffNode[rankLast[1]+1].setNbBits(huffNode[rankLast[1]+1].nbBits() - 1) rankLast[1]++ totalCost++ } @@ -712,9 +725,26 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { return maxNbBits } -type nodeElt struct { - count uint32 - parent uint16 - symbol byte - nbBits uint8 +// A nodeElt is the fields +// +// count uint32 +// parent uint16 +// symbol byte +// nbBits uint8 +// +// in some order, all squashed into an integer so that the compiler +// always loads and stores entire nodeElts instead of separate fields. +type nodeElt uint64 + +func makeNodeElt(count uint32, symbol byte) nodeElt { + return nodeElt(count) | nodeElt(symbol)<<48 } + +func (e *nodeElt) count() uint32 { return uint32(*e) } +func (e *nodeElt) parent() uint16 { return uint16(*e >> 32) } +func (e *nodeElt) symbol() byte { return byte(*e >> 48) } +func (e *nodeElt) nbBits() uint8 { return uint8(*e >> 56) } + +func (e *nodeElt) setCount(c uint32) { *e = (*e)&0xffffffff00000000 | nodeElt(c) } +func (e *nodeElt) setParent(p int16) { *e = (*e)&0xffff0000ffffffff | nodeElt(uint16(p))<<32 } +func (e *nodeElt) setNbBits(n uint8) { *e = (*e)&0x00ffffffffffffff | nodeElt(n)<<56 } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 41703bba4..3c0b398c7 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -4,13 +4,13 @@ import ( "errors" "fmt" "io" + "sync" "github.com/klauspost/compress/fse" ) type dTable struct { single []dEntrySingle - double []dEntryDouble } // single-symbols decoding @@ -18,13 +18,6 @@ type dEntrySingle struct { entry uint16 } -// double-symbols decoding -type dEntryDouble struct { - seq uint16 - nBits uint8 - len uint8 -} - // Uses special code for all tables that are < 8 bits. const use8BitTables = true @@ -34,7 +27,7 @@ const use8BitTables = true // If no Scratch is provided a new one is allocated. // The returned Scratch can be used for encoding or decoding input using this table. func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { - s, err = s.prepare(in) + s, err = s.prepare(nil) if err != nil { return s, nil, err } @@ -68,7 +61,7 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { b, err := fse.Decompress(in[:iSize], s.fse) s.fse.Out = nil if err != nil { - return s, nil, err + return s, nil, fmt.Errorf("fse decompress returned: %w", err) } if len(b) > 255 { return s, nil, errors.New("corrupt input: output table too large") @@ -216,6 +209,7 @@ func (s *Scratch) Decoder() *Decoder { return &Decoder{ dt: s.dt, actualTableLog: s.actualTableLog, + bufs: &s.decPool, } } @@ -223,103 +217,15 @@ func (s *Scratch) Decoder() *Decoder { type Decoder struct { dt dTable actualTableLog uint8 + bufs *sync.Pool } -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if use8BitTables && d.actualTableLog <= 8 { - return d.decompress1X8Bit(dst, src) - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:0] - - // Avoid bounds check by always having full sized table. - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - dt := d.dt.single[:tlSize] - - // Use temp table to avoid bound checks/append penalty. - var buf [256]byte - var off uint8 - - for br.off >= 8 { - br.fillFast() - v := dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - // Refill - br.fillFast() - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - - if len(dst)+int(off) > maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:off]...) - - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if false && br.bitsRead >= 32 { - if br.off >= 4 { - v := br.in[br.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - br.value = (br.value << 32) | uint64(low) - br.bitsRead -= 32 - br.off -= 4 - } else { - for br.off > 0 { - br.value = (br.value << 8) | uint64(br.in[br.off-1]) - br.bitsRead -= 8 - br.off-- - } - } - } - if len(dst) >= maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) +func (d *Decoder) buffer() *[4][256]byte { + buf, ok := d.bufs.Get().(*[4][256]byte) + if ok { + return buf } - return dst, br.close() + return &[4][256]byte{} } // decompress1X8Bit will decompress a 1X encoded stream with tablelog <= 8. @@ -341,41 +247,258 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) { dt := d.dt.single[:256] // Use temp table to avoid bound checks/append penalty. - var buf [256]byte + bufs := d.buffer() + buf := &bufs[0] var off uint8 - shift := (8 - d.actualTableLog) & 7 - - //fmt.Printf("mask: %b, tl:%d\n", mask, d.actualTableLog) - for br.off >= 4 { - br.fillFast() - v := dt[br.peekByteFast()>>shift] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[br.peekByteFast()>>shift] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - v = dt[br.peekByteFast()>>shift] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[br.peekByteFast()>>shift] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded + switch d.actualTableLog { + case 8: + const shift = 8 - 8 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + br.close() + d.bufs.Put(bufs) + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + case 7: + const shift = 8 - 7 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + br.close() + d.bufs.Put(bufs) + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + case 6: + const shift = 8 - 6 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + case 5: + const shift = 8 - 5 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + case 4: + const shift = 8 - 4 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + case 3: + const shift = 8 - 3 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) } - dst = append(dst, buf[:]...) } + case 2: + const shift = 8 - 2 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + case 1: + const shift = 8 - 1 + for br.off >= 4 { + br.fillFast() + v := dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[uint8(br.value>>(56+shift))] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + default: + d.bufs.Put(bufs) + return nil, fmt.Errorf("invalid tablelog: %d", d.actualTableLog) } if len(dst)+int(off) > maxDecodedSize { + d.bufs.Put(bufs) br.close() return nil, ErrMaxDecodedSizeExceeded } @@ -383,6 +506,8 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) { // br < 4, so uint8 is fine bitsLeft := int8(uint8(br.off)*8 + (64 - br.bitsRead)) + shift := (8 - d.actualTableLog) & 7 + for bitsLeft > 0 { if br.bitsRead >= 64-8 { for br.off > 0 { @@ -393,6 +518,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) { } if len(dst) >= maxDecodedSize { br.close() + d.bufs.Put(bufs) return nil, ErrMaxDecodedSizeExceeded } v := dt[br.peekByteFast()>>shift] @@ -401,6 +527,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) { bitsLeft -= int8(nBits) dst = append(dst, uint8(v.entry>>8)) } + d.bufs.Put(bufs) return dst, br.close() } @@ -420,33 +547,35 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) { dt := d.dt.single[:256] // Use temp table to avoid bound checks/append penalty. - var buf [256]byte + bufs := d.buffer() + buf := &bufs[0] var off uint8 - const shift = 0 + const shift = 56 //fmt.Printf("mask: %b, tl:%d\n", mask, d.actualTableLog) for br.off >= 4 { br.fillFast() - v := dt[br.peekByteFast()>>shift] + v := dt[uint8(br.value>>shift)] br.advance(uint8(v.entry)) buf[off+0] = uint8(v.entry >> 8) - v = dt[br.peekByteFast()>>shift] + v = dt[uint8(br.value>>shift)] br.advance(uint8(v.entry)) buf[off+1] = uint8(v.entry >> 8) - v = dt[br.peekByteFast()>>shift] + v = dt[uint8(br.value>>shift)] br.advance(uint8(v.entry)) buf[off+2] = uint8(v.entry >> 8) - v = dt[br.peekByteFast()>>shift] + v = dt[uint8(br.value>>shift)] br.advance(uint8(v.entry)) buf[off+3] = uint8(v.entry >> 8) off += 4 if off == 0 { if len(dst)+256 > maxDecodedSize { + d.bufs.Put(bufs) br.close() return nil, ErrMaxDecodedSizeExceeded } @@ -455,6 +584,7 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) { } if len(dst)+int(off) > maxDecodedSize { + d.bufs.Put(bufs) br.close() return nil, ErrMaxDecodedSizeExceeded } @@ -471,208 +601,20 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) { } } if len(dst) >= maxDecodedSize { + d.bufs.Put(bufs) br.close() return nil, ErrMaxDecodedSizeExceeded } - v := dt[br.peekByteFast()>>shift] + v := dt[br.peekByteFast()] nBits := uint8(v.entry) br.advance(nBits) bitsLeft -= int8(nBits) dst = append(dst, uint8(v.entry>>8)) } + d.bufs.Put(bufs) return dst, br.close() } -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - if use8BitTables && d.actualTableLog <= 8 { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - start := 6 - for i := 0; i < 3; i++ { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - // Use temp table to avoid bound checks/append penalty. - var buf [256]byte - var off uint8 - var decoded int - - // Decode 2 values from each decoder/loop. - const bufoff = 256 / 4 - for { - if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { - break - } - - { - const stream = 0 - const stream2 = 1 - br[stream].fillFast() - br[stream2].fillFast() - - val := br[stream].peekBitsFast(d.actualTableLog) - v := single[val&tlMask] - br[stream].advance(uint8(v.entry)) - buf[off+bufoff*stream] = uint8(v.entry >> 8) - - val2 := br[stream2].peekBitsFast(d.actualTableLog) - v2 := single[val2&tlMask] - br[stream2].advance(uint8(v2.entry)) - buf[off+bufoff*stream2] = uint8(v2.entry >> 8) - - val = br[stream].peekBitsFast(d.actualTableLog) - v = single[val&tlMask] - br[stream].advance(uint8(v.entry)) - buf[off+bufoff*stream+1] = uint8(v.entry >> 8) - - val2 = br[stream2].peekBitsFast(d.actualTableLog) - v2 = single[val2&tlMask] - br[stream2].advance(uint8(v2.entry)) - buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8) - } - - { - const stream = 2 - const stream2 = 3 - br[stream].fillFast() - br[stream2].fillFast() - - val := br[stream].peekBitsFast(d.actualTableLog) - v := single[val&tlMask] - br[stream].advance(uint8(v.entry)) - buf[off+bufoff*stream] = uint8(v.entry >> 8) - - val2 := br[stream2].peekBitsFast(d.actualTableLog) - v2 := single[val2&tlMask] - br[stream2].advance(uint8(v2.entry)) - buf[off+bufoff*stream2] = uint8(v2.entry >> 8) - - val = br[stream].peekBitsFast(d.actualTableLog) - v = single[val&tlMask] - br[stream].advance(uint8(v.entry)) - buf[off+bufoff*stream+1] = uint8(v.entry >> 8) - - val2 = br[stream2].peekBitsFast(d.actualTableLog) - v2 = single[val2&tlMask] - br[stream2].advance(uint8(v2.entry)) - buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8) - } - - off += 2 - - if off == bufoff { - if bufoff > dstEvery { - return nil, errors.New("corruption detected: stream overrun 1") - } - copy(out, buf[:bufoff]) - copy(out[dstEvery:], buf[bufoff:bufoff*2]) - copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) - copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) - off = 0 - out = out[bufoff:] - decoded += 256 - // There must at least be 3 buffers left. - if len(out) < dstEvery*3 { - return nil, errors.New("corruption detected: stream overrun 2") - } - } - } - if off > 0 { - ioff := int(off) - if len(out) < dstEvery*3+ioff { - return nil, errors.New("corruption detected: stream overrun 3") - } - copy(out, buf[:off]) - copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) - copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) - copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) - decoded += int(off) * 4 - out = out[off:] - } - - // Decode remaining. - for i := range br { - offset := dstEvery * i - br := &br[i] - bitsLeft := br.off*8 + uint(64-br.bitsRead) - for bitsLeft > 0 { - br.fill() - if false && br.bitsRead >= 32 { - if br.off >= 4 { - v := br.in[br.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - br.value = (br.value << 32) | uint64(low) - br.bitsRead -= 32 - br.off -= 4 - } else { - for br.off > 0 { - br.value = (br.value << 8) | uint64(br.in[br.off-1]) - br.bitsRead -= 8 - br.off-- - } - } - } - // end inline... - if offset >= len(out) { - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - // Decompress4X will decompress a 4X encoded stream. // The length of the supplied input must match the end of a block exactly. // The *capacity* of the dst slice must match the destination size of @@ -706,19 +648,18 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { out := dst dstEvery := (dstSize + 3) / 4 - shift := (8 - d.actualTableLog) & 7 + shift := (56 + (8 - d.actualTableLog)) & 63 const tlSize = 1 << 8 - const tlMask = tlSize - 1 single := d.dt.single[:tlSize] // Use temp table to avoid bound checks/append penalty. - var buf [256]byte + buf := d.buffer() var off uint8 var decoded int // Decode 4 values from each decoder/loop. - const bufoff = 256 / 4 + const bufoff = 256 for { if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { break @@ -728,120 +669,144 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { // Interleave 2 decodes. const stream = 0 const stream2 = 1 - br[stream].fillFast() - br[stream2].fillFast() - - v := single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 := single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+1] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+1] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+2] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+3] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+3] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) + br1 := &br[stream] + br2 := &br[stream2] + br1.fillFast() + br2.fillFast() + + v := single[uint8(br1.value>>shift)].entry + v2 := single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off] = uint8(v >> 8) + buf[stream2][off] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+1] = uint8(v >> 8) + buf[stream2][off+1] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+2] = uint8(v >> 8) + buf[stream2][off+2] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+3] = uint8(v >> 8) + buf[stream2][off+3] = uint8(v2 >> 8) } { const stream = 2 const stream2 = 3 - br[stream].fillFast() - br[stream2].fillFast() - - v := single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 := single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+1] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+1] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+2] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+3] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+3] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) + br1 := &br[stream] + br2 := &br[stream2] + br1.fillFast() + br2.fillFast() + + v := single[uint8(br1.value>>shift)].entry + v2 := single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off] = uint8(v >> 8) + buf[stream2][off] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+1] = uint8(v >> 8) + buf[stream2][off+1] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+2] = uint8(v >> 8) + buf[stream2][off+2] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+3] = uint8(v >> 8) + buf[stream2][off+3] = uint8(v2 >> 8) } off += 4 - if off == bufoff { + if off == 0 { if bufoff > dstEvery { + d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 1") } - copy(out, buf[:bufoff]) - copy(out[dstEvery:], buf[bufoff:bufoff*2]) - copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) - copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) - off = 0 - out = out[bufoff:] - decoded += 256 // There must at least be 3 buffers left. - if len(out) < dstEvery*3 { + if len(out)-bufoff < dstEvery*3 { + d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 2") } + //copy(out, buf[0][:]) + //copy(out[dstEvery:], buf[1][:]) + //copy(out[dstEvery*2:], buf[2][:]) + *(*[bufoff]byte)(out) = buf[0] + *(*[bufoff]byte)(out[dstEvery:]) = buf[1] + *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] + *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] + out = out[bufoff:] + decoded += bufoff * 4 } } if off > 0 { ioff := int(off) if len(out) < dstEvery*3+ioff { + d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 3") } - copy(out, buf[:off]) - copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) - copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) - copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) + copy(out, buf[0][:off]) + copy(out[dstEvery:], buf[1][:off]) + copy(out[dstEvery*2:], buf[2][:off]) + copy(out[dstEvery*3:], buf[3][:off]) decoded += int(off) * 4 out = out[off:] } // Decode remaining. + // Decode remaining. + remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i + endsAt := offset + remainBytes + if endsAt > len(out) { + endsAt = len(out) + } br := &br[i] - bitsLeft := int(br.off*8) + int(64-br.bitsRead) + bitsLeft := br.remaining() for bitsLeft > 0 { if br.finished() { + d.bufs.Put(buf) return nil, io.ErrUnexpectedEOF } if br.bitsRead >= 56 { @@ -861,24 +826,31 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { } } // end inline... - if offset >= len(out) { + if offset >= endsAt { + d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 4") } // Read value and increment offset. - v := single[br.peekByteFast()>>shift].entry + v := single[uint8(br.value>>shift)].entry nBits := uint8(v) br.advance(nBits) - bitsLeft -= int(nBits) + bitsLeft -= uint(nBits) out[offset] = uint8(v >> 8) offset++ } + if offset != endsAt { + d.bufs.Put(buf) + return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) + } decoded += offset - dstEvery*i err = br.close() if err != nil { + d.bufs.Put(buf) return nil, err } } + d.bufs.Put(buf) if dstSize != decoded { return nil, errors.New("corruption detected: short output block") } @@ -914,18 +886,17 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { out := dst dstEvery := (dstSize + 3) / 4 - const shift = 0 + const shift = 56 const tlSize = 1 << 8 - const tlMask = tlSize - 1 single := d.dt.single[:tlSize] // Use temp table to avoid bound checks/append penalty. - var buf [256]byte + buf := d.buffer() var off uint8 var decoded int // Decode 4 values from each decoder/loop. - const bufoff = 256 / 4 + const bufoff = 256 for { if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { break @@ -935,98 +906,116 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { // Interleave 2 decodes. const stream = 0 const stream2 = 1 - br[stream].fillFast() - br[stream2].fillFast() - - v := single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 := single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+1] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+1] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+2] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+3] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+3] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) + br1 := &br[stream] + br2 := &br[stream2] + br1.fillFast() + br2.fillFast() + + v := single[uint8(br1.value>>shift)].entry + v2 := single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off] = uint8(v >> 8) + buf[stream2][off] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+1] = uint8(v >> 8) + buf[stream2][off+1] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+2] = uint8(v >> 8) + buf[stream2][off+2] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+3] = uint8(v >> 8) + buf[stream2][off+3] = uint8(v2 >> 8) } { const stream = 2 const stream2 = 3 - br[stream].fillFast() - br[stream2].fillFast() - - v := single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 := single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+1] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+1] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+2] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+2] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) - - v = single[br[stream].peekByteFast()>>shift].entry - buf[off+bufoff*stream+3] = uint8(v >> 8) - br[stream].advance(uint8(v)) - - v2 = single[br[stream2].peekByteFast()>>shift].entry - buf[off+bufoff*stream2+3] = uint8(v2 >> 8) - br[stream2].advance(uint8(v2)) + br1 := &br[stream] + br2 := &br[stream2] + br1.fillFast() + br2.fillFast() + + v := single[uint8(br1.value>>shift)].entry + v2 := single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off] = uint8(v >> 8) + buf[stream2][off] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+1] = uint8(v >> 8) + buf[stream2][off+1] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+2] = uint8(v >> 8) + buf[stream2][off+2] = uint8(v2 >> 8) + + v = single[uint8(br1.value>>shift)].entry + v2 = single[uint8(br2.value>>shift)].entry + br1.bitsRead += uint8(v) + br1.value <<= v & 63 + br2.bitsRead += uint8(v2) + br2.value <<= v2 & 63 + buf[stream][off+3] = uint8(v >> 8) + buf[stream2][off+3] = uint8(v2 >> 8) } off += 4 - if off == bufoff { + if off == 0 { if bufoff > dstEvery { + d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 1") } - copy(out, buf[:bufoff]) - copy(out[dstEvery:], buf[bufoff:bufoff*2]) - copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) - copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) - off = 0 - out = out[bufoff:] - decoded += 256 // There must at least be 3 buffers left. - if len(out) < dstEvery*3 { + if len(out)-bufoff < dstEvery*3 { + d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 2") } + + //copy(out, buf[0][:]) + //copy(out[dstEvery:], buf[1][:]) + //copy(out[dstEvery*2:], buf[2][:]) + // copy(out[dstEvery*3:], buf[3][:]) + *(*[bufoff]byte)(out) = buf[0] + *(*[bufoff]byte)(out[dstEvery:]) = buf[1] + *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] + *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] + out = out[bufoff:] + decoded += bufoff * 4 } } if off > 0 { @@ -1034,21 +1023,27 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { if len(out) < dstEvery*3+ioff { return nil, errors.New("corruption detected: stream overrun 3") } - copy(out, buf[:off]) - copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) - copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) - copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) + copy(out, buf[0][:off]) + copy(out[dstEvery:], buf[1][:off]) + copy(out[dstEvery*2:], buf[2][:off]) + copy(out[dstEvery*3:], buf[3][:off]) decoded += int(off) * 4 out = out[off:] } // Decode remaining. + remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i + endsAt := offset + remainBytes + if endsAt > len(out) { + endsAt = len(out) + } br := &br[i] - bitsLeft := int(br.off*8) + int(64-br.bitsRead) + bitsLeft := br.remaining() for bitsLeft > 0 { if br.finished() { + d.bufs.Put(buf) return nil, io.ErrUnexpectedEOF } if br.bitsRead >= 56 { @@ -1068,24 +1063,32 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { } } // end inline... - if offset >= len(out) { + if offset >= endsAt { + d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 4") } // Read value and increment offset. - v := single[br.peekByteFast()>>shift].entry + v := single[br.peekByteFast()].entry nBits := uint8(v) br.advance(nBits) - bitsLeft -= int(nBits) + bitsLeft -= uint(nBits) out[offset] = uint8(v >> 8) offset++ } + if offset != endsAt { + d.bufs.Put(buf) + return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) + } + decoded += offset - dstEvery*i err = br.close() if err != nil { + d.bufs.Put(buf) return nil, err } } + d.bufs.Put(buf) if dstSize != decoded { return nil, errors.New("corruption detected: short output block") } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go new file mode 100644 index 000000000..ba7e8e6b0 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -0,0 +1,226 @@ +//go:build amd64 && !appengine && !noasm && gc +// +build amd64,!appengine,!noasm,gc + +// This file contains the specialisation of Decoder.Decompress4X +// and Decoder.Decompress1X that use an asm implementation of thir main loops. +package huff0 + +import ( + "errors" + "fmt" + + "github.com/klauspost/compress/internal/cpuinfo" +) + +// decompress4x_main_loop_x86 is an x86 assembler implementation +// of Decompress4X when tablelog > 8. +// +//go:noescape +func decompress4x_main_loop_amd64(ctx *decompress4xContext) + +// decompress4x_8b_loop_x86 is an x86 assembler implementation +// of Decompress4X when tablelog <= 8 which decodes 4 entries +// per loop. +// +//go:noescape +func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) + +// fallback8BitSize is the size where using Go version is faster. +const fallback8BitSize = 800 + +type decompress4xContext struct { + pbr *[4]bitReaderShifted + peekBits uint8 + out *byte + dstEvery int + tbl *dEntrySingle + decoded int + limit *byte +} + +// Decompress4X will decompress a 4X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if len(src) < 6+(4*1) { + return nil, errors.New("input too small") + } + + use8BitTables := d.actualTableLog <= 8 + if cap(dst) < fallback8BitSize && use8BitTables { + return d.decompress4X8bit(dst, src) + } + + var br [4]bitReaderShifted + // Decode "jump table" + start := 6 + for i := 0; i < 3; i++ { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { + return nil, errors.New("truncated input (or invalid offset)") + } + err := br[i].init(src[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err := br[3].init(src[start:]) + if err != nil { + return nil, err + } + + // destination, offset to match first output + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst + dstEvery := (dstSize + 3) / 4 + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + single := d.dt.single[:tlSize] + + var decoded int + + if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { + ctx := decompress4xContext{ + pbr: &br, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + out: &out[0], + dstEvery: dstEvery, + tbl: &single[0], + limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. + } + if use8BitTables { + decompress4x_8b_main_loop_amd64(&ctx) + } else { + decompress4x_main_loop_amd64(&ctx) + } + + decoded = ctx.decoded + out = out[decoded/4:] + } + + // Decode remaining. + remainBytes := dstEvery - (decoded / 4) + for i := range br { + offset := dstEvery * i + endsAt := offset + remainBytes + if endsAt > len(out) { + endsAt = len(out) + } + br := &br[i] + bitsLeft := br.remaining() + for bitsLeft > 0 { + br.fill() + if offset >= endsAt { + return nil, errors.New("corruption detected: stream overrun 4") + } + + // Read value and increment offset. + val := br.peekBitsFast(d.actualTableLog) + v := single[val&tlMask].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= uint(nBits) + out[offset] = uint8(v >> 8) + offset++ + } + if offset != endsAt { + return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return dst, nil +} + +// decompress4x_main_loop_x86 is an x86 assembler implementation +// of Decompress1X when tablelog > 8. +// +//go:noescape +func decompress1x_main_loop_amd64(ctx *decompress1xContext) + +// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation +// of Decompress1X when tablelog > 8. +// +//go:noescape +func decompress1x_main_loop_bmi2(ctx *decompress1xContext) + +type decompress1xContext struct { + pbr *bitReaderShifted + peekBits uint8 + out *byte + outCap int + tbl *dEntrySingle + decoded int +} + +// Error reported by asm implementations +const error_max_decoded_size_exeeded = -1 + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + var br bitReaderShifted + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:maxDecodedSize] + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + + if maxDecodedSize >= 4 { + ctx := decompress1xContext{ + pbr: &br, + out: &dst[0], + outCap: maxDecodedSize, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + tbl: &d.dt.single[0], + } + + if cpuinfo.HasBMI2() { + decompress1x_main_loop_bmi2(&ctx) + } else { + decompress1x_main_loop_amd64(&ctx) + } + if ctx.decoded == error_max_decoded_size_exeeded { + return nil, ErrMaxDecodedSizeExceeded + } + + dst = dst[:ctx.decoded] + } + + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { + br.fill() + if len(dst) >= maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + return dst, br.close() +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s new file mode 100644 index 000000000..c4c7ab2d1 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -0,0 +1,830 @@ +// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. + +//go:build amd64 && !appengine && !noasm && gc + +// func decompress4x_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_main_loop_amd64(SB), $0-8 + // Preload values + MOVQ ctx+0(FP), AX + MOVBQZX 8(AX), DI + MOVQ 16(AX), BX + MOVQ 48(AX), SI + MOVQ 24(AX), R8 + MOVQ 32(AX), R9 + MOVQ (AX), R10 + + // Main loop +main_loop: + XORL DX, DX + CMPQ BX, SI + SETGE DL + + // br0.fillFast32() + MOVQ 32(R10), R11 + MOVBQZX 40(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill0 + MOVQ 24(R10), AX + SUBQ $0x20, R12 + SUBQ $0x04, AX + MOVQ (R10), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 24(R10) + ORQ R13, R11 + + // exhausted += (br0.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br0.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br0.peekTopBits(peekBits) + MOVQ DI, CX + MOVQ R11, R13 + SHRQ CL, R13 + + // v1 := table[val1&mask] + MOVW (R9)(R13*2), CX + + // br0.advance(uint8(v1.entry)) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVW AX, (BX) + + // update the bitreader structure + MOVQ R11, 32(R10) + MOVB R12, 40(R10) + + // br1.fillFast32() + MOVQ 80(R10), R11 + MOVBQZX 88(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill1 + MOVQ 72(R10), AX + SUBQ $0x20, R12 + SUBQ $0x04, AX + MOVQ 48(R10), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 72(R10) + ORQ R13, R11 + + // exhausted += (br1.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br1.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br1.peekTopBits(peekBits) + MOVQ DI, CX + MOVQ R11, R13 + SHRQ CL, R13 + + // v1 := table[val1&mask] + MOVW (R9)(R13*2), CX + + // br1.advance(uint8(v1.entry)) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVW AX, (BX)(R8*1) + + // update the bitreader structure + MOVQ R11, 80(R10) + MOVB R12, 88(R10) + + // br2.fillFast32() + MOVQ 128(R10), R11 + MOVBQZX 136(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill2 + MOVQ 120(R10), AX + SUBQ $0x20, R12 + SUBQ $0x04, AX + MOVQ 96(R10), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 120(R10) + ORQ R13, R11 + + // exhausted += (br2.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br2.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br2.peekTopBits(peekBits) + MOVQ DI, CX + MOVQ R11, R13 + SHRQ CL, R13 + + // v1 := table[val1&mask] + MOVW (R9)(R13*2), CX + + // br2.advance(uint8(v1.entry)) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVW AX, (BX)(R8*2) + + // update the bitreader structure + MOVQ R11, 128(R10) + MOVB R12, 136(R10) + + // br3.fillFast32() + MOVQ 176(R10), R11 + MOVBQZX 184(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill3 + MOVQ 168(R10), AX + SUBQ $0x20, R12 + SUBQ $0x04, AX + MOVQ 144(R10), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 168(R10) + ORQ R13, R11 + + // exhausted += (br3.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br3.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br3.peekTopBits(peekBits) + MOVQ DI, CX + MOVQ R11, R13 + SHRQ CL, R13 + + // v1 := table[val1&mask] + MOVW (R9)(R13*2), CX + + // br3.advance(uint8(v1.entry)) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + LEAQ (R8)(R8*2), CX + MOVW AX, (BX)(CX*1) + + // update the bitreader structure + MOVQ R11, 176(R10) + MOVB R12, 184(R10) + ADDQ $0x02, BX + TESTB DL, DL + JZ main_loop + MOVQ ctx+0(FP), AX + SUBQ 16(AX), BX + SHLQ $0x02, BX + MOVQ BX, 40(AX) + RET + +// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8 + // Preload values + MOVQ ctx+0(FP), CX + MOVBQZX 8(CX), DI + MOVQ 16(CX), BX + MOVQ 48(CX), SI + MOVQ 24(CX), R8 + MOVQ 32(CX), R9 + MOVQ (CX), R10 + + // Main loop +main_loop: + XORL DX, DX + CMPQ BX, SI + SETGE DL + + // br0.fillFast32() + MOVQ 32(R10), R11 + MOVBQZX 40(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill0 + MOVQ 24(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ (R10), R14 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 24(R10) + ORQ R14, R11 + + // exhausted += (br0.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br0.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br0.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v1 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br0.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // val2 := br0.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v2 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br0.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // val3 := br0.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v3 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br0.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVL AX, (BX) + + // update the bitreader structure + MOVQ R11, 32(R10) + MOVB R12, 40(R10) + + // br1.fillFast32() + MOVQ 80(R10), R11 + MOVBQZX 88(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill1 + MOVQ 72(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ 48(R10), R14 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 72(R10) + ORQ R14, R11 + + // exhausted += (br1.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br1.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br1.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v1 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br1.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // val2 := br1.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v2 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br1.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // val3 := br1.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v3 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br1.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVL AX, (BX)(R8*1) + + // update the bitreader structure + MOVQ R11, 80(R10) + MOVB R12, 88(R10) + + // br2.fillFast32() + MOVQ 128(R10), R11 + MOVBQZX 136(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill2 + MOVQ 120(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ 96(R10), R14 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 120(R10) + ORQ R14, R11 + + // exhausted += (br2.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br2.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br2.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v1 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br2.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // val2 := br2.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v2 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br2.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // val3 := br2.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v3 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br2.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVL AX, (BX)(R8*2) + + // update the bitreader structure + MOVQ R11, 128(R10) + MOVB R12, 136(R10) + + // br3.fillFast32() + MOVQ 176(R10), R11 + MOVBQZX 184(R10), R12 + CMPQ R12, $0x20 + JBE skip_fill3 + MOVQ 168(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ 144(R10), R14 + + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 168(R10) + ORQ R14, R11 + + // exhausted += (br3.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v0 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br3.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + + // val1 := br3.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v1 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br3.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // val2 := br3.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v2 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br3.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R11 + ADDB CL, R12 + + // val3 := br3.peekTopBits(peekBits) + MOVQ R11, R13 + MOVQ DI, CX + SHRQ CL, R13 + + // v3 := table[val0&mask] + MOVW (R9)(R13*2), CX + + // br3.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R11 + ADDB CL, R12 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + LEAQ (R8)(R8*2), CX + MOVL AX, (BX)(CX*1) + + // update the bitreader structure + MOVQ R11, 176(R10) + MOVB R12, 184(R10) + ADDQ $0x04, BX + TESTB DL, DL + JZ main_loop + MOVQ ctx+0(FP), AX + SUBQ 16(AX), BX + SHLQ $0x02, BX + MOVQ BX, 40(AX) + RET + +// func decompress1x_main_loop_amd64(ctx *decompress1xContext) +TEXT ·decompress1x_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), CX + MOVQ 16(CX), DX + MOVQ 24(CX), BX + CMPQ BX, $0x04 + JB error_max_decoded_size_exceeded + LEAQ (DX)(BX*1), BX + MOVQ (CX), SI + MOVQ (SI), R8 + MOVQ 24(SI), R9 + MOVQ 32(SI), R10 + MOVBQZX 40(SI), R11 + MOVQ 32(CX), SI + MOVBQZX 8(CX), DI + JMP loop_condition + +main_loop: + // Check if we have room for 4 bytes in the output buffer + LEAQ 4(DX), CX + CMPQ CX, BX + JGE error_max_decoded_size_exceeded + + // Decode 4 values + CMPQ R11, $0x20 + JL bitReader_fillFast_1_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), R12 + MOVQ R11, CX + SHLQ CL, R12 + ORQ R12, R10 + +bitReader_fillFast_1_end: + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + BSWAPL AX + CMPQ R11, $0x20 + JL bitReader_fillFast_2_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), R12 + MOVQ R11, CX + SHLQ CL, R12 + ORQ R12, R10 + +bitReader_fillFast_2_end: + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + BSWAPL AX + + // Store the decoded values + MOVL AX, (DX) + ADDQ $0x04, DX + +loop_condition: + CMPQ R9, $0x08 + JGE main_loop + + // Update ctx structure + MOVQ ctx+0(FP), AX + SUBQ 16(AX), DX + MOVQ DX, 40(AX) + MOVQ (AX), AX + MOVQ R9, 24(AX) + MOVQ R10, 32(AX) + MOVB R11, 40(AX) + RET + + // Report error +error_max_decoded_size_exceeded: + MOVQ ctx+0(FP), AX + MOVQ $-1, CX + MOVQ CX, 40(AX) + RET + +// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2 +TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), CX + MOVQ 16(CX), DX + MOVQ 24(CX), BX + CMPQ BX, $0x04 + JB error_max_decoded_size_exceeded + LEAQ (DX)(BX*1), BX + MOVQ (CX), SI + MOVQ (SI), R8 + MOVQ 24(SI), R9 + MOVQ 32(SI), R10 + MOVBQZX 40(SI), R11 + MOVQ 32(CX), SI + MOVBQZX 8(CX), DI + JMP loop_condition + +main_loop: + // Check if we have room for 4 bytes in the output buffer + LEAQ 4(DX), CX + CMPQ CX, BX + JGE error_max_decoded_size_exceeded + + // Decode 4 values + CMPQ R11, $0x20 + JL bitReader_fillFast_1_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), CX + SHLXQ R11, CX, CX + ORQ CX, R10 + +bitReader_fillFast_1_end: + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + BSWAPL AX + CMPQ R11, $0x20 + JL bitReader_fillFast_2_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), CX + SHLXQ R11, CX, CX + ORQ CX, R10 + +bitReader_fillFast_2_end: + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + BSWAPL AX + + // Store the decoded values + MOVL AX, (DX) + ADDQ $0x04, DX + +loop_condition: + CMPQ R9, $0x08 + JGE main_loop + + // Update ctx structure + MOVQ ctx+0(FP), AX + SUBQ 16(AX), DX + MOVQ DX, 40(AX) + MOVQ (AX), AX + MOVQ R9, 24(AX) + MOVQ R10, 32(AX) + MOVB R11, 40(AX) + RET + + // Report error +error_max_decoded_size_exceeded: + MOVQ ctx+0(FP), AX + MOVQ $-1, CX + MOVQ CX, 40(AX) + RET diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go new file mode 100644 index 000000000..908c17de6 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -0,0 +1,299 @@ +//go:build !amd64 || appengine || !gc || noasm +// +build !amd64 appengine !gc noasm + +// This file contains a generic implementation of Decoder.Decompress4X. +package huff0 + +import ( + "errors" + "fmt" +) + +// Decompress4X will decompress a 4X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if len(src) < 6+(4*1) { + return nil, errors.New("input too small") + } + if use8BitTables && d.actualTableLog <= 8 { + return d.decompress4X8bit(dst, src) + } + + var br [4]bitReaderShifted + // Decode "jump table" + start := 6 + for i := 0; i < 3; i++ { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { + return nil, errors.New("truncated input (or invalid offset)") + } + err := br[i].init(src[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err := br[3].init(src[start:]) + if err != nil { + return nil, err + } + + // destination, offset to match first output + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst + dstEvery := (dstSize + 3) / 4 + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + single := d.dt.single[:tlSize] + + // Use temp table to avoid bound checks/append penalty. + buf := d.buffer() + var off uint8 + var decoded int + + // Decode 2 values from each decoder/loop. + const bufoff = 256 + for { + if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { + break + } + + { + const stream = 0 + const stream2 = 1 + br[stream].fillFast() + br[stream2].fillFast() + + val := br[stream].peekBitsFast(d.actualTableLog) + val2 := br[stream2].peekBitsFast(d.actualTableLog) + v := single[val&tlMask] + v2 := single[val2&tlMask] + br[stream].advance(uint8(v.entry)) + br[stream2].advance(uint8(v2.entry)) + buf[stream][off] = uint8(v.entry >> 8) + buf[stream2][off] = uint8(v2.entry >> 8) + + val = br[stream].peekBitsFast(d.actualTableLog) + val2 = br[stream2].peekBitsFast(d.actualTableLog) + v = single[val&tlMask] + v2 = single[val2&tlMask] + br[stream].advance(uint8(v.entry)) + br[stream2].advance(uint8(v2.entry)) + buf[stream][off+1] = uint8(v.entry >> 8) + buf[stream2][off+1] = uint8(v2.entry >> 8) + } + + { + const stream = 2 + const stream2 = 3 + br[stream].fillFast() + br[stream2].fillFast() + + val := br[stream].peekBitsFast(d.actualTableLog) + val2 := br[stream2].peekBitsFast(d.actualTableLog) + v := single[val&tlMask] + v2 := single[val2&tlMask] + br[stream].advance(uint8(v.entry)) + br[stream2].advance(uint8(v2.entry)) + buf[stream][off] = uint8(v.entry >> 8) + buf[stream2][off] = uint8(v2.entry >> 8) + + val = br[stream].peekBitsFast(d.actualTableLog) + val2 = br[stream2].peekBitsFast(d.actualTableLog) + v = single[val&tlMask] + v2 = single[val2&tlMask] + br[stream].advance(uint8(v.entry)) + br[stream2].advance(uint8(v2.entry)) + buf[stream][off+1] = uint8(v.entry >> 8) + buf[stream2][off+1] = uint8(v2.entry >> 8) + } + + off += 2 + + if off == 0 { + if bufoff > dstEvery { + d.bufs.Put(buf) + return nil, errors.New("corruption detected: stream overrun 1") + } + // There must at least be 3 buffers left. + if len(out)-bufoff < dstEvery*3 { + d.bufs.Put(buf) + return nil, errors.New("corruption detected: stream overrun 2") + } + //copy(out, buf[0][:]) + //copy(out[dstEvery:], buf[1][:]) + //copy(out[dstEvery*2:], buf[2][:]) + //copy(out[dstEvery*3:], buf[3][:]) + *(*[bufoff]byte)(out) = buf[0] + *(*[bufoff]byte)(out[dstEvery:]) = buf[1] + *(*[bufoff]byte)(out[dstEvery*2:]) = buf[2] + *(*[bufoff]byte)(out[dstEvery*3:]) = buf[3] + out = out[bufoff:] + decoded += bufoff * 4 + } + } + if off > 0 { + ioff := int(off) + if len(out) < dstEvery*3+ioff { + d.bufs.Put(buf) + return nil, errors.New("corruption detected: stream overrun 3") + } + copy(out, buf[0][:off]) + copy(out[dstEvery:], buf[1][:off]) + copy(out[dstEvery*2:], buf[2][:off]) + copy(out[dstEvery*3:], buf[3][:off]) + decoded += int(off) * 4 + out = out[off:] + } + + // Decode remaining. + remainBytes := dstEvery - (decoded / 4) + for i := range br { + offset := dstEvery * i + endsAt := offset + remainBytes + if endsAt > len(out) { + endsAt = len(out) + } + br := &br[i] + bitsLeft := br.remaining() + for bitsLeft > 0 { + br.fill() + if offset >= endsAt { + d.bufs.Put(buf) + return nil, errors.New("corruption detected: stream overrun 4") + } + + // Read value and increment offset. + val := br.peekBitsFast(d.actualTableLog) + v := single[val&tlMask].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= uint(nBits) + out[offset] = uint8(v >> 8) + offset++ + } + if offset != endsAt { + d.bufs.Put(buf) + return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + d.bufs.Put(buf) + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return dst, nil +} + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if use8BitTables && d.actualTableLog <= 8 { + return d.decompress1X8Bit(dst, src) + } + var br bitReaderShifted + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:0] + + // Avoid bounds check by always having full sized table. + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + dt := d.dt.single[:tlSize] + + // Use temp table to avoid bound checks/append penalty. + bufs := d.buffer() + buf := &bufs[0] + var off uint8 + + for br.off >= 8 { + br.fillFast() + v := dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + // Refill + br.fillFast() + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + br.close() + d.bufs.Put(bufs) + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + + if len(dst)+int(off) > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:off]...) + + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { + br.fill() + if false && br.bitsRead >= 32 { + if br.off >= 4 { + v := br.in[br.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + br.value = (br.value << 32) | uint64(low) + br.bitsRead -= 32 + br.off -= 4 + } else { + for br.off > 0 { + br.value = (br.value << 8) | uint64(br.in[br.off-1]) + br.bitsRead -= 8 + br.off-- + } + } + } + if len(dst) >= maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + d.bufs.Put(bufs) + return dst, br.close() +} diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index 3ee00ecb4..e8ad17ad0 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -8,6 +8,7 @@ import ( "fmt" "math" "math/bits" + "sync" "github.com/klauspost/compress/fse" ) @@ -116,6 +117,7 @@ type Scratch struct { nodes []nodeElt tmpOut [4][]byte fse *fse.Scratch + decPool sync.Pool // *[4][256]byte buffers. huffWeight [maxSymbolValue + 1]byte } diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go new file mode 100644 index 000000000..3954c5121 --- /dev/null +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go @@ -0,0 +1,34 @@ +// Package cpuinfo gives runtime info about the current CPU. +// +// This is a very limited module meant for use internally +// in this project. For more versatile solution check +// https://github.com/klauspost/cpuid. +package cpuinfo + +// HasBMI1 checks whether an x86 CPU supports the BMI1 extension. +func HasBMI1() bool { + return hasBMI1 +} + +// HasBMI2 checks whether an x86 CPU supports the BMI2 extension. +func HasBMI2() bool { + return hasBMI2 +} + +// DisableBMI2 will disable BMI2, for testing purposes. +// Call returned function to restore previous state. +func DisableBMI2() func() { + old := hasBMI2 + hasBMI2 = false + return func() { + hasBMI2 = old + } +} + +// HasBMI checks whether an x86 CPU supports both BMI1 and BMI2 extensions. +func HasBMI() bool { + return HasBMI1() && HasBMI2() +} + +var hasBMI1 bool +var hasBMI2 bool diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go new file mode 100644 index 000000000..e802579c4 --- /dev/null +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go @@ -0,0 +1,11 @@ +//go:build amd64 && !appengine && !noasm && gc +// +build amd64,!appengine,!noasm,gc + +package cpuinfo + +// go:noescape +func x86extensions() (bmi1, bmi2 bool) + +func init() { + hasBMI1, hasBMI2 = x86extensions() +} diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s new file mode 100644 index 000000000..4465fbe9e --- /dev/null +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s @@ -0,0 +1,36 @@ +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" +#include "funcdata.h" +#include "go_asm.h" + +TEXT ·x86extensions(SB), NOSPLIT, $0 + // 1. determine max EAX value + XORQ AX, AX + CPUID + + CMPQ AX, $7 + JB unsupported + + // 2. EAX = 7, ECX = 0 --- see Table 3-8 "Information Returned by CPUID Instruction" + MOVQ $7, AX + MOVQ $0, CX + CPUID + + BTQ $3, BX // bit 3 = BMI1 + SETCS AL + + BTQ $8, BX // bit 8 = BMI2 + SETCS AH + + MOVB AL, bmi1+0(FP) + MOVB AH, bmi2+1(FP) + RET + +unsupported: + XORQ AX, AX + MOVB AL, bmi1+0(FP) + MOVB AL, bmi2+1(FP) + RET diff --git a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go b/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go index 511bba65d..05db94d39 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go @@ -18,6 +18,7 @@ func load64(b []byte, i int) uint64 { // emitLiteral writes a literal chunk and returns the number of bytes written. // // It assumes that: +// // dst is long enough to hold the encoded bytes // 1 <= len(lit) && len(lit) <= 65536 func emitLiteral(dst, lit []byte) int { @@ -42,6 +43,7 @@ func emitLiteral(dst, lit []byte) int { // emitCopy writes a copy chunk and returns the number of bytes written. // // It assumes that: +// // dst is long enough to hold the encoded bytes // 1 <= offset && offset <= 65535 // 4 <= length && length <= 65535 @@ -89,6 +91,7 @@ func emitCopy(dst []byte, offset, length int) int { // src[i:i+k-j] and src[j:k] have the same contents. // // It assumes that: +// // 0 <= i && i < j && j <= len(src) func extendMatch(src []byte, i, j int) int { for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { @@ -100,13 +103,36 @@ func hash(u, shift uint32) uint32 { return (u * 0x1e35a7bd) >> shift } +// EncodeBlockInto exposes encodeBlock but checks dst size. +func EncodeBlockInto(dst, src []byte) (d int) { + if MaxEncodedLen(len(src)) > len(dst) { + return 0 + } + + // encodeBlock breaks on too big blocks, so split. + for len(src) > 0 { + p := src + src = nil + if len(p) > maxBlockSize { + p, src = p[:maxBlockSize], p[maxBlockSize:] + } + if len(p) < minNonLiteralBlockSize { + d += emitLiteral(dst[d:], p) + } else { + d += encodeBlock(dst[d:], p) + } + } + return d +} + // encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It // assumes that the varint-encoded length of the decompressed bytes has already // been written. // // It also assumes that: +// // len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize +// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize func encodeBlock(dst, src []byte) (d int) { // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. // The table element type is uint16, as s < sLimit and sLimit < len(src) diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index c8f0f16fc..65b38abed 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -12,6 +12,8 @@ The `zstd` package is provided as open source software using a Go standard licen Currently the package is heavily optimized for 64 bit processors and will be significantly slower on 32 bit processors. +For seekable zstd streams, see [this excellent package](https://github.com/SaveTheRbtz/zstd-seekable-format-go). + ## Installation Install using `go get -u github.com/klauspost/compress`. The package is located in `github.com/klauspost/compress/zstd`. @@ -78,6 +80,9 @@ of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is in the future. So if you want to limit concurrency for future updates, specify the concurrency you would like. +If you would like stream encoding to be done without spawning async goroutines, use `WithEncoderConcurrency(1)` +which will compress input as each block is completed, blocking on writes until each has completed. + You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined compression settings can be specified. @@ -104,7 +109,8 @@ and seems to ignore concatenated streams, even though [it is part of the spec](h For compressing small blocks, the returned encoder has a function called `EncodeAll(src, dst []byte) []byte`. `EncodeAll` will encode all input in src and append it to dst. -This function can be called concurrently, but each call will only run on a single goroutine. +This function can be called concurrently. +Each call will only run on a same goroutine as the caller. Encoded blocks can be concatenated and the result will be the combined input stream. Data compressed with EncodeAll can be decoded with the Decoder, using either a stream or `DecodeAll`. @@ -149,10 +155,10 @@ http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip This package: file out level insize outsize millis mb/s -silesia.tar zskp 1 211947520 73101992 643 313.87 -silesia.tar zskp 2 211947520 67504318 969 208.38 -silesia.tar zskp 3 211947520 64595893 2007 100.68 -silesia.tar zskp 4 211947520 60995370 8825 22.90 +silesia.tar zskp 1 211947520 73821326 634 318.47 +silesia.tar zskp 2 211947520 67655404 1508 133.96 +silesia.tar zskp 3 211947520 64746933 3000 67.37 +silesia.tar zskp 4 211947520 60073508 16926 11.94 cgo zstd: silesia.tar zstd 1 211947520 73605392 543 371.56 @@ -161,94 +167,94 @@ silesia.tar zstd 6 211947520 62916450 1913 105.66 silesia.tar zstd 9 211947520 60212393 5063 39.92 gzip, stdlib/this package: -silesia.tar gzstd 1 211947520 80007735 1654 122.21 -silesia.tar gzkp 1 211947520 80136201 1152 175.45 +silesia.tar gzstd 1 211947520 80007735 1498 134.87 +silesia.tar gzkp 1 211947520 80088272 1009 200.31 GOB stream of binary data. Highly compressible. https://files.klauspost.com/compress/gob-stream.7z file out level insize outsize millis mb/s -gob-stream zskp 1 1911399616 235022249 3088 590.30 -gob-stream zskp 2 1911399616 205669791 3786 481.34 -gob-stream zskp 3 1911399616 175034659 9636 189.17 -gob-stream zskp 4 1911399616 165609838 50369 36.19 +gob-stream zskp 1 1911399616 233948096 3230 564.34 +gob-stream zskp 2 1911399616 203997694 4997 364.73 +gob-stream zskp 3 1911399616 173526523 13435 135.68 +gob-stream zskp 4 1911399616 162195235 47559 38.33 gob-stream zstd 1 1911399616 249810424 2637 691.26 gob-stream zstd 3 1911399616 208192146 3490 522.31 gob-stream zstd 6 1911399616 193632038 6687 272.56 gob-stream zstd 9 1911399616 177620386 16175 112.70 -gob-stream gzstd 1 1911399616 357382641 10251 177.82 -gob-stream gzkp 1 1911399616 359753026 5438 335.20 +gob-stream gzstd 1 1911399616 357382013 9046 201.49 +gob-stream gzkp 1 1911399616 359136669 4885 373.08 The test data for the Large Text Compression Benchmark is the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. http://mattmahoney.net/dc/textdata.html file out level insize outsize millis mb/s -enwik9 zskp 1 1000000000 343848582 3609 264.18 -enwik9 zskp 2 1000000000 317276632 5746 165.97 -enwik9 zskp 3 1000000000 292243069 12162 78.41 -enwik9 zskp 4 1000000000 262183768 82837 11.51 +enwik9 zskp 1 1000000000 343833605 3687 258.64 +enwik9 zskp 2 1000000000 317001237 7672 124.29 +enwik9 zskp 3 1000000000 291915823 15923 59.89 +enwik9 zskp 4 1000000000 261710291 77697 12.27 enwik9 zstd 1 1000000000 358072021 3110 306.65 enwik9 zstd 3 1000000000 313734672 4784 199.35 enwik9 zstd 6 1000000000 295138875 10290 92.68 enwik9 zstd 9 1000000000 278348700 28549 33.40 -enwik9 gzstd 1 1000000000 382578136 9604 99.30 -enwik9 gzkp 1 1000000000 383825945 6544 145.73 +enwik9 gzstd 1 1000000000 382578136 8608 110.78 +enwik9 gzkp 1 1000000000 382781160 5628 169.45 Highly compressible JSON file. https://files.klauspost.com/compress/github-june-2days-2019.json.zst file out level insize outsize millis mb/s -github-june-2days-2019.json zskp 1 6273951764 699045015 10620 563.40 -github-june-2days-2019.json zskp 2 6273951764 617881763 11687 511.96 -github-june-2days-2019.json zskp 3 6273951764 524340691 34043 175.75 -github-june-2days-2019.json zskp 4 6273951764 470320075 170190 35.16 +github-june-2days-2019.json zskp 1 6273951764 697439532 9789 611.17 +github-june-2days-2019.json zskp 2 6273951764 610876538 18553 322.49 +github-june-2days-2019.json zskp 3 6273951764 517662858 44186 135.41 +github-june-2days-2019.json zskp 4 6273951764 464617114 165373 36.18 github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00 github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57 github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18 github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16 -github-june-2days-2019.json gzstd 1 6273951764 1164400847 29948 199.79 -github-june-2days-2019.json gzkp 1 6273951764 1125417694 21788 274.61 +github-june-2days-2019.json gzstd 1 6273951764 1164397768 26793 223.32 +github-june-2days-2019.json gzkp 1 6273951764 1120631856 17693 338.16 VM Image, Linux mint with a few installed applications: https://files.klauspost.com/compress/rawstudio-mint14.7z file out level insize outsize millis mb/s -rawstudio-mint14.tar zskp 1 8558382592 3667489370 20210 403.84 -rawstudio-mint14.tar zskp 2 8558382592 3364592300 31873 256.07 -rawstudio-mint14.tar zskp 3 8558382592 3158085214 77675 105.08 -rawstudio-mint14.tar zskp 4 8558382592 2965110639 857750 9.52 +rawstudio-mint14.tar zskp 1 8558382592 3718400221 18206 448.29 +rawstudio-mint14.tar zskp 2 8558382592 3326118337 37074 220.15 +rawstudio-mint14.tar zskp 3 8558382592 3163842361 87306 93.49 +rawstudio-mint14.tar zskp 4 8558382592 2970480650 783862 10.41 rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27 rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92 rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77 rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91 -rawstudio-mint14.tar gzstd 1 8558382592 3926257486 57722 141.40 -rawstudio-mint14.tar gzkp 1 8558382592 3962605659 45113 180.92 +rawstudio-mint14.tar gzstd 1 8558382592 3926234992 51345 158.96 +rawstudio-mint14.tar gzkp 1 8558382592 3960117298 36722 222.26 CSV data: https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst file out level insize outsize millis mb/s -nyc-taxi-data-10M.csv zskp 1 3325605752 641339945 8925 355.35 -nyc-taxi-data-10M.csv zskp 2 3325605752 591748091 11268 281.44 -nyc-taxi-data-10M.csv zskp 3 3325605752 530289687 25239 125.66 -nyc-taxi-data-10M.csv zskp 4 3325605752 476268884 135958 23.33 +nyc-taxi-data-10M.csv zskp 1 3325605752 641319332 9462 335.17 +nyc-taxi-data-10M.csv zskp 2 3325605752 588976126 17570 180.50 +nyc-taxi-data-10M.csv zskp 3 3325605752 529329260 32432 97.79 +nyc-taxi-data-10M.csv zskp 4 3325605752 474949772 138025 22.98 nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18 nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07 nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27 nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12 -nyc-taxi-data-10M.csv gzstd 1 3325605752 928656485 23876 132.83 -nyc-taxi-data-10M.csv gzkp 1 3325605752 922257165 16780 189.00 +nyc-taxi-data-10M.csv gzstd 1 3325605752 928654908 21270 149.11 +nyc-taxi-data-10M.csv gzkp 1 3325605752 922273214 13929 227.68 ``` ## Decompressor @@ -283,8 +289,13 @@ func Decompress(in io.Reader, out io.Writer) error { } ``` -It is important to use the "Close" function when you no longer need the Reader to stop running goroutines. -See "Allocation-less operation" below. +It is important to use the "Close" function when you no longer need the Reader to stop running goroutines, +when running with default settings. +Goroutines will exit once an error has been returned, including `io.EOF` at the end of a stream. + +Streams are decoded concurrently in 4 asynchronous stages to give the best possible throughput. +However, if you prefer synchronous decompression, use `WithDecoderConcurrency(1)` which will decompress data +as it is being requested only. For decoding buffers, it could look something like this: @@ -293,7 +304,7 @@ import "github.com/klauspost/compress/zstd" // Create a reader that caches decompressors. // For this operation type we supply a nil Reader. -var decoder, _ = zstd.NewReader(nil) +var decoder, _ = zstd.NewReader(nil, WithDecoderConcurrency(0)) // Decompress a buffer. We don't supply a destination buffer, // so it will be allocated by the decoder. @@ -303,9 +314,12 @@ func Decompress(src []byte) ([]byte, error) { ``` Both of these cases should provide the functionality needed. -The decoder can be used for *concurrent* decompression of multiple buffers. +The decoder can be used for *concurrent* decompression of multiple buffers. +By default 4 decompressors will be created. + It will only allow a certain number of concurrent operations to run. -To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder. +To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder. +It is possible to use `WithDecoderConcurrency(0)` to create GOMAXPROCS decoders. ### Dictionaries @@ -357,62 +371,48 @@ In this case no unneeded allocations should be made. The buffer decoder does everything on the same goroutine and does nothing concurrently. It can however decode several buffers concurrently. Use `WithDecoderConcurrency(n)` to limit that. -The stream decoder operates on +The stream decoder will create goroutines that: -* One goroutine reads input and splits the input to several block decoders. -* A number of decoders will decode blocks. -* A goroutine coordinates these blocks and sends history from one to the next. +1) Reads input and splits the input into blocks. +2) Decompression of literals. +3) Decompression of sequences. +4) Reconstruction of output stream. So effectively this also means the decoder will "read ahead" and prepare data to always be available for output. +The concurrency level will, for streams, determine how many blocks ahead the compression will start. + Since "blocks" are quite dependent on the output of the previous block stream decoding will only have limited concurrency. -In practice this means that concurrency is often limited to utilizing about 2 cores effectively. - - +In practice this means that concurrency is often limited to utilizing about 3 cores effectively. + ### Benchmarks -These are some examples of performance compared to [datadog cgo library](https://github.com/DataDog/zstd). - The first two are streaming decodes and the last are smaller inputs. - + +Running on AMD Ryzen 9 3950X 16-Core Processor. AMD64 assembly used. + ``` -BenchmarkDecoderSilesia-8 3 385000067 ns/op 550.51 MB/s 5498 B/op 8 allocs/op -BenchmarkDecoderSilesiaCgo-8 6 197666567 ns/op 1072.25 MB/s 270672 B/op 8 allocs/op - -BenchmarkDecoderEnwik9-8 1 2027001600 ns/op 493.34 MB/s 10496 B/op 18 allocs/op -BenchmarkDecoderEnwik9Cgo-8 2 979499200 ns/op 1020.93 MB/s 270672 B/op 8 allocs/op - -Concurrent performance: - -BenchmarkDecoder_DecodeAllParallel/kppkn.gtb.zst-16 28915 42469 ns/op 4340.07 MB/s 114 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/geo.protodata.zst-16 116505 9965 ns/op 11900.16 MB/s 16 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/plrabn12.txt.zst-16 8952 134272 ns/op 3588.70 MB/s 915 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/lcet10.txt.zst-16 11820 102538 ns/op 4161.90 MB/s 594 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/asyoulik.txt.zst-16 34782 34184 ns/op 3661.88 MB/s 60 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/alice29.txt.zst-16 27712 43447 ns/op 3500.58 MB/s 99 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/html_x_4.zst-16 62826 18750 ns/op 21845.10 MB/s 104 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/paper-100k.pdf.zst-16 631545 1794 ns/op 57078.74 MB/s 2 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/fireworks.jpeg.zst-16 1690140 712 ns/op 172938.13 MB/s 1 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/urls.10K.zst-16 10432 113593 ns/op 6180.73 MB/s 1143 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/html.zst-16 113206 10671 ns/op 9596.27 MB/s 15 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallel/comp-data.bin.zst-16 1530615 779 ns/op 5229.49 MB/s 0 B/op 0 allocs/op - -BenchmarkDecoder_DecodeAllParallelCgo/kppkn.gtb.zst-16 65217 16192 ns/op 11383.34 MB/s 46 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/geo.protodata.zst-16 292671 4039 ns/op 29363.19 MB/s 6 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/plrabn12.txt.zst-16 26314 46021 ns/op 10470.43 MB/s 293 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/lcet10.txt.zst-16 33897 34900 ns/op 12227.96 MB/s 205 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/asyoulik.txt.zst-16 104348 11433 ns/op 10949.01 MB/s 20 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/alice29.txt.zst-16 75949 15510 ns/op 9805.60 MB/s 32 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/html_x_4.zst-16 173910 6756 ns/op 60624.29 MB/s 37 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/paper-100k.pdf.zst-16 923076 1339 ns/op 76474.87 MB/s 1 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/fireworks.jpeg.zst-16 922920 1351 ns/op 91102.57 MB/s 2 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/urls.10K.zst-16 27649 43618 ns/op 16096.19 MB/s 407 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/html.zst-16 279073 4160 ns/op 24614.18 MB/s 6 B/op 0 allocs/op -BenchmarkDecoder_DecodeAllParallelCgo/comp-data.bin.zst-16 749938 1579 ns/op 2581.71 MB/s 0 B/op 0 allocs/op +BenchmarkDecoderSilesia-32 5 206878840 ns/op 1024.50 MB/s 49808 B/op 43 allocs/op +BenchmarkDecoderEnwik9-32 1 1271809000 ns/op 786.28 MB/s 72048 B/op 52 allocs/op + +Concurrent blocks, performance: + +BenchmarkDecoder_DecodeAllParallel/kppkn.gtb.zst-32 67356 17857 ns/op 10321.96 MB/s 22.48 pct 102 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/geo.protodata.zst-32 266656 4421 ns/op 26823.21 MB/s 11.89 pct 19 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/plrabn12.txt.zst-32 20992 56842 ns/op 8477.17 MB/s 39.90 pct 754 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/lcet10.txt.zst-32 27456 43932 ns/op 9714.01 MB/s 33.27 pct 524 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/asyoulik.txt.zst-32 78432 15047 ns/op 8319.15 MB/s 40.34 pct 66 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/alice29.txt.zst-32 65800 18436 ns/op 8249.63 MB/s 37.75 pct 88 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/html_x_4.zst-32 102993 11523 ns/op 35546.09 MB/s 3.637 pct 143 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/paper-100k.pdf.zst-32 1000000 1070 ns/op 95720.98 MB/s 80.53 pct 3 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/fireworks.jpeg.zst-32 749802 1752 ns/op 70272.35 MB/s 100.0 pct 5 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/urls.10K.zst-32 22640 52934 ns/op 13263.37 MB/s 26.25 pct 1014 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/html.zst-32 226412 5232 ns/op 19572.27 MB/s 14.49 pct 20 B/op 0 allocs/op +BenchmarkDecoder_DecodeAllParallel/comp-data.bin.zst-32 923041 1276 ns/op 3194.71 MB/s 31.26 pct 0 B/op 0 allocs/op ``` -This reflects the performance around May 2020, but this may be out of date. +This reflects the performance around May 2022, but this may be out of date. ## Zstd inside ZIP files diff --git a/vendor/github.com/klauspost/compress/zstd/bitreader.go b/vendor/github.com/klauspost/compress/zstd/bitreader.go index 854458537..97299d499 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitreader.go +++ b/vendor/github.com/klauspost/compress/zstd/bitreader.go @@ -7,6 +7,7 @@ package zstd import ( "encoding/binary" "errors" + "fmt" "io" "math/bits" ) @@ -50,16 +51,16 @@ func (b *bitReader) getBits(n uint8) int { if n == 0 /*|| b.bitsRead >= 64 */ { return 0 } - return b.getBitsFast(n) + return int(b.get32BitsFast(n)) } -// getBitsFast requires that at least one bit is requested every time. +// get32BitsFast requires that at least one bit is requested every time. // There are no checks if the buffer is filled. -func (b *bitReader) getBitsFast(n uint8) int { +func (b *bitReader) get32BitsFast(n uint8) uint32 { const regMask = 64 - 1 v := uint32((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) b.bitsRead += n - return int(v) + return v } // fillFast() will make sure at least 32 bits are available. @@ -125,6 +126,9 @@ func (b *bitReader) remain() uint { func (b *bitReader) close() error { // Release reference. b.in = nil + if !b.finished() { + return fmt.Errorf("%d extra bits on block, should be 0", b.remain()) + } if b.bitsRead > 64 { return io.ErrUnexpectedEOF } diff --git a/vendor/github.com/klauspost/compress/zstd/bitwriter.go b/vendor/github.com/klauspost/compress/zstd/bitwriter.go index 303ae90f9..78b3c61be 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitwriter.go +++ b/vendor/github.com/klauspost/compress/zstd/bitwriter.go @@ -5,8 +5,6 @@ package zstd -import "fmt" - // bitWriter will write bits. // First bit will be LSB of the first byte of output. type bitWriter struct { @@ -38,7 +36,7 @@ func (b *bitWriter) addBits16NC(value uint16, bits uint8) { b.nBits += bits } -// addBits32NC will add up to 32 bits. +// addBits32NC will add up to 31 bits. // It will not check if there is space for them, // so the caller must ensure that it has flushed recently. func (b *bitWriter) addBits32NC(value uint32, bits uint8) { @@ -46,6 +44,26 @@ func (b *bitWriter) addBits32NC(value uint32, bits uint8) { b.nBits += bits } +// addBits64NC will add up to 64 bits. +// There must be space for 32 bits. +func (b *bitWriter) addBits64NC(value uint64, bits uint8) { + if bits <= 31 { + b.addBits32Clean(uint32(value), bits) + return + } + b.addBits32Clean(uint32(value), 32) + b.flush32() + b.addBits32Clean(uint32(value>>32), bits-32) +} + +// addBits32Clean will add up to 32 bits. +// It will not check if there is space for them. +// The input must not contain more bits than specified. +func (b *bitWriter) addBits32Clean(value uint32, bits uint8) { + b.bitContainer |= uint64(value) << (b.nBits & 63) + b.nBits += bits +} + // addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. // It will not check if there is space for them, so the caller must ensure that it has flushed recently. func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { @@ -53,80 +71,6 @@ func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { b.nBits += bits } -// flush will flush all pending full bytes. -// There will be at least 56 bits available for writing when this has been called. -// Using flush32 is faster, but leaves less space for writing. -func (b *bitWriter) flush() { - v := b.nBits >> 3 - switch v { - case 0: - case 1: - b.out = append(b.out, - byte(b.bitContainer), - ) - case 2: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - ) - case 3: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - ) - case 4: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - ) - case 5: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - ) - case 6: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - ) - case 7: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - ) - case 8: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - byte(b.bitContainer>>56), - ) - default: - panic(fmt.Errorf("bits (%d) > 64", b.nBits)) - } - b.bitContainer >>= v << 3 - b.nBits &= 7 -} - // flush32 will flush out, so there are at least 32 bits available for writing. func (b *bitWriter) flush32() { if b.nBits < 32 { diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 8a98c4562..2445bb4fe 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -5,9 +5,13 @@ package zstd import ( + "bytes" + "encoding/binary" "errors" "fmt" "io" + "os" + "path/filepath" "sync" "github.com/klauspost/compress/huff0" @@ -38,14 +42,14 @@ const ( // maxCompressedBlockSize is the biggest allowed compressed block size (128KB) maxCompressedBlockSize = 128 << 10 + compressedBlockOverAlloc = 16 + maxCompressedBlockSizeAlloc = 128<<10 + compressedBlockOverAlloc + // Maximum possible block size (all Raw+Uncompressed). maxBlockSize = (1 << 21) - 1 - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#literals_section_header - maxCompressedLiteralSize = 1 << 18 - maxRLELiteralSize = 1 << 20 - maxMatchLen = 131074 - maxSequences = 0x7f00 + 0xffff + maxMatchLen = 131074 + maxSequences = 0x7f00 + 0xffff // We support slightly less than the reference decoder to be able to // use ints on 32 bit archs. @@ -76,20 +80,28 @@ type blockDec struct { // Window size of the block. WindowSize uint64 - history chan *history - input chan struct{} - result chan decodeOutput - sequenceBuf []seq - err error - decWG sync.WaitGroup + err error + + // Check against this crc, if hasCRC is true. + checkCRC uint32 + hasCRC bool // Frame to use for singlethreaded decoding. // Should not be used by the decoder itself since parent may be another frame. localFrame *frameDec + sequence []seqVals + + async struct { + newHist *history + literals []byte + seqData []byte + seqSize int // Size of uncompressed sequences + fcs uint64 + } + // Block is RLE, this is the size. RLESize uint32 - tmp [4]byte Type blockType @@ -109,13 +121,8 @@ func (b *blockDec) String() string { func newBlockDec(lowMem bool) *blockDec { b := blockDec{ - lowMem: lowMem, - result: make(chan decodeOutput, 1), - input: make(chan struct{}, 1), - history: make(chan *history, 1), + lowMem: lowMem, } - b.decWG.Add(1) - go b.startDecoder() return &b } @@ -133,11 +140,17 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { b.Type = blockType((bh >> 1) & 3) // find size. cSize := int(bh >> 3) - maxSize := maxBlockSize + maxSize := maxCompressedBlockSizeAlloc switch b.Type { case blockTypeReserved: return ErrReservedBlockType case blockTypeRLE: + if cSize > maxCompressedBlockSize || cSize > int(b.WindowSize) { + if debugDecoder { + printf("rle block too big: csize:%d block: %+v\n", uint64(cSize), b) + } + return ErrWindowSizeExceeded + } b.RLESize = uint32(cSize) if b.lowMem { maxSize = cSize @@ -148,9 +161,9 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { println("Data size on stream:", cSize) } b.RLESize = 0 - maxSize = maxCompressedBlockSize + maxSize = maxCompressedBlockSizeAlloc if windowSize < maxCompressedBlockSize && b.lowMem { - maxSize = int(windowSize) + maxSize = int(windowSize) + compressedBlockOverAlloc } if cSize > maxCompressedBlockSize || uint64(cSize) > b.WindowSize { if debugDecoder { @@ -158,7 +171,19 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { } return ErrCompressedSizeTooBig } + // Empty compressed blocks must at least be 2 bytes + // for Literals_Block_Type and one for Sequences_Section_Header. + if cSize < 2 { + return ErrBlockTooSmall + } case blockTypeRaw: + if cSize > maxCompressedBlockSize || cSize > int(b.WindowSize) { + if debugDecoder { + printf("rle block too big: csize:%d block: %+v\n", uint64(cSize), b) + } + return ErrWindowSizeExceeded + } + b.RLESize = 0 // We do not need a destination for raw blocks. maxSize = -1 @@ -167,16 +192,14 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { } // Read block data. - if cap(b.dataStorage) < cSize { + if _, ok := br.(*byteBuf); !ok && cap(b.dataStorage) < cSize { + // byteBuf doesn't need a destination buffer. if b.lowMem || cSize > maxCompressedBlockSize { - b.dataStorage = make([]byte, 0, cSize) + b.dataStorage = make([]byte, 0, cSize+compressedBlockOverAlloc) } else { - b.dataStorage = make([]byte, 0, maxCompressedBlockSize) + b.dataStorage = make([]byte, 0, maxCompressedBlockSizeAlloc) } } - if cap(b.dst) <= maxSize { - b.dst = make([]byte, 0, maxSize+1) - } b.data, err = br.readBig(cSize, b.dataStorage) if err != nil { if debugDecoder { @@ -185,6 +208,9 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { } return err } + if cap(b.dst) <= maxSize { + b.dst = make([]byte, 0, maxSize+1) + } return nil } @@ -193,85 +219,14 @@ func (b *blockDec) sendErr(err error) { b.Last = true b.Type = blockTypeReserved b.err = err - b.input <- struct{}{} } // Close will release resources. // Closed blockDec cannot be reset. func (b *blockDec) Close() { - close(b.input) - close(b.history) - close(b.result) - b.decWG.Wait() -} - -// decodeAsync will prepare decoding the block when it receives input. -// This will separate output and history. -func (b *blockDec) startDecoder() { - defer b.decWG.Done() - for range b.input { - //println("blockDec: Got block input") - switch b.Type { - case blockTypeRLE: - if cap(b.dst) < int(b.RLESize) { - if b.lowMem { - b.dst = make([]byte, b.RLESize) - } else { - b.dst = make([]byte, maxBlockSize) - } - } - o := decodeOutput{ - d: b, - b: b.dst[:b.RLESize], - err: nil, - } - v := b.data[0] - for i := range o.b { - o.b[i] = v - } - hist := <-b.history - hist.append(o.b) - b.result <- o - case blockTypeRaw: - o := decodeOutput{ - d: b, - b: b.data, - err: nil, - } - hist := <-b.history - hist.append(o.b) - b.result <- o - case blockTypeCompressed: - b.dst = b.dst[:0] - err := b.decodeCompressed(nil) - o := decodeOutput{ - d: b, - b: b.dst, - err: err, - } - if debugDecoder { - println("Decompressed to", len(b.dst), "bytes, error:", err) - } - b.result <- o - case blockTypeReserved: - // Used for returning errors. - <-b.history - b.result <- decodeOutput{ - d: b, - b: nil, - err: b.err, - } - default: - panic("Invalid block type") - } - if debugDecoder { - println("blockDec: Finished block") - } - } } -// decodeAsync will prepare decoding the block when it receives the history. -// If history is provided, it will not fetch it from the channel. +// decodeBuf func (b *blockDec) decodeBuf(hist *history) error { switch b.Type { case blockTypeRLE: @@ -279,7 +234,7 @@ func (b *blockDec) decodeBuf(hist *history) error { if b.lowMem { b.dst = make([]byte, b.RLESize) } else { - b.dst = make([]byte, maxBlockSize) + b.dst = make([]byte, maxCompressedBlockSize) } } b.dst = b.dst[:b.RLESize] @@ -294,14 +249,23 @@ func (b *blockDec) decodeBuf(hist *history) error { return nil case blockTypeCompressed: saved := b.dst - b.dst = hist.b - hist.b = nil + // Append directly to history + if hist.ignoreBuffer == 0 { + b.dst = hist.b + hist.b = nil + } else { + b.dst = b.dst[:0] + } err := b.decodeCompressed(hist) if debugDecoder { println("Decompressed to total", len(b.dst), "bytes, hash:", xxhash.Sum64(b.dst), "error:", err) } - hist.b = b.dst - b.dst = saved + if hist.ignoreBuffer == 0 { + hist.b = b.dst + b.dst = saved + } else { + hist.appendKeep(b.dst) + } return err case blockTypeReserved: // Used for returning errors. @@ -311,30 +275,18 @@ func (b *blockDec) decodeBuf(hist *history) error { } } -// decodeCompressed will start decompressing a block. -// If no history is supplied the decoder will decodeAsync as much as possible -// before fetching from blockDec.history -func (b *blockDec) decodeCompressed(hist *history) error { - in := b.data - delayedHistory := hist == nil - - if delayedHistory { - // We must always grab history. - defer func() { - if hist == nil { - <-b.history - } - }() - } +func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err error) { // There must be at least one byte for Literals_Block_Type and one for Sequences_Section_Header if len(in) < 2 { - return ErrBlockTooSmall + return in, ErrBlockTooSmall } + litType := literalsBlockType(in[0] & 3) var litRegenSize int var litCompSize int sizeFormat := (in[0] >> 2) & 3 var fourStreams bool + var literals []byte switch litType { case literalsBlockRaw, literalsBlockRLE: switch sizeFormat { @@ -350,7 +302,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { // Regenerated_Size uses 20 bits (0-1048575). Literals_Section_Header uses 3 bytes. if len(in) < 3 { println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) + (int(in[2]) << 12) in = in[3:] @@ -361,7 +313,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { // Both Regenerated_Size and Compressed_Size use 10 bits (0-1023). if len(in) < 3 { println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) litRegenSize = int(n & 1023) @@ -372,7 +324,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { fourStreams = true if len(in) < 4 { println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) litRegenSize = int(n & 16383) @@ -382,7 +334,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { fourStreams = true if len(in) < 5 { println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) + (uint64(in[4]) << 28) litRegenSize = int(n & 262143) @@ -393,13 +345,15 @@ func (b *blockDec) decodeCompressed(hist *history) error { if debugDecoder { println("literals type:", litType, "litRegenSize:", litRegenSize, "litCompSize:", litCompSize, "sizeFormat:", sizeFormat, "4X:", fourStreams) } - var literals []byte - var huff *huff0.Scratch + if litRegenSize > int(b.WindowSize) || litRegenSize > maxCompressedBlockSize { + return in, ErrWindowSizeExceeded + } + switch litType { case literalsBlockRaw: if len(in) < litRegenSize { println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litRegenSize) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } literals = in[:litRegenSize] in = in[litRegenSize:] @@ -407,19 +361,13 @@ func (b *blockDec) decodeCompressed(hist *history) error { case literalsBlockRLE: if len(in) < 1 { println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", 1) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } if cap(b.literalBuf) < litRegenSize { if b.lowMem { - b.literalBuf = make([]byte, litRegenSize) + b.literalBuf = make([]byte, litRegenSize, litRegenSize+compressedBlockOverAlloc) } else { - if litRegenSize > maxCompressedLiteralSize { - // Exceptional - b.literalBuf = make([]byte, litRegenSize) - } else { - b.literalBuf = make([]byte, litRegenSize, maxCompressedLiteralSize) - - } + b.literalBuf = make([]byte, litRegenSize, maxCompressedBlockSize+compressedBlockOverAlloc) } } literals = b.literalBuf[:litRegenSize] @@ -434,7 +382,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { case literalsBlockTreeless: if len(in) < litCompSize { println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } // Store compressed literals, so we defer decoding until we get history. literals = in[:litCompSize] @@ -442,31 +390,65 @@ func (b *blockDec) decodeCompressed(hist *history) error { if debugDecoder { printf("Found %d compressed literals\n", litCompSize) } + huff := hist.huffTree + if huff == nil { + return in, errors.New("literal block was treeless, but no history was defined") + } + // Ensure we have space to store it. + if cap(b.literalBuf) < litRegenSize { + if b.lowMem { + b.literalBuf = make([]byte, 0, litRegenSize+compressedBlockOverAlloc) + } else { + b.literalBuf = make([]byte, 0, maxCompressedBlockSize+compressedBlockOverAlloc) + } + } + var err error + // Use our out buffer. + huff.MaxDecodedSize = litRegenSize + if fourStreams { + literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) + } else { + literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) + } + // Make sure we don't leak our literals buffer + if err != nil { + println("decompressing literals:", err) + return in, err + } + if len(literals) != litRegenSize { + return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) + } + case literalsBlockCompressed: if len(in) < litCompSize { println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) - return ErrBlockTooSmall + return in, ErrBlockTooSmall } literals = in[:litCompSize] in = in[litCompSize:] - huff = huffDecoderPool.Get().(*huff0.Scratch) - var err error // Ensure we have space to store it. if cap(b.literalBuf) < litRegenSize { if b.lowMem { - b.literalBuf = make([]byte, 0, litRegenSize) + b.literalBuf = make([]byte, 0, litRegenSize+compressedBlockOverAlloc) } else { - b.literalBuf = make([]byte, 0, maxCompressedLiteralSize) + b.literalBuf = make([]byte, 0, maxCompressedBlockSize+compressedBlockOverAlloc) } } - if huff == nil { - huff = &huff0.Scratch{} + huff := hist.huffTree + if huff == nil || (hist.dict != nil && huff == hist.dict.litEnc) { + huff = huffDecoderPool.Get().(*huff0.Scratch) + if huff == nil { + huff = &huff0.Scratch{} + } } + var err error huff, literals, err = huff0.ReadTable(literals, huff) if err != nil { println("reading huffman table:", err) - return err + return in, err } + hist.huffTree = huff + huff.MaxDecodedSize = litRegenSize // Use our out buffer. if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) @@ -475,27 +457,63 @@ func (b *blockDec) decodeCompressed(hist *history) error { } if err != nil { println("decoding compressed literals:", err) - return err + return in, err } // Make sure we don't leak our literals buffer if len(literals) != litRegenSize { - return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) + return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) } + // Re-cap to get extra size. + literals = b.literalBuf[:len(literals)] if debugDecoder { printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize) } } + hist.decoders.literals = literals + return in, nil +} + +// decodeCompressed will start decompressing a block. +func (b *blockDec) decodeCompressed(hist *history) error { + in := b.data + in, err := b.decodeLiterals(in, hist) + if err != nil { + return err + } + err = b.prepareSequences(in, hist) + if err != nil { + return err + } + if hist.decoders.nSeqs == 0 { + b.dst = append(b.dst, hist.decoders.literals...) + return nil + } + before := len(hist.decoders.out) + err = hist.decoders.decodeSync(hist.b[hist.ignoreBuffer:]) + if err != nil { + return err + } + if hist.decoders.maxSyncLen > 0 { + hist.decoders.maxSyncLen += uint64(before) + hist.decoders.maxSyncLen -= uint64(len(hist.decoders.out)) + } + b.dst = hist.decoders.out + hist.recentOffsets = hist.decoders.prevOffset + return nil +} +func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { + if debugDecoder { + printf("prepareSequences: %d byte(s) input\n", len(in)) + } // Decode Sequences // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#sequences-section if len(in) < 1 { return ErrBlockTooSmall } + var nSeqs int seqHeader := in[0] - nSeqs := 0 switch { - case seqHeader == 0: - in = in[1:] case seqHeader < 128: nSeqs = int(seqHeader) in = in[1:] @@ -512,19 +530,16 @@ func (b *blockDec) decodeCompressed(hist *history) error { nSeqs = 0x7f00 + int(in[1]) + (int(in[2]) << 8) in = in[3:] } - // Allocate sequences - if cap(b.sequenceBuf) < nSeqs { - if b.lowMem { - b.sequenceBuf = make([]seq, nSeqs) - } else { - // Allocate max - b.sequenceBuf = make([]seq, nSeqs, maxSequences) + if nSeqs == 0 && len(in) != 0 { + // When no sequences, there should not be any more data... + if debugDecoder { + printf("prepareSequences: 0 sequences, but %d byte(s) left on stream\n", len(in)) } - } else { - // Reuse buffer - b.sequenceBuf = b.sequenceBuf[:nSeqs] + return ErrUnexpectedBlockSize } - var seqs = &sequenceDecs{} + + var seqs = &hist.decoders + seqs.nSeqs = nSeqs if nSeqs > 0 { if len(in) < 1 { return ErrBlockTooSmall @@ -553,6 +568,9 @@ func (b *blockDec) decodeCompressed(hist *history) error { } switch mode { case compModePredefined: + if seq.fse != nil && !seq.fse.preDefined { + fseDecoderPool.Put(seq.fse) + } seq.fse = &fsePredef[i] case compModeRLE: if br.remain() < 1 { @@ -560,34 +578,36 @@ func (b *blockDec) decodeCompressed(hist *history) error { } v := br.Uint8() br.advance(1) - dec := fseDecoderPool.Get().(*fseDecoder) + if seq.fse == nil || seq.fse.preDefined { + seq.fse = fseDecoderPool.Get().(*fseDecoder) + } symb, err := decSymbolValue(v, symbolTableX[i]) if err != nil { printf("RLE Transform table (%v) error: %v", tableIndex(i), err) return err } - dec.setRLE(symb) - seq.fse = dec + seq.fse.setRLE(symb) if debugDecoder { printf("RLE set to %+v, code: %v", symb, v) } case compModeFSE: println("Reading table for", tableIndex(i)) - dec := fseDecoderPool.Get().(*fseDecoder) - err := dec.readNCount(&br, uint16(maxTableSymbol[i])) + if seq.fse == nil || seq.fse.preDefined { + seq.fse = fseDecoderPool.Get().(*fseDecoder) + } + err := seq.fse.readNCount(&br, uint16(maxTableSymbol[i])) if err != nil { println("Read table error:", err) return err } - err = dec.transform(symbolTableX[i]) + err = seq.fse.transform(symbolTableX[i]) if err != nil { println("Transform table error:", err) return err } if debugDecoder { - println("Read table ok", "symbolLen:", dec.symbolLen) + println("Read table ok", "symbolLen:", seq.fse.symbolLen) } - seq.fse = dec case compModeRepeat: seq.repeat = true } @@ -597,140 +617,106 @@ func (b *blockDec) decodeCompressed(hist *history) error { } in = br.unread() } - - // Wait for history. - // All time spent after this is critical since it is strictly sequential. - if hist == nil { - hist = <-b.history - if hist.error { - return ErrDecoderClosed - } - } - - // Decode treeless literal block. - if litType == literalsBlockTreeless { - // TODO: We could send the history early WITHOUT the stream history. - // This would allow decoding treeless literals before the byte history is available. - // Silencia stats: Treeless 4393, with: 32775, total: 37168, 11% treeless. - // So not much obvious gain here. - - if hist.huffTree == nil { - return errors.New("literal block was treeless, but no history was defined") - } - // Ensure we have space to store it. - if cap(b.literalBuf) < litRegenSize { - if b.lowMem { - b.literalBuf = make([]byte, 0, litRegenSize) - } else { - b.literalBuf = make([]byte, 0, maxCompressedLiteralSize) - } - } - var err error - // Use our out buffer. - huff = hist.huffTree - if fourStreams { - literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) - } else { - literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) - } - // Make sure we don't leak our literals buffer - if err != nil { - println("decompressing literals:", err) - return err - } - if len(literals) != litRegenSize { - return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) - } - } else { - if hist.huffTree != nil && huff != nil { - if hist.dict == nil || hist.dict.litEnc != hist.huffTree { - huffDecoderPool.Put(hist.huffTree) - } - hist.huffTree = nil - } - } - if huff != nil { - hist.huffTree = huff - } if debugDecoder { - println("Final literals:", len(literals), "hash:", xxhash.Sum64(literals), "and", nSeqs, "sequences.") + println("Literals:", len(seqs.literals), "hash:", xxhash.Sum64(seqs.literals), "and", seqs.nSeqs, "sequences.") } if nSeqs == 0 { - // Decompressed content is defined entirely as Literals Section content. - b.dst = append(b.dst, literals...) - if delayedHistory { - hist.append(literals) + if len(b.sequence) > 0 { + b.sequence = b.sequence[:0] } return nil } + br := seqs.br + if br == nil { + br = &bitReader{} + } + if err := br.init(in); err != nil { + return err + } - seqs, err := seqs.mergeHistory(&hist.decoders) - if err != nil { + if err := seqs.initialize(br, hist, b.dst); err != nil { + println("initializing sequences:", err) return err } - if debugDecoder { - println("History merged ok") + // Extract blocks... + if false && hist.dict == nil { + fatalErr := func(err error) { + if err != nil { + panic(err) + } + } + fn := fmt.Sprintf("n-%d-lits-%d-prev-%d-%d-%d-win-%d.blk", hist.decoders.nSeqs, len(hist.decoders.literals), hist.recentOffsets[0], hist.recentOffsets[1], hist.recentOffsets[2], hist.windowSize) + var buf bytes.Buffer + fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.litLengths.fse)) + fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse)) + fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse)) + buf.Write(in) + os.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm) } - br := &bitReader{} - if err := br.init(in); err != nil { - return err + + return nil +} + +func (b *blockDec) decodeSequences(hist *history) error { + if cap(b.sequence) < hist.decoders.nSeqs { + if b.lowMem { + b.sequence = make([]seqVals, 0, hist.decoders.nSeqs) + } else { + b.sequence = make([]seqVals, 0, 0x7F00+0xffff) + } + } + b.sequence = b.sequence[:hist.decoders.nSeqs] + if hist.decoders.nSeqs == 0 { + hist.decoders.seqSize = len(hist.decoders.literals) + return nil } + hist.decoders.windowSize = hist.windowSize + hist.decoders.prevOffset = hist.recentOffsets - // TODO: Investigate if sending history without decoders are faster. - // This would allow the sequences to be decoded async and only have to construct stream history. - // If only recent offsets were not transferred, this would be an obvious win. - // Also, if first 3 sequences don't reference recent offsets, all sequences can be decoded. + err := hist.decoders.decode(b.sequence) + hist.recentOffsets = hist.decoders.prevOffset + return err +} +func (b *blockDec) executeSequences(hist *history) error { hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] - // We do not need history any more. + // We do not need history anymore. if hist.dict != nil { hist.dict.content = nil } } - - if err := seqs.initialize(br, hist, literals, b.dst); err != nil { - println("initializing sequences:", err) - return err - } - - err = seqs.decode(nSeqs, br, hbytes) + hist.decoders.windowSize = hist.windowSize + hist.decoders.out = b.dst[:0] + err := hist.decoders.execute(b.sequence, hbytes) if err != nil { return err } - if !br.finished() { - return fmt.Errorf("%d extra bits on block, should be 0", br.remain()) - } + return b.updateHistory(hist) +} - err = br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) - } +func (b *blockDec) updateHistory(hist *history) error { if len(b.data) > maxCompressedBlockSize { return fmt.Errorf("compressed block size too large (%d)", len(b.data)) } // Set output and release references. - b.dst = seqs.out - seqs.out, seqs.literals, seqs.hist = nil, nil, nil + b.dst = hist.decoders.out + hist.recentOffsets = hist.decoders.prevOffset - if !delayedHistory { - // If we don't have delayed history, no need to update. - hist.recentOffsets = seqs.prevOffset - return nil - } if b.Last { // if last block we don't care about history. println("Last block, no history returned") hist.b = hist.b[:0] return nil + } else { + hist.append(b.dst) + if debugDecoder { + println("Finished block with ", len(b.sequence), "sequences. Added", len(b.dst), "to history, now length", len(hist.b)) + } } - hist.append(b.dst) - hist.recentOffsets = seqs.prevOffset - if debugDecoder { - println("Finished block with literals:", len(literals), "and", nSeqs, "sequences.") - } + hist.decoders.out, hist.decoders.literals = nil, nil return nil } diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index 3df185ee4..12e8f6f0b 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -51,7 +51,7 @@ func (b *blockEnc) init() { if cap(b.literals) < maxCompressedBlockSize { b.literals = make([]byte, 0, maxCompressedBlockSize) } - const defSeqs = 200 + const defSeqs = 2000 if cap(b.sequences) < defSeqs { b.sequences = make([]seq, 0, defSeqs) } @@ -426,7 +426,7 @@ func fuzzFseEncoder(data []byte) int { return 0 } enc := fseEncoder{} - hist := enc.Histogram()[:256] + hist := enc.Histogram() maxSym := uint8(0) for i, v := range data { v = v & 63 @@ -722,52 +722,53 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { println("Encoded seq", seq, s, "codes:", s.llCode, s.mlCode, s.ofCode, "states:", ll.state, ml.state, of.state, "bits:", llB, mlB, ofB) } seq-- - if llEnc.maxBits+mlEnc.maxBits+ofEnc.maxBits <= 32 { - // No need to flush (common) - for seq >= 0 { - s = b.sequences[seq] - wr.flush32() - llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode] - // tabelog max is 8 for all. - of.encode(ofB) - ml.encode(mlB) - ll.encode(llB) - wr.flush32() - - // We checked that all can stay within 32 bits - wr.addBits32NC(s.litLen, llB.outBits) - wr.addBits32NC(s.matchLen, mlB.outBits) - wr.addBits32NC(s.offset, ofB.outBits) - - if debugSequences { - println("Encoded seq", seq, s) - } - - seq-- - } - } else { - for seq >= 0 { - s = b.sequences[seq] - wr.flush32() - llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode] - // tabelog max is below 8 for each. - of.encode(ofB) - ml.encode(mlB) - ll.encode(llB) - wr.flush32() - - // ml+ll = max 32 bits total - wr.addBits32NC(s.litLen, llB.outBits) - wr.addBits32NC(s.matchLen, mlB.outBits) - wr.flush32() - wr.addBits32NC(s.offset, ofB.outBits) - - if debugSequences { - println("Encoded seq", seq, s) - } - - seq-- - } + // Store sequences in reverse... + for seq >= 0 { + s = b.sequences[seq] + + ofB := ofTT[s.ofCode] + wr.flush32() // tablelog max is below 8 for each, so it will fill max 24 bits. + //of.encode(ofB) + nbBitsOut := (uint32(of.state) + ofB.deltaNbBits) >> 16 + dstState := int32(of.state>>(nbBitsOut&15)) + int32(ofB.deltaFindState) + wr.addBits16NC(of.state, uint8(nbBitsOut)) + of.state = of.stateTable[dstState] + + // Accumulate extra bits. + outBits := ofB.outBits & 31 + extraBits := uint64(s.offset & bitMask32[outBits]) + extraBitsN := outBits + + mlB := mlTT[s.mlCode] + //ml.encode(mlB) + nbBitsOut = (uint32(ml.state) + mlB.deltaNbBits) >> 16 + dstState = int32(ml.state>>(nbBitsOut&15)) + int32(mlB.deltaFindState) + wr.addBits16NC(ml.state, uint8(nbBitsOut)) + ml.state = ml.stateTable[dstState] + + outBits = mlB.outBits & 31 + extraBits = extraBits<> 16 + dstState = int32(ll.state>>(nbBitsOut&15)) + int32(llB.deltaFindState) + wr.addBits16NC(ll.state, uint8(nbBitsOut)) + ll.state = ll.stateTable[dstState] + + outBits = llB.outBits & 31 + extraBits = extraBits< math.MaxUint16 { panic("can only encode up to 64K sequences") } // No bounds checks after here: - llH := b.coders.llEnc.Histogram()[:256] - ofH := b.coders.ofEnc.Histogram()[:256] - mlH := b.coders.mlEnc.Histogram()[:256] + llH := b.coders.llEnc.Histogram() + ofH := b.coders.ofEnc.Histogram() + mlH := b.coders.mlEnc.Histogram() for i := range llH { llH[i] = 0 } @@ -820,7 +820,8 @@ func (b *blockEnc) genCodes() { } var llMax, ofMax, mlMax uint8 - for i, seq := range b.sequences { + for i := range b.sequences { + seq := &b.sequences[i] v := llCode(seq.litLen) seq.llCode = v llH[v]++ @@ -844,7 +845,6 @@ func (b *blockEnc) genCodes() { panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d), matchlen: %d", mlMax, seq.matchLen)) } } - b.sequences[i] = seq } maxCount := func(a []uint32) int { var max uint32 diff --git a/vendor/github.com/klauspost/compress/zstd/bytebuf.go b/vendor/github.com/klauspost/compress/zstd/bytebuf.go index aab71c6cf..176788f25 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytebuf.go +++ b/vendor/github.com/klauspost/compress/zstd/bytebuf.go @@ -7,7 +7,6 @@ package zstd import ( "fmt" "io" - "io/ioutil" ) type byteBuffer interface { @@ -23,7 +22,7 @@ type byteBuffer interface { readByte() (byte, error) // Skip n bytes. - skipN(n int) error + skipN(n int64) error } // in-memory buffer @@ -52,10 +51,6 @@ func (b *byteBuf) readBig(n int, dst []byte) ([]byte, error) { return r, nil } -func (b *byteBuf) remain() []byte { - return *b -} - func (b *byteBuf) readByte() (byte, error) { bb := *b if len(bb) < 1 { @@ -66,9 +61,12 @@ func (b *byteBuf) readByte() (byte, error) { return r, nil } -func (b *byteBuf) skipN(n int) error { +func (b *byteBuf) skipN(n int64) error { bb := *b - if len(bb) < n { + if n < 0 { + return fmt.Errorf("negative skip (%d) requested", n) + } + if int64(len(bb)) < n { return io.ErrUnexpectedEOF } *b = bb[n:] @@ -113,6 +111,9 @@ func (r *readerWrapper) readBig(n int, dst []byte) ([]byte, error) { func (r *readerWrapper) readByte() (byte, error) { n2, err := r.r.Read(r.tmp[:1]) if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } return 0, err } if n2 != 1 { @@ -121,9 +122,9 @@ func (r *readerWrapper) readByte() (byte, error) { return r.tmp[0], nil } -func (r *readerWrapper) skipN(n int) error { - n2, err := io.CopyN(ioutil.Discard, r.r, int64(n)) - if n2 != int64(n) { +func (r *readerWrapper) skipN(n int64) error { + n2, err := io.CopyN(io.Discard, r.r, n) + if n2 != n { err = io.ErrUnexpectedEOF } return err diff --git a/vendor/github.com/klauspost/compress/zstd/bytereader.go b/vendor/github.com/klauspost/compress/zstd/bytereader.go index 2c4fca17f..0e59a242d 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytereader.go +++ b/vendor/github.com/klauspost/compress/zstd/bytereader.go @@ -13,12 +13,6 @@ type byteReader struct { off int } -// init will initialize the reader and set the input. -func (b *byteReader) init(in []byte) { - b.b = in - b.off = 0 -} - // advance the stream b n bytes. func (b *byteReader) advance(n uint) { b.off += int(n) diff --git a/vendor/github.com/klauspost/compress/zstd/decodeheader.go b/vendor/github.com/klauspost/compress/zstd/decodeheader.go index 69736e8d4..f6a240970 100644 --- a/vendor/github.com/klauspost/compress/zstd/decodeheader.go +++ b/vendor/github.com/klauspost/compress/zstd/decodeheader.go @@ -4,7 +4,7 @@ package zstd import ( - "bytes" + "encoding/binary" "errors" "io" ) @@ -15,18 +15,50 @@ const HeaderMaxSize = 14 + 3 // Header contains information about the first frame and block within that. type Header struct { - // Window Size the window of data to keep while decoding. - // Will only be set if HasFCS is false. - WindowSize uint64 + // SingleSegment specifies whether the data is to be decompressed into a + // single contiguous memory segment. + // It implies that WindowSize is invalid and that FrameContentSize is valid. + SingleSegment bool - // Frame content size. - // Expected size of the entire frame. - FrameContentSize uint64 + // WindowSize is the window of data to keep while decoding. + // Will only be set if SingleSegment is false. + WindowSize uint64 // Dictionary ID. // If 0, no dictionary. DictionaryID uint32 + // HasFCS specifies whether FrameContentSize has a valid value. + HasFCS bool + + // FrameContentSize is the expected uncompressed size of the entire frame. + FrameContentSize uint64 + + // Skippable will be true if the frame is meant to be skipped. + // This implies that FirstBlock.OK is false. + Skippable bool + + // SkippableID is the user-specific ID for the skippable frame. + // Valid values are between 0 to 15, inclusive. + SkippableID int + + // SkippableSize is the length of the user data to skip following + // the header. + SkippableSize uint32 + + // HeaderSize is the raw size of the frame header. + // + // For normal frames, it includes the size of the magic number and + // the size of the header (per section 3.1.1.1). + // It does not include the size for any data blocks (section 3.1.1.2) nor + // the size for the trailing content checksum. + // + // For skippable frames, this counts the size of the magic number + // along with the size of the size field of the payload. + // It does not include the size of the skippable payload itself. + // The total frame size is the HeaderSize plus the SkippableSize. + HeaderSize int + // First block information. FirstBlock struct { // OK will be set if first block could be decoded. @@ -51,17 +83,9 @@ type Header struct { CompressedSize int } - // Skippable will be true if the frame is meant to be skipped. - // No other information will be populated. - Skippable bool - // If set there is a checksum present for the block content. + // The checksum field at the end is always 4 bytes long. HasCheckSum bool - - // If this is true FrameContentSize will have a valid value - HasFCS bool - - SingleSegment bool } // Decode the header from the beginning of the stream. @@ -71,39 +95,46 @@ type Header struct { // If there isn't enough input, io.ErrUnexpectedEOF is returned. // The FirstBlock.OK will indicate if enough information was available to decode the first block header. func (h *Header) Decode(in []byte) error { + *h = Header{} if len(in) < 4 { return io.ErrUnexpectedEOF } + h.HeaderSize += 4 b, in := in[:4], in[4:] - if !bytes.Equal(b, frameMagic) { - if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 { + if string(b) != frameMagic { + if string(b[1:4]) != skippableFrameMagic || b[0]&0xf0 != 0x50 { return ErrMagicMismatch } - *h = Header{Skippable: true} + if len(in) < 4 { + return io.ErrUnexpectedEOF + } + h.HeaderSize += 4 + h.Skippable = true + h.SkippableID = int(b[0] & 0xf) + h.SkippableSize = binary.LittleEndian.Uint32(in) return nil } + + // Read Window_Descriptor + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor if len(in) < 1 { return io.ErrUnexpectedEOF } - - // Clear output - *h = Header{} fhd, in := in[0], in[1:] + h.HeaderSize++ h.SingleSegment = fhd&(1<<5) != 0 h.HasCheckSum = fhd&(1<<2) != 0 - if fhd&(1<<3) != 0 { return errors.New("reserved bit set on frame header") } - // Read Window_Descriptor - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor if !h.SingleSegment { if len(in) < 1 { return io.ErrUnexpectedEOF } var wd byte wd, in = in[0], in[1:] + h.HeaderSize++ windowLog := 10 + (wd >> 3) windowBase := uint64(1) << windowLog windowAdd := (windowBase / 8) * uint64(wd&0x7) @@ -120,10 +151,8 @@ func (h *Header) Decode(in []byte) error { return io.ErrUnexpectedEOF } b, in = in[:size], in[size:] - if b == nil { - return io.ErrUnexpectedEOF - } - switch size { + h.HeaderSize += int(size) + switch len(b) { case 1: h.DictionaryID = uint32(b[0]) case 2: @@ -152,10 +181,8 @@ func (h *Header) Decode(in []byte) error { return io.ErrUnexpectedEOF } b, in = in[:fcsSize], in[fcsSize:] - if b == nil { - return io.ErrUnexpectedEOF - } - switch fcsSize { + h.HeaderSize += int(fcsSize) + switch len(b) { case 1: h.FrameContentSize = uint64(b[0]) case 2: diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index f430f58b5..7113e69ee 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -5,9 +5,12 @@ package zstd import ( - "errors" + "context" + "encoding/binary" "io" "sync" + + "github.com/klauspost/compress/zstd/internal/xxhash" ) // Decoder provides decoding of zstandard streams. @@ -22,15 +25,22 @@ type Decoder struct { // Unreferenced decoders, ready for use. decoders chan *blockDec - // Streams ready to be decoded. - stream chan decodeStream - // Current read position used for Reader functionality. current decoderState + // sync stream decoding + syncStream struct { + decodedFrame uint64 + br readerWrapper + enabled bool + inFrame bool + dstBuf []byte + } + + frame *frameDec + // Custom dictionaries. - // Always uses copies. - dicts map[uint32]dict + dicts map[uint32]*dict // streamWg is the waitgroup for all streams streamWg sync.WaitGroup @@ -46,7 +56,10 @@ type decoderState struct { output chan decodeOutput // cancel remaining output. - cancel chan struct{} + cancel context.CancelFunc + + // crc of current frame + crc *xxhash.Digest flushed bool } @@ -81,7 +94,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { return nil, err } } - d.current.output = make(chan decodeOutput, d.o.concurrent) + d.current.crc = xxhash.New() d.current.flushed = true if r == nil { @@ -89,7 +102,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { } // Transfer option dicts. - d.dicts = make(map[uint32]dict, len(d.o.dicts)) + d.dicts = make(map[uint32]*dict, len(d.o.dicts)) for _, dc := range d.o.dicts { d.dicts[dc.id] = dc } @@ -130,7 +143,7 @@ func (d *Decoder) Read(p []byte) (int, error) { break } if !d.nextBlock(n == 0) { - return n, nil + return n, d.current.err } } } @@ -162,6 +175,7 @@ func (d *Decoder) Reset(r io.Reader) error { d.drainOutput() + d.syncStream.br.r = nil if r == nil { d.current.err = ErrDecoderNilInput if len(d.current.b) > 0 { @@ -172,21 +186,23 @@ func (d *Decoder) Reset(r io.Reader) error { } // If bytes buffer and < 5MB, do sync decoding anyway. - if bb, ok := r.(byter); ok && bb.Len() < 5<<20 { + if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap { bb2 := bb if debugDecoder { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) } b := bb2.Bytes() var dst []byte - if cap(d.current.b) > 0 { - dst = d.current.b + if cap(d.syncStream.dstBuf) > 0 { + dst = d.syncStream.dstBuf[:0] } - dst, err := d.DecodeAll(b, dst[:0]) + dst, err := d.DecodeAll(b, dst) if err == nil { err = io.EOF } + // Save output buffer + d.syncStream.dstBuf = dst d.current.b = dst d.current.err = err d.current.flushed = true @@ -195,33 +211,40 @@ func (d *Decoder) Reset(r io.Reader) error { } return nil } - - if d.stream == nil { - d.stream = make(chan decodeStream, 1) - d.streamWg.Add(1) - go d.startStreamDecoder(d.stream) - } - // Remove current block. + d.stashDecoder() d.current.decodeOutput = decodeOutput{} d.current.err = nil - d.current.cancel = make(chan struct{}) d.current.flushed = false d.current.d = nil + d.syncStream.dstBuf = nil - d.stream <- decodeStream{ - r: r, - output: d.current.output, - cancel: d.current.cancel, + // Ensure no-one else is still running... + d.streamWg.Wait() + if d.frame == nil { + d.frame = newFrameDec(d.o) } + + if d.o.concurrent == 1 { + return d.startSyncDecoder(r) + } + + d.current.output = make(chan decodeOutput, d.o.concurrent) + ctx, cancel := context.WithCancel(context.Background()) + d.current.cancel = cancel + d.streamWg.Add(1) + go d.startStreamDecoder(ctx, r, d.current.output) + return nil } // drainOutput will drain the output until errEndOfStream is sent. func (d *Decoder) drainOutput() { if d.current.cancel != nil { - println("cancelling current") - close(d.current.cancel) + if debugDecoder { + println("cancelling current") + } + d.current.cancel() d.current.cancel = nil } if d.current.d != nil { @@ -243,12 +266,9 @@ func (d *Decoder) drainOutput() { } d.decoders <- v.d } - if v.err == errEndOfStream { - println("current flushed") - d.current.flushed = true - return - } } + d.current.output = nil + d.current.flushed = true } // WriteTo writes data to w until there's no more data to write or when an error occurs. @@ -287,19 +307,23 @@ func (d *Decoder) WriteTo(w io.Writer) (int64, error) { // DecodeAll can be used concurrently. // The Decoder concurrency limits will be respected. func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { - if d.current.err == ErrDecoderClosed { + if d.decoders == nil { return dst, ErrDecoderClosed } // Grab a block decoder and frame decoder. block := <-d.decoders frame := block.localFrame + initialSize := len(dst) defer func() { if debugDecoder { printf("re-adding decoder: %p", block) } frame.rawInput = nil frame.bBuf = nil + if frame.history.decoders.br != nil { + frame.history.decoders.br.in = nil + } d.decoders <- block }() frame.bBuf = input @@ -307,34 +331,45 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { for { frame.history.reset() err := frame.reset(&frame.bBuf) - if err == io.EOF { - if debugDecoder { - println("frame reset return EOF") - } - return dst, nil - } - if frame.DictionaryID != nil { - dict, ok := d.dicts[*frame.DictionaryID] - if !ok { - return nil, ErrUnknownDictionary - } - frame.history.setDict(&dict) - } if err != nil { + if err == io.EOF { + if debugDecoder { + println("frame reset return EOF") + } + return dst, nil + } return dst, err } - if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) { - return dst, ErrDecoderSizeExceeded + if err = d.setDict(frame); err != nil { + return nil, err + } + if frame.WindowSize > d.o.maxWindowSize { + if debugDecoder { + println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize) + } + return dst, ErrWindowSizeExceeded } - if frame.FrameContentSize > 0 && frame.FrameContentSize < 1<<30 { - // Never preallocate moe than 1 GB up front. + if frame.FrameContentSize != fcsUnknown { + if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) { + if debugDecoder { + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) + } + return dst, ErrDecoderSizeExceeded + } + if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { + if debugDecoder { + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst)) + } + return dst, ErrDecoderSizeExceeded + } if cap(dst)-len(dst) < int(frame.FrameContentSize) { - dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)) + dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)+compressedBlockOverAlloc) copy(dst2, dst) dst = dst2 } } - if cap(dst) == 0 { + + if cap(dst) == 0 && !d.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. size := len(input) * 2 @@ -352,6 +387,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if err != nil { return dst, err } + if uint64(len(dst)-initialSize) > d.o.maxDecodedSize { + return dst, ErrDecoderSizeExceeded + } if len(frame.bBuf) == 0 { if debugDecoder { println("frame dbuf empty") @@ -368,33 +406,172 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { // If non-blocking mode is used the returned boolean will be false // if no data was available without blocking. func (d *Decoder) nextBlock(blocking bool) (ok bool) { - if d.current.d != nil { - if debugDecoder { - printf("re-adding current decoder %p", d.current.d) - } - d.decoders <- d.current.d - d.current.d = nil - } if d.current.err != nil { // Keep error state. - return blocking + return false } + d.current.b = d.current.b[:0] + // SYNC: + if d.syncStream.enabled { + if !blocking { + return false + } + ok = d.nextBlockSync() + if !ok { + d.stashDecoder() + } + return ok + } + + //ASYNC: + d.stashDecoder() if blocking { - d.current.decodeOutput = <-d.current.output + d.current.decodeOutput, ok = <-d.current.output } else { select { - case d.current.decodeOutput = <-d.current.output: + case d.current.decodeOutput, ok = <-d.current.output: default: return false } } + if !ok { + // This should not happen, so signal error state... + d.current.err = io.ErrUnexpectedEOF + return false + } + next := d.current.decodeOutput + if next.d != nil && next.d.async.newHist != nil { + d.current.crc.Reset() + } if debugDecoder { - println("got", len(d.current.b), "bytes, error:", d.current.err) + var tmp [4]byte + binary.LittleEndian.PutUint32(tmp[:], uint32(xxhash.Sum64(next.b))) + println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) + } + + if d.o.ignoreChecksum { + return true + } + + if len(next.b) > 0 { + n, err := d.current.crc.Write(next.b) + if err == nil { + if n != len(next.b) { + d.current.err = io.ErrShortWrite + } + } + } + if next.err == nil && next.d != nil && next.d.hasCRC { + got := uint32(d.current.crc.Sum64()) + if got != next.d.checkCRC { + if debugDecoder { + printf("CRC Check Failed: %08x (got) != %08x (on stream)\n", got, next.d.checkCRC) + } + d.current.err = ErrCRCMismatch + } else { + if debugDecoder { + printf("CRC ok %08x\n", got) + } + } + } + + return true +} + +func (d *Decoder) nextBlockSync() (ok bool) { + if d.current.d == nil { + d.current.d = <-d.decoders + } + for len(d.current.b) == 0 { + if !d.syncStream.inFrame { + d.frame.history.reset() + d.current.err = d.frame.reset(&d.syncStream.br) + if d.current.err == nil { + d.current.err = d.setDict(d.frame) + } + if d.current.err != nil { + return false + } + if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { + d.current.err = ErrDecoderSizeExceeded + return false + } + + d.syncStream.decodedFrame = 0 + d.syncStream.inFrame = true + } + d.current.err = d.frame.next(d.current.d) + if d.current.err != nil { + return false + } + d.frame.history.ensureBlock() + if debugDecoder { + println("History trimmed:", len(d.frame.history.b), "decoded already:", d.syncStream.decodedFrame) + } + histBefore := len(d.frame.history.b) + d.current.err = d.current.d.decodeBuf(&d.frame.history) + + if d.current.err != nil { + println("error after:", d.current.err) + return false + } + d.current.b = d.frame.history.b[histBefore:] + if debugDecoder { + println("history after:", len(d.frame.history.b)) + } + + // Check frame size (before CRC) + d.syncStream.decodedFrame += uint64(len(d.current.b)) + if d.syncStream.decodedFrame > d.frame.FrameContentSize { + if debugDecoder { + printf("DecodedFrame (%d) > FrameContentSize (%d)\n", d.syncStream.decodedFrame, d.frame.FrameContentSize) + } + d.current.err = ErrFrameSizeExceeded + return false + } + + // Check FCS + if d.current.d.Last && d.frame.FrameContentSize != fcsUnknown && d.syncStream.decodedFrame != d.frame.FrameContentSize { + if debugDecoder { + printf("DecodedFrame (%d) != FrameContentSize (%d)\n", d.syncStream.decodedFrame, d.frame.FrameContentSize) + } + d.current.err = ErrFrameSizeMismatch + return false + } + + // Update/Check CRC + if d.frame.HasCheckSum { + if !d.o.ignoreChecksum { + d.frame.crc.Write(d.current.b) + } + if d.current.d.Last { + if !d.o.ignoreChecksum { + d.current.err = d.frame.checkCRC() + } else { + d.current.err = d.frame.consumeCRC() + } + if d.current.err != nil { + println("CRC error:", d.current.err) + return false + } + } + } + d.syncStream.inFrame = !d.current.d.Last } return true } +func (d *Decoder) stashDecoder() { + if d.current.d != nil { + if debugDecoder { + printf("re-adding current decoder %p", d.current.d) + } + d.decoders <- d.current.d + d.current.d = nil + } +} + // Close will release all resources. // It is NOT possible to reuse the decoder after this. func (d *Decoder) Close() { @@ -402,10 +579,10 @@ func (d *Decoder) Close() { return } d.drainOutput() - if d.stream != nil { - close(d.stream) + if d.current.cancel != nil { + d.current.cancel() d.streamWg.Wait() - d.stream = nil + d.current.cancel = nil } if d.decoders != nil { close(d.decoders) @@ -456,100 +633,321 @@ type decodeOutput struct { err error } -type decodeStream struct { - r io.Reader - - // Blocks ready to be written to output. - output chan decodeOutput - - // cancel reading from the input - cancel chan struct{} +func (d *Decoder) startSyncDecoder(r io.Reader) error { + d.frame.history.reset() + d.syncStream.br = readerWrapper{r: r} + d.syncStream.inFrame = false + d.syncStream.enabled = true + d.syncStream.decodedFrame = 0 + return nil } -// errEndOfStream indicates that everything from the stream was read. -var errEndOfStream = errors.New("end-of-stream") - // Create Decoder: -// Spawn n block decoders. These accept tasks to decode a block. -// Create goroutine that handles stream processing, this will send history to decoders as they are available. -// Decoders update the history as they decode. -// When a block is returned: -// a) history is sent to the next decoder, -// b) content written to CRC. -// c) return data to WRITER. -// d) wait for next block to return data. -// Once WRITTEN, the decoders reused by the writer frame decoder for re-use. -func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { +// ASYNC: +// Spawn 3 go routines. +// 0: Read frames and decode block literals. +// 1: Decode sequences. +// 2: Execute sequences, send to output. +func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output chan decodeOutput) { defer d.streamWg.Done() - frame := newFrameDec(d.o) - for stream := range inStream { - if debugDecoder { - println("got new stream") + br := readerWrapper{r: r} + + var seqDecode = make(chan *blockDec, d.o.concurrent) + var seqExecute = make(chan *blockDec, d.o.concurrent) + + // Async 1: Decode sequences... + go func() { + var hist history + var hasErr bool + + for block := range seqDecode { + if hasErr { + if block != nil { + seqExecute <- block + } + continue + } + if block.async.newHist != nil { + if debugDecoder { + println("Async 1: new history, recent:", block.async.newHist.recentOffsets) + } + hist.reset() + hist.decoders = block.async.newHist.decoders + hist.recentOffsets = block.async.newHist.recentOffsets + hist.windowSize = block.async.newHist.windowSize + if block.async.newHist.dict != nil { + hist.setDict(block.async.newHist.dict) + } + } + if block.err != nil || block.Type != blockTypeCompressed { + hasErr = block.err != nil + seqExecute <- block + continue + } + + hist.decoders.literals = block.async.literals + block.err = block.prepareSequences(block.async.seqData, &hist) + if debugDecoder && block.err != nil { + println("prepareSequences returned:", block.err) + } + hasErr = block.err != nil + if block.err == nil { + block.err = block.decodeSequences(&hist) + if debugDecoder && block.err != nil { + println("decodeSequences returned:", block.err) + } + hasErr = block.err != nil + // block.async.sequence = hist.decoders.seq[:hist.decoders.nSeqs] + block.async.seqSize = hist.decoders.seqSize + } + seqExecute <- block } - br := readerWrapper{r: stream.r} - decodeStream: - for { - frame.history.reset() - err := frame.reset(&br) - if debugDecoder && err != nil { - println("Frame decoder returned", err) + close(seqExecute) + hist.reset() + }() + + var wg sync.WaitGroup + wg.Add(1) + + // Async 3: Execute sequences... + frameHistCache := d.frame.history.b + go func() { + var hist history + var decodedFrame uint64 + var fcs uint64 + var hasErr bool + for block := range seqExecute { + out := decodeOutput{err: block.err, d: block} + if block.err != nil || hasErr { + hasErr = true + output <- out + continue + } + if block.async.newHist != nil { + if debugDecoder { + println("Async 2: new history") + } + hist.reset() + hist.windowSize = block.async.newHist.windowSize + hist.allocFrameBuffer = block.async.newHist.allocFrameBuffer + if block.async.newHist.dict != nil { + hist.setDict(block.async.newHist.dict) + } + + if cap(hist.b) < hist.allocFrameBuffer { + if cap(frameHistCache) >= hist.allocFrameBuffer { + hist.b = frameHistCache + } else { + hist.b = make([]byte, 0, hist.allocFrameBuffer) + println("Alloc history sized", hist.allocFrameBuffer) + } + } + hist.b = hist.b[:0] + fcs = block.async.fcs + decodedFrame = 0 } - if err == nil && frame.DictionaryID != nil { - dict, ok := d.dicts[*frame.DictionaryID] - if !ok { - err = ErrUnknownDictionary + do := decodeOutput{err: block.err, d: block} + switch block.Type { + case blockTypeRLE: + if debugDecoder { + println("add rle block length:", block.RLESize) + } + + if cap(block.dst) < int(block.RLESize) { + if block.lowMem { + block.dst = make([]byte, block.RLESize) + } else { + block.dst = make([]byte, maxCompressedBlockSize) + } + } + block.dst = block.dst[:block.RLESize] + v := block.data[0] + for i := range block.dst { + block.dst[i] = v + } + hist.append(block.dst) + do.b = block.dst + case blockTypeRaw: + if debugDecoder { + println("add raw block length:", len(block.data)) + } + hist.append(block.data) + do.b = block.data + case blockTypeCompressed: + if debugDecoder { + println("execute with history length:", len(hist.b), "window:", hist.windowSize) + } + hist.decoders.seqSize = block.async.seqSize + hist.decoders.literals = block.async.literals + do.err = block.executeSequences(&hist) + hasErr = do.err != nil + if debugDecoder && hasErr { + println("executeSequences returned:", do.err) + } + do.b = block.dst + } + if !hasErr { + decodedFrame += uint64(len(do.b)) + if decodedFrame > fcs { + println("fcs exceeded", block.Last, fcs, decodedFrame) + do.err = ErrFrameSizeExceeded + hasErr = true + } else if block.Last && fcs != fcsUnknown && decodedFrame != fcs { + do.err = ErrFrameSizeMismatch + hasErr = true } else { - frame.history.setDict(&dict) + if debugDecoder { + println("fcs ok", block.Last, fcs, decodedFrame) + } } } - if err != nil { - stream.output <- decodeOutput{ - err: err, + output <- do + } + close(output) + frameHistCache = hist.b + wg.Done() + if debugDecoder { + println("decoder goroutines finished") + } + hist.reset() + }() + + var hist history +decodeStream: + for { + var hasErr bool + hist.reset() + decodeBlock := func(block *blockDec) { + if hasErr { + if block != nil { + seqDecode <- block } - break + return + } + if block.err != nil || block.Type != blockTypeCompressed { + hasErr = block.err != nil + seqDecode <- block + return + } + + remain, err := block.decodeLiterals(block.data, &hist) + block.err = err + hasErr = block.err != nil + if err == nil { + block.async.literals = hist.decoders.literals + block.async.seqData = remain + } else if debugDecoder { + println("decodeLiterals error:", err) } + seqDecode <- block + } + frame := d.frame + if debugDecoder { + println("New frame...") + } + var historySent bool + frame.history.reset() + err := frame.reset(&br) + if debugDecoder && err != nil { + println("Frame decoder returned", err) + } + if err == nil { + err = d.setDict(frame) + } + if err == nil && d.frame.WindowSize > d.o.maxWindowSize { if debugDecoder { - println("starting frame decoder") - } - - // This goroutine will forward history between frames. - frame.frameDone.Add(1) - frame.initAsync() - - go frame.startDecoder(stream.output) - decodeFrame: - // Go through all blocks of the frame. - for { - dec := <-d.decoders - select { - case <-stream.cancel: - if !frame.sendErr(dec, io.EOF) { - // To not let the decoder dangle, send it back. - stream.output <- decodeOutput{d: dec} - } - break decodeStream - default: + println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize) + } + + err = ErrDecoderSizeExceeded + } + if err != nil { + select { + case <-ctx.Done(): + case dec := <-d.decoders: + dec.sendErr(err) + decodeBlock(dec) + } + break decodeStream + } + + // Go through all blocks of the frame. + for { + var dec *blockDec + select { + case <-ctx.Done(): + break decodeStream + case dec = <-d.decoders: + // Once we have a decoder, we MUST return it. + } + err := frame.next(dec) + if !historySent { + h := frame.history + if debugDecoder { + println("Alloc History:", h.allocFrameBuffer) } - err := frame.next(dec) - switch err { - case io.EOF: - // End of current frame, no error - println("EOF on next block") - break decodeFrame - case nil: - continue - default: - println("block decoder returned", err) - break decodeStream + hist.reset() + if h.dict != nil { + hist.setDict(h.dict) } + dec.async.newHist = &h + dec.async.fcs = frame.FrameContentSize + historySent = true + } else { + dec.async.newHist = nil + } + if debugDecoder && err != nil { + println("next block returned error:", err) + } + dec.err = err + dec.hasCRC = false + if dec.Last && frame.HasCheckSum && err == nil { + crc, err := frame.rawInput.readSmall(4) + if len(crc) < 4 { + if err == nil { + err = io.ErrUnexpectedEOF + + } + println("CRC missing?", err) + dec.err = err + } else { + dec.checkCRC = binary.LittleEndian.Uint32(crc) + dec.hasCRC = true + if debugDecoder { + printf("found crc to check: %08x\n", dec.checkCRC) + } + } + } + err = dec.err + last := dec.Last + decodeBlock(dec) + if err != nil { + break decodeStream + } + if last { + break } - // All blocks have started decoding, check if there are more frames. - println("waiting for done") - frame.frameDone.Wait() - println("done waiting...") } - frame.frameDone.Wait() - println("Sending EOS") - stream.output <- decodeOutput{err: errEndOfStream} } + close(seqDecode) + wg.Wait() + hist.reset() + d.frame.history.b = frameHistCache +} + +func (d *Decoder) setDict(frame *frameDec) (err error) { + dict, ok := d.dicts[frame.DictionaryID] + if ok { + if debugDecoder { + println("setting dict", frame.DictionaryID) + } + frame.history.setDict(dict) + } else if frame.DictionaryID != 0 { + // A zero or missing dictionary id is ambiguous: + // either dictionary zero, or no dictionary. In particular, + // zstd --patch-from uses this id for the source file, + // so only return an error if the dictionary id is not zero. + err = ErrUnknownDictionary + } + return err } diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index 95cc9b8b8..07a90dd7a 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -6,6 +6,8 @@ package zstd import ( "errors" + "fmt" + "math/bits" "runtime" ) @@ -14,21 +16,28 @@ type DOption func(*decoderOptions) error // options retains accumulated state of multiple options. type decoderOptions struct { - lowMem bool - concurrent int - maxDecodedSize uint64 - maxWindowSize uint64 - dicts []dict + lowMem bool + concurrent int + maxDecodedSize uint64 + maxWindowSize uint64 + dicts []*dict + ignoreChecksum bool + limitToCap bool + decodeBufsBelow int } func (o *decoderOptions) setDefault() { *o = decoderOptions{ // use less ram: true for now, but may change. - lowMem: true, - concurrent: runtime.GOMAXPROCS(0), - maxWindowSize: MaxWindowSize, + lowMem: true, + concurrent: runtime.GOMAXPROCS(0), + maxWindowSize: MaxWindowSize, + decodeBufsBelow: 128 << 10, } - o.maxDecodedSize = 1 << 63 + if o.concurrent > 4 { + o.concurrent = 4 + } + o.maxDecodedSize = 64 << 30 } // WithDecoderLowmem will set whether to use a lower amount of memory, @@ -37,16 +46,25 @@ func WithDecoderLowmem(b bool) DOption { return func(o *decoderOptions) error { o.lowMem = b; return nil } } -// WithDecoderConcurrency will set the concurrency, -// meaning the maximum number of decoders to run concurrently. -// The value supplied must be at least 1. -// By default this will be set to GOMAXPROCS. +// WithDecoderConcurrency sets the number of created decoders. +// When decoding block with DecodeAll, this will limit the number +// of possible concurrently running decodes. +// When decoding streams, this will limit the number of +// inflight blocks. +// When decoding streams and setting maximum to 1, +// no async decoding will be done. +// When a value of 0 is provided GOMAXPROCS will be used. +// By default this will be set to 4 or GOMAXPROCS, whatever is lower. func WithDecoderConcurrency(n int) DOption { return func(o *decoderOptions) error { - if n <= 0 { + if n < 0 { return errors.New("concurrency must be at least 1") } - o.concurrent = n + if n == 0 { + o.concurrent = runtime.GOMAXPROCS(0) + } else { + o.concurrent = n + } return nil } } @@ -54,7 +72,7 @@ func WithDecoderConcurrency(n int) DOption { // WithDecoderMaxMemory allows to set a maximum decoded size for in-memory // non-streaming operations or maximum window size for streaming operations. // This can be used to control memory usage of potentially hostile content. -// Maximum and default is 1 << 63 bytes. +// Maximum is 1 << 63 bytes. Default is 64GiB. func WithDecoderMaxMemory(n uint64) DOption { return func(o *decoderOptions) error { if n == 0 { @@ -69,7 +87,13 @@ func WithDecoderMaxMemory(n uint64) DOption { } // WithDecoderDicts allows to register one or more dictionaries for the decoder. -// If several dictionaries with the same ID is provided the last one will be used. +// +// Each slice in dict must be in the [dictionary format] produced by +// "zstd --train" from the Zstandard reference implementation. +// +// If several dictionaries with the same ID are provided, the last one will be used. +// +// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithDecoderDicts(dicts ...[]byte) DOption { return func(o *decoderOptions) error { for _, b := range dicts { @@ -77,8 +101,20 @@ func WithDecoderDicts(dicts ...[]byte) DOption { if err != nil { return err } - o.dicts = append(o.dicts, *d) + o.dicts = append(o.dicts, d) + } + return nil + } +} + +// WithEncoderDictRaw registers a dictionary that may be used by the decoder. +// The slice content can be arbitrary data. +func WithDecoderDictRaw(id uint32, content []byte) DOption { + return func(o *decoderOptions) error { + if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { + return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) } + o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}) return nil } } @@ -100,3 +136,34 @@ func WithDecoderMaxWindow(size uint64) DOption { return nil } } + +// WithDecodeAllCapLimit will limit DecodeAll to decoding cap(dst)-len(dst) bytes, +// or any size set in WithDecoderMaxMemory. +// This can be used to limit decoding to a specific maximum output size. +// Disabled by default. +func WithDecodeAllCapLimit(b bool) DOption { + return func(o *decoderOptions) error { + o.limitToCap = b + return nil + } +} + +// WithDecodeBuffersBelow will fully decode readers that have a +// `Bytes() []byte` and `Len() int` interface similar to bytes.Buffer. +// This typically uses less allocations but will have the full decompressed object in memory. +// Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less. +// Default is 128KiB. +func WithDecodeBuffersBelow(size int) DOption { + return func(o *decoderOptions) error { + o.decodeBufsBelow = size + return nil + } +} + +// IgnoreChecksum allows to forcibly ignore checksum checking. +func IgnoreChecksum(b bool) DOption { + return func(o *decoderOptions) error { + o.ignoreChecksum = b + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index a36ae83ef..ca0951452 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -1,7 +1,6 @@ package zstd import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -20,7 +19,10 @@ type dict struct { content []byte } -var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec} +const dictMagic = "\x37\xa4\x30\xec" + +// Maximum dictionary size for the reference implementation (1.5.3) is 2 GiB. +const dictMaxLength = 1 << 31 // ID returns the dictionary id or 0 if d is nil. func (d *dict) ID() uint32 { @@ -30,14 +32,38 @@ func (d *dict) ID() uint32 { return d.id } -// DictContentSize returns the dictionary content size or 0 if d is nil. -func (d *dict) DictContentSize() int { +// ContentSize returns the dictionary content size or 0 if d is nil. +func (d *dict) ContentSize() int { if d == nil { return 0 } return len(d.content) } +// Content returns the dictionary content. +func (d *dict) Content() []byte { + if d == nil { + return nil + } + return d.content +} + +// Offsets returns the initial offsets. +func (d *dict) Offsets() [3]int { + if d == nil { + return [3]int{} + } + return d.offsets +} + +// LitEncoder returns the literal encoder. +func (d *dict) LitEncoder() *huff0.Scratch { + if d == nil { + return nil + } + return d.litEnc +} + // Load a dictionary as described in // https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format func loadDict(b []byte) (*dict, error) { @@ -50,7 +76,7 @@ func loadDict(b []byte) (*dict, error) { ofDec: sequenceDec{fse: &fseDecoder{}}, mlDec: sequenceDec{fse: &fseDecoder{}}, } - if !bytes.Equal(b[:4], dictMagic[:]) { + if string(b[:4]) != dictMagic { return nil, ErrMagicMismatch } d.id = binary.LittleEndian.Uint32(b[4:8]) @@ -62,7 +88,7 @@ func loadDict(b []byte) (*dict, error) { var err error d.litEnc, b, err = huff0.ReadTable(b[8:], nil) if err != nil { - return nil, err + return nil, fmt.Errorf("loading literal table: %w", err) } d.litEnc.Reuse = huff0.ReusePolicyMust @@ -120,3 +146,16 @@ func loadDict(b []byte) (*dict, error) { return &d, nil } + +// InspectDictionary loads a zstd dictionary and provides functions to inspect the content. +func InspectDictionary(b []byte) (interface { + ID() uint32 + ContentSize() int + Content() []byte + Offsets() [3]int + LitEncoder() *huff0.Scratch +}, error) { + initPredefined() + d, err := loadDict(b) + return d, err +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index 295cd602a..e008b9929 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -16,6 +16,7 @@ type fastBase struct { cur int32 // maximum offset. Should be at least 2x block size. maxMatchOff int32 + bufferReset int32 hist []byte crc *xxhash.Digest tmp [8]byte @@ -56,8 +57,8 @@ func (e *fastBase) Block() *blockEnc { } func (e *fastBase) addBlock(src []byte) int32 { - if debugAsserts && e.cur > bufferReset { - panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, bufferReset)) + if debugAsserts && e.cur > e.bufferReset { + panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, e.bufferReset)) } // check if we have space already if len(e.hist)+len(src) > cap(e.hist) { @@ -108,11 +109,6 @@ func (e *fastBase) UseBlock(enc *blockEnc) { e.blk = enc } -func (e *fastBase) matchlenNoHist(s, t int32, src []byte) int32 { - // Extend the match to be as long as possible. - return int32(matchLen(src[s:], src[t:])) -} - func (e *fastBase) matchlen(s, t int32, src []byte) int32 { if debugAsserts { if s < 0 { @@ -131,8 +127,6 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 { panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize)) } } - - // Extend the match to be as long as possible. return int32(matchLen(src[s:], src[t:])) } @@ -155,13 +149,13 @@ func (e *fastBase) resetBase(d *dict, singleBlock bool) { if singleBlock { e.lowMem = true } - e.ensureHist(d.DictContentSize() + maxCompressedBlockSize) + e.ensureHist(d.ContentSize() + maxCompressedBlockSize) e.lowMem = low } // We offset current position so everything will be out of reach. // If above reset line, history will be purged. - if e.cur < bufferReset { + if e.cur < e.bufferReset { e.cur += e.maxMatchOff + int32(len(e.hist)) } e.hist = e.hist[:0] diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index 96028ecd8..830f5ba74 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -32,6 +32,7 @@ type match struct { length int32 rep int32 est int32 + _ [12]byte // Aligned size to cache line: 4+4+4+4+4 bytes + 12 bytes padding = 32 bytes } const highScore = 25000 @@ -84,14 +85,10 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = prevEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = prevEntry{} - } + e.table = [bestShortTableSize]prevEntry{} + e.longTable = [bestLongTableSize]prevEntry{} e.cur = e.maxMatchOff break } @@ -192,8 +189,8 @@ encodeLoop: panic("offset0 was 0") } - bestOf := func(a, b match) match { - if a.est+(a.s-b.s)*bitsPerByte>>10 < b.est+(b.s-a.s)*bitsPerByte>>10 { + bestOf := func(a, b *match) *match { + if a.est-b.est+(a.s-b.s)*bitsPerByte>>10 < 0 { return a } return b @@ -219,22 +216,26 @@ encodeLoop: return m } - best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1)) + m1 := matchAt(candidateL.offset-e.cur, s, uint32(cv), -1) + m2 := matchAt(candidateL.prev-e.cur, s, uint32(cv), -1) + m3 := matchAt(candidateS.offset-e.cur, s, uint32(cv), -1) + m4 := matchAt(candidateS.prev-e.cur, s, uint32(cv), -1) + best := bestOf(bestOf(&m1, &m2), bestOf(&m3, &m4)) if canRepeat && best.length < goodEnough { cv32 := uint32(cv >> 8) spp := s + 1 - best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1)) - best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2)) - best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3)) + m1 := matchAt(spp-offset1, spp, cv32, 1) + m2 := matchAt(spp-offset2, spp, cv32, 2) + m3 := matchAt(spp-offset3, spp, cv32, 3) + best = bestOf(bestOf(best, &m1), bestOf(&m2, &m3)) if best.length > 0 { cv32 = uint32(cv >> 24) spp += 2 - best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1)) - best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2)) - best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3)) + m1 := matchAt(spp-offset1, spp, cv32, 1) + m2 := matchAt(spp-offset2, spp, cv32, 2) + m3 := matchAt(spp-offset3, spp, cv32, 3) + best = bestOf(bestOf(best, &m1), bestOf(&m2, &m3)) } } // Load next and check... @@ -261,26 +262,33 @@ encodeLoop: candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)] // Short at s+1 - best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) + m1 := matchAt(candidateS.offset-e.cur, s, uint32(cv), -1) // Long at s+1, s+2 - best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)) - best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)) + m2 := matchAt(candidateL.offset-e.cur, s, uint32(cv), -1) + m3 := matchAt(candidateL.prev-e.cur, s, uint32(cv), -1) + m4 := matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1) + m5 := matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1) + best = bestOf(bestOf(bestOf(best, &m1), &m2), bestOf(bestOf(&m3, &m4), &m5)) if false { // Short at s+3. // Too often worse... - best = bestOf(best, matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1)) + m := matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1) + best = bestOf(best, &m) } // See if we can find a better match by checking where the current best ends. // Use that offset to see if we can find a better full match. if sAt := best.s + best.length; sAt < sLimit { nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen) candidateEnd := e.longTable[nextHashL] - if pos := candidateEnd.offset - e.cur - best.length; pos >= 0 { - bestEnd := bestOf(best, matchAt(pos, best.s, load3232(src, best.s), -1)) - if pos := candidateEnd.prev - e.cur - best.length; pos >= 0 { - bestEnd = bestOf(bestEnd, matchAt(pos, best.s, load3232(src, best.s), -1)) + // Start check at a fixed offset to allow for a few mismatches. + // For this compression level 2 yields the best results. + const skipBeginning = 2 + if pos := candidateEnd.offset - e.cur - best.length + skipBeginning; pos >= 0 { + m := matchAt(pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1) + bestEnd := bestOf(best, &m) + if pos := candidateEnd.prev - e.cur - best.length + skipBeginning; pos >= 0 { + m := matchAt(pos, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1) + bestEnd = bestOf(bestEnd, &m) } best = bestEnd } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 602c05ee0..8582f31a7 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -62,14 +62,10 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = prevEntry{} - } + e.table = [betterShortTableSize]tableEntry{} + e.longTable = [betterLongTableSize]prevEntry{} e.cur = e.maxMatchOff break } @@ -156,8 +152,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -416,15 +412,23 @@ encodeLoop: // Try to find a better match by searching for a long match at the end of the current best match if s+matched < sLimit { + // Allow some bytes at the beginning to mismatch. + // Sweet spot is around 3 bytes, but depends on input. + // The skipped bytes are tested in Extend backwards, + // and still picked up as part of the match if they do. + const skipBeginning = 3 + nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) - cv := load3232(src, s) + s2 := s + skipBeginning + cv := load3232(src, s2) candidateL := e.longTable[nextHashL] - coffsetL := candidateL.offset - e.cur - matched - if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + coffsetL := candidateL.offset - e.cur - matched + skipBeginning + if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4 if matchedNext > matched { t = coffsetL + s = s2 matched = matchedNext if debugMatches { println("long match at end-of-match") @@ -434,12 +438,13 @@ encodeLoop: // Check prev long... if true { - coffsetL = candidateL.prev - e.cur - matched - if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + coffsetL = candidateL.prev - e.cur - matched + skipBeginning + if coffsetL >= 0 && coffsetL < s2 && s2-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { // Found a long match, at least 4 bytes. - matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + matchedNext := e.matchlen(s2+4, coffsetL+4, src) + 4 if matchedNext > matched { t = coffsetL + s = s2 matched = matchedNext if debugMatches { println("prev long match at end-of-match") @@ -518,8 +523,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -578,7 +583,7 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { for i := range e.table[:] { e.table[i] = tableEntry{} @@ -674,8 +679,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -1047,8 +1052,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index d6b310424..7d425109a 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -44,14 +44,10 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = tableEntry{} - } + e.table = [dFastShortTableSize]tableEntry{} + e.longTable = [dFastLongTableSize]tableEntry{} e.cur = e.maxMatchOff break } @@ -127,8 +123,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -388,7 +384,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - if e.cur >= bufferReset { + if e.cur >= e.bufferReset { for i := range e.table[:] { e.table[i] = tableEntry{} } @@ -439,8 +435,8 @@ encodeLoop: var t int32 for { - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -685,7 +681,7 @@ encodeLoop: } // We do not store history, so we must offset e.cur to avoid false matches for next user. - if e.cur < bufferReset { + if e.cur < e.bufferReset { e.cur += int32(len(src)) } } @@ -700,7 +696,7 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { for i := range e.table[:] { e.table[i] = tableEntry{} @@ -785,8 +781,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -969,7 +965,7 @@ encodeLoop: te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} longHash1 := hashLen(cv0, dFastLongTableBits, dFastLongLen) - longHash2 := hashLen(cv0, dFastLongTableBits, dFastLongLen) + longHash2 := hashLen(cv1, dFastLongTableBits, dFastLongLen) e.longTable[longHash1] = te0 e.longTable[longHash2] = te1 e.markLongShardDirty(longHash1) @@ -1002,8 +998,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -1103,7 +1099,8 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } if allDirty || dirtyShardCnt > dLongTableShardCnt/2 { - copy(e.longTable[:], e.dictLongTable) + //copy(e.longTable[:], e.dictLongTable) + e.longTable = *(*[dFastLongTableSize]tableEntry)(e.dictLongTable) for i := range e.longTableShardDirty { e.longTableShardDirty[i] = false } @@ -1114,7 +1111,9 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { continue } - copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize]) + // copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize]) + *(*[dLongTableShardSize]tableEntry)(e.longTable[i*dLongTableShardSize:]) = *(*[dLongTableShardSize]tableEntry)(e.dictLongTable[i*dLongTableShardSize:]) + e.longTableShardDirty[i] = false } } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index f2502629b..315b1a8f2 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -6,8 +6,6 @@ package zstd import ( "fmt" - "math" - "math/bits" ) const ( @@ -45,7 +43,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { for i := range e.table[:] { e.table[i] = tableEntry{} @@ -87,7 +85,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { // TEMPLATE const hashLog = tableBits // seems global, but would be nice to tweak. - const kSearchStrength = 7 + const kSearchStrength = 6 // nextEmit is where in src the next emitLiteral should start from. nextEmit := s @@ -136,20 +134,7 @@ encodeLoop: // Consider history as well. var seq seq var length int32 - // length = 4 + e.matchlen(s+6, repIndex+4, src) - { - a := src[s+6:] - b := src[repIndex+4:] - endI := len(a) & (math.MaxInt32 - 7) - length = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - length = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } - + length = 4 + e.matchlen(s+6, repIndex+4, src) seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. @@ -236,20 +221,7 @@ encodeLoop: } // Extend the 4-byte match as long as possible. - //l := e.matchlen(s+4, t+4, src) + 4 - var l int32 - { - a := src[s+4:] - b := src[t+4:] - endI := len(a) & (math.MaxInt32 - 7) - l = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards tMin := s - e.maxMatchOff @@ -286,20 +258,7 @@ encodeLoop: if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { // We have at least 4 byte match. // No need to check backwards. We come straight from a match - //l := 4 + e.matchlen(s+4, o2+4, src) - var l int32 - { - a := src[s+4:] - b := src[o2+4:] - endI := len(a) & (math.MaxInt32 - 7) - l = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + l := 4 + e.matchlen(s+4, o2+4, src) // Store this, since we have it. nextHash := hashLen(cv, hashLog, tableFastHashLen) @@ -345,13 +304,13 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { minNonLiteralBlockSize = 1 + 1 + inputMargin ) if debugEncoder { - if len(src) > maxBlockSize { + if len(src) > maxCompressedBlockSize { panic("src too big") } } // Protect against e.cur wraparound. - if e.cur >= bufferReset { + if e.cur >= e.bufferReset { for i := range e.table[:] { e.table[i] = tableEntry{} } @@ -375,7 +334,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { // TEMPLATE const hashLog = tableBits // seems global, but would be nice to tweak. - const kSearchStrength = 8 + const kSearchStrength = 6 // nextEmit is where in src the next emitLiteral should start from. nextEmit := s @@ -418,21 +377,7 @@ encodeLoop: if len(blk.sequences) > 2 && load3232(src, repIndex) == uint32(cv>>16) { // Consider history as well. var seq seq - // length := 4 + e.matchlen(s+6, repIndex+4, src) - // length := 4 + int32(matchLen(src[s+6:], src[repIndex+4:])) - var length int32 - { - a := src[s+6:] - b := src[repIndex+4:] - endI := len(a) & (math.MaxInt32 - 7) - length = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - length = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + length := 4 + e.matchlen(s+6, repIndex+4, src) seq.matchLen = uint32(length - zstdMinMatch) @@ -522,21 +467,7 @@ encodeLoop: panic(fmt.Sprintf("t (%d) < 0 ", t)) } // Extend the 4-byte match as long as possible. - //l := e.matchlenNoHist(s+4, t+4, src) + 4 - // l := int32(matchLen(src[s+4:], src[t+4:])) + 4 - var l int32 - { - a := src[s+4:] - b := src[t+4:] - endI := len(a) & (math.MaxInt32 - 7) - l = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards tMin := s - e.maxMatchOff @@ -573,21 +504,7 @@ encodeLoop: if o2 := s - offset2; len(blk.sequences) > 2 && load3232(src, o2) == uint32(cv) { // We have at least 4 byte match. // No need to check backwards. We come straight from a match - //l := 4 + e.matchlenNoHist(s+4, o2+4, src) - // l := 4 + int32(matchLen(src[s+4:], src[o2+4:])) - var l int32 - { - a := src[s+4:] - b := src[o2+4:] - endI := len(a) & (math.MaxInt32 - 7) - l = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + l := 4 + e.matchlen(s+4, o2+4, src) // Store this, since we have it. nextHash := hashLen(cv, hashLog, tableFastHashLen) @@ -621,7 +538,7 @@ encodeLoop: println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } // We do not store history, so we must offset e.cur to avoid false matches for next user. - if e.cur < bufferReset { + if e.cur < e.bufferReset { e.cur += int32(len(src)) } } @@ -638,11 +555,9 @@ func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) { return } // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } + e.table = [tableSize]tableEntry{} e.cur = e.maxMatchOff break } @@ -731,19 +646,7 @@ encodeLoop: // Consider history as well. var seq seq var length int32 - // length = 4 + e.matchlen(s+6, repIndex+4, src) - { - a := src[s+6:] - b := src[repIndex+4:] - endI := len(a) & (math.MaxInt32 - 7) - length = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - length = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + length = 4 + e.matchlen(s+6, repIndex+4, src) seq.matchLen = uint32(length - zstdMinMatch) @@ -831,20 +734,7 @@ encodeLoop: } // Extend the 4-byte match as long as possible. - //l := e.matchlen(s+4, t+4, src) + 4 - var l int32 - { - a := src[s+4:] - b := src[t+4:] - endI := len(a) & (math.MaxInt32 - 7) - l = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards tMin := s - e.maxMatchOff @@ -881,20 +771,7 @@ encodeLoop: if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { // We have at least 4 byte match. // No need to check backwards. We come straight from a match - //l := 4 + e.matchlen(s+4, o2+4, src) - var l int32 - { - a := src[s+4:] - b := src[o2+4:] - endI := len(a) & (math.MaxInt32 - 7) - l = int32(endI) + 4 - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 - break - } - } - } + l := 4 + e.matchlen(s+4, o2+4, src) // Store this, since we have it. nextHash := hashLen(cv, hashLog, tableFastHashLen) @@ -992,7 +869,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { const shardCnt = tableShardCnt const shardSize = tableShardSize if e.allDirty || dirtyShardCnt > shardCnt*4/6 { - copy(e.table[:], e.dictTable) + //copy(e.table[:], e.dictTable) + e.table = *(*[tableSize]tableEntry)(e.dictTable) for i := range e.tableShardDirty { e.tableShardDirty[i] = false } @@ -1004,7 +882,8 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { continue } - copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) + //copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) + *(*[shardSize]tableEntry)(e.table[i*shardSize:]) = *(*[shardSize]tableEntry)(e.dictTable[i*shardSize:]) e.tableShardDirty[i] = false } e.allDirty = false diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index e6e315969..65c6c36dc 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "fmt" "io" + "math" rdebug "runtime/debug" "sync" @@ -98,23 +99,25 @@ func (e *Encoder) Reset(w io.Writer) { if cap(s.filling) == 0 { s.filling = make([]byte, 0, e.o.blockSize) } - if cap(s.current) == 0 { - s.current = make([]byte, 0, e.o.blockSize) - } - if cap(s.previous) == 0 { - s.previous = make([]byte, 0, e.o.blockSize) + if e.o.concurrent > 1 { + if cap(s.current) == 0 { + s.current = make([]byte, 0, e.o.blockSize) + } + if cap(s.previous) == 0 { + s.previous = make([]byte, 0, e.o.blockSize) + } + s.current = s.current[:0] + s.previous = s.previous[:0] + if s.writing == nil { + s.writing = &blockEnc{lowMem: e.o.lowMem} + s.writing.init() + } + s.writing.initNewEncode() } if s.encoder == nil { s.encoder = e.o.encoder() } - if s.writing == nil { - s.writing = &blockEnc{lowMem: e.o.lowMem} - s.writing.init() - } - s.writing.initNewEncode() s.filling = s.filling[:0] - s.current = s.current[:0] - s.previous = s.previous[:0] s.encoder.Reset(e.o.dict, false) s.headerWritten = false s.eofWritten = false @@ -258,6 +261,46 @@ func (e *Encoder) nextBlock(final bool) error { return s.err } + // SYNC: + if e.o.concurrent == 1 { + src := s.filling + s.nInput += int64(len(s.filling)) + if debugEncoder { + println("Adding sync block,", len(src), "bytes, final:", final) + } + enc := s.encoder + blk := enc.Block() + blk.reset(nil) + enc.Encode(blk, src) + blk.last = final + if final { + s.eofWritten = true + } + + err := errIncompressible + // If we got the exact same number of literals as input, + // assume the literals cannot be compressed. + if len(src) != len(blk.literals) || len(src) != e.o.blockSize { + err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) + } + switch err { + case errIncompressible: + if debugEncoder { + println("Storing incompressible block as raw") + } + blk.encodeRaw(src) + // In fast mode, we do not transfer offsets, so we don't have to deal with changing the. + case nil: + default: + s.err = err + return err + } + _, s.err = s.w.Write(blk.output) + s.nWritten += int64(len(blk.output)) + s.filling = s.filling[:0] + return s.err + } + // Move blocks forward. s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current s.nInput += int64(len(s.current)) @@ -486,8 +529,8 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { // If a non-single block is needed the encoder will reset again. e.encoders <- enc }() - // Use single segments when above minimum window and below 1MB. - single := len(src) < 1<<20 && len(src) > MinWindowSize + // Use single segments when above minimum window and below window size. + single := len(src) <= e.o.windowSize && len(src) > MinWindowSize if e.o.single != nil { single = *e.o.single } @@ -509,7 +552,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } // If we can do everything in one block, prefer that. - if len(src) <= maxCompressedBlockSize { + if len(src) <= e.o.blockSize { enc.Reset(e.o.dict, true) // Slightly faster with no history and everything in one block. if e.o.crc { @@ -597,3 +640,37 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } return dst } + +// MaxEncodedSize returns the expected maximum +// size of an encoded block or stream. +func (e *Encoder) MaxEncodedSize(size int) int { + frameHeader := 4 + 2 // magic + frame header & window descriptor + if e.o.dict != nil { + frameHeader += 4 + } + // Frame content size: + if size < 256 { + frameHeader++ + } else if size < 65536+256 { + frameHeader += 2 + } else if size < math.MaxInt32 { + frameHeader += 4 + } else { + frameHeader += 8 + } + // Final crc + if e.o.crc { + frameHeader += 4 + } + + // Max overhead is 3 bytes/block. + // There cannot be 0 blocks. + blocks := (size + e.o.blockSize) / e.o.blockSize + + // Combine, add padding. + maxSz := frameHeader + 3*blocks + size + if e.o.pad > 1 { + maxSz += calcSkippableFrame(int64(maxSz), int64(e.o.pad)) + } + return maxSz +} diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 7d29e1d68..8e15be2f7 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -3,6 +3,8 @@ package zstd import ( "errors" "fmt" + "math" + "math/bits" "runtime" "strings" ) @@ -24,6 +26,7 @@ type encoderOptions struct { allLitEntropy bool customWindow bool customALEntropy bool + customBlockSize bool lowMem bool dict *dict } @@ -33,7 +36,7 @@ func (o *encoderOptions) setDefault() { concurrent: runtime.GOMAXPROCS(0), crc: true, single: nil, - blockSize: 1 << 16, + blockSize: maxCompressedBlockSize, windowSize: 8 << 20, level: SpeedDefault, allLitEntropy: true, @@ -46,22 +49,22 @@ func (o encoderOptions) encoder() encoder { switch o.level { case SpeedFastest: if o.dict != nil { - return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} } - return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} + return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} case SpeedDefault: if o.dict != nil { - return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}} + return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}} } - return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} case SpeedBetterCompression: if o.dict != nil { - return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} } - return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} + return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} case SpeedBestCompression: - return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} + return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} } panic("unknown compression level") } @@ -75,6 +78,7 @@ func WithEncoderCRC(b bool) EOption { // WithEncoderConcurrency will set the concurrency, // meaning the maximum number of encoders to run concurrently. // The value supplied must be at least 1. +// For streams, setting a value of 1 will disable async compression. // By default this will be set to GOMAXPROCS. func WithEncoderConcurrency(n int) EOption { return func(o *encoderOptions) error { @@ -106,6 +110,7 @@ func WithWindowSize(n int) EOption { o.customWindow = true if o.blockSize > o.windowSize { o.blockSize = o.windowSize + o.customBlockSize = true } return nil } @@ -188,10 +193,9 @@ func EncoderLevelFromZstd(level int) EncoderLevel { return SpeedDefault case level >= 6 && level < 10: return SpeedBetterCompression - case level >= 10: + default: return SpeedBestCompression } - return SpeedDefault } // String provides a string representation of the compression level. @@ -222,6 +226,9 @@ func WithEncoderLevel(l EncoderLevel) EOption { switch o.level { case SpeedFastest: o.windowSize = 4 << 20 + if !o.customBlockSize { + o.blockSize = 1 << 16 + } case SpeedDefault: o.windowSize = 8 << 20 case SpeedBetterCompression: @@ -278,7 +285,7 @@ func WithNoEntropyCompression(b bool) EOption { // a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. // For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB. // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. -// If this is not specified, block encodes will automatically choose this based on the input size. +// If this is not specified, block encodes will automatically choose this based on the input size and the window size. // This setting has no effect on streamed encodes. func WithSingleSegment(b bool) EOption { return func(o *encoderOptions) error { @@ -299,7 +306,13 @@ func WithLowerEncoderMem(b bool) EOption { } // WithEncoderDict allows to register a dictionary that will be used for the encode. +// +// The slice dict must be in the [dictionary format] produced by +// "zstd --train" from the Zstandard reference implementation. +// // The encoder *may* choose to use no dictionary instead for certain payloads. +// +// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithEncoderDict(dict []byte) EOption { return func(o *encoderOptions) error { d, err := loadDict(dict) @@ -310,3 +323,17 @@ func WithEncoderDict(dict []byte) EOption { return nil } } + +// WithEncoderDictRaw registers a dictionary that may be used by the encoder. +// +// The slice content may contain arbitrary data. It will be used as an initial +// history. +func WithEncoderDictRaw(id uint32, content []byte) EOption { + return func(o *encoderOptions) error { + if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { + return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) + } + o.dict = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}} + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 989c79f8c..d8e8a05bd 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -5,26 +5,20 @@ package zstd import ( - "bytes" + "encoding/binary" "encoding/hex" "errors" - "hash" "io" - "sync" "github.com/klauspost/compress/zstd/internal/xxhash" ) type frameDec struct { - o decoderOptions - crc hash.Hash64 - offset int64 + o decoderOptions + crc *xxhash.Digest WindowSize uint64 - // In order queue of blocks being decoded. - decoding chan *blockDec - // Frame history passed between blocks history history @@ -34,15 +28,10 @@ type frameDec struct { bBuf byteBuf FrameContentSize uint64 - frameDone sync.WaitGroup - DictionaryID *uint32 + DictionaryID uint32 HasCheckSum bool SingleSegment bool - - // asyncRunning indicates whether the async routine processes input on 'decoding'. - asyncRunningMu sync.Mutex - asyncRunning bool } const ( @@ -54,9 +43,9 @@ const ( MaxWindowSize = 1 << 29 ) -var ( - frameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} - skippableFrameMagic = []byte{0x2a, 0x4d, 0x18} +const ( + frameMagic = "\x28\xb5\x2f\xfd" + skippableFrameMagic = "\x2a\x4d\x18" ) func newFrameDec(o decoderOptions) *frameDec { @@ -100,9 +89,9 @@ func (d *frameDec) reset(br byteBuffer) error { copy(signature[1:], b) } - if !bytes.Equal(signature[1:4], skippableFrameMagic) || signature[0]&0xf0 != 0x50 { + if string(signature[1:4]) != skippableFrameMagic || signature[0]&0xf0 != 0x50 { if debugDecoder { - println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString(skippableFrameMagic)) + println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString([]byte(skippableFrameMagic))) } // Break if not skippable frame. break @@ -117,7 +106,7 @@ func (d *frameDec) reset(br byteBuffer) error { } n := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) println("Skipping frame with", n, "bytes.") - err = br.skipN(int(n)) + err = br.skipN(int64(n)) if err != nil { if debugDecoder { println("Reading discarded frame", err) @@ -125,9 +114,9 @@ func (d *frameDec) reset(br byteBuffer) error { return err } } - if !bytes.Equal(signature[:], frameMagic) { + if string(signature[:]) != frameMagic { if debugDecoder { - println("Got magic numbers: ", signature, "want:", frameMagic) + println("Got magic numbers: ", signature, "want:", []byte(frameMagic)) } return ErrMagicMismatch } @@ -166,7 +155,7 @@ func (d *frameDec) reset(br byteBuffer) error { // Read Dictionary_ID // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id - d.DictionaryID = nil + d.DictionaryID = 0 if size := fhd & 3; size != 0 { if size == 3 { size = 4 @@ -178,7 +167,7 @@ func (d *frameDec) reset(br byteBuffer) error { return err } var id uint32 - switch size { + switch len(b) { case 1: id = uint32(b[0]) case 2: @@ -189,11 +178,7 @@ func (d *frameDec) reset(br byteBuffer) error { if debugDecoder { println("Dict size", size, "ID:", id) } - if id > 0 { - // ID 0 means "sorry, no dictionary anyway". - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format - d.DictionaryID = &id - } + d.DictionaryID = id } // Read Frame_Content_Size @@ -208,14 +193,14 @@ func (d *frameDec) reset(br byteBuffer) error { default: fcsSize = 1 << v } - d.FrameContentSize = 0 + d.FrameContentSize = fcsUnknown if fcsSize > 0 { b, err := br.readSmall(fcsSize) if err != nil { println("Reading Frame content", err) return err } - switch fcsSize { + switch len(b) { case 1: d.FrameContentSize = uint64(b[0]) case 2: @@ -229,9 +214,10 @@ func (d *frameDec) reset(br byteBuffer) error { d.FrameContentSize = uint64(d1) | (uint64(d2) << 32) } if debugDecoder { - println("field size bits:", v, "fcsSize:", fcsSize, "FrameContentSize:", d.FrameContentSize, hex.EncodeToString(b[:fcsSize]), "singleseg:", d.SingleSegment, "window:", d.WindowSize) + println("Read FCS:", d.FrameContentSize) } } + // Move this to shared. d.HasCheckSum = fhd&(1<<2) != 0 if d.HasCheckSum { @@ -241,20 +227,27 @@ func (d *frameDec) reset(br byteBuffer) error { d.crc.Reset() } + if d.WindowSize > d.o.maxWindowSize { + if debugDecoder { + printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) + } + return ErrWindowSizeExceeded + } + if d.WindowSize == 0 && d.SingleSegment { // We may not need window in this case. d.WindowSize = d.FrameContentSize if d.WindowSize < MinWindowSize { d.WindowSize = MinWindowSize } - } - - if d.WindowSize > uint64(d.o.maxWindowSize) { - if debugDecoder { - printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) + if d.WindowSize > d.o.maxDecodedSize { + if debugDecoder { + printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) + } + return ErrDecoderSizeExceeded } - return ErrWindowSizeExceeded } + // The minimum Window_Size is 1 KB. if d.WindowSize < MinWindowSize { if debugDecoder { @@ -263,11 +256,23 @@ func (d *frameDec) reset(br byteBuffer) error { return ErrWindowSizeTooSmall } d.history.windowSize = int(d.WindowSize) - if d.o.lowMem && d.history.windowSize < maxBlockSize { - d.history.maxSize = d.history.windowSize * 2 + if !d.o.lowMem || d.history.windowSize < maxBlockSize { + // Alloc 2x window size if not low-mem, or window size below 2MB. + d.history.allocFrameBuffer = d.history.windowSize * 2 } else { - d.history.maxSize = d.history.windowSize + maxBlockSize + if d.o.lowMem { + // Alloc with 1MB extra. + d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize/2 + } else { + // Alloc with 2MB extra. + d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize + } } + + if debugDecoder { + println("Frame: Dict:", d.DictionaryID, "FrameContentSize:", d.FrameContentSize, "singleseg:", d.SingleSegment, "window:", d.WindowSize, "crc:", d.HasCheckSum) + } + // history contains input - maybe we do something d.rawInput = br return nil @@ -276,209 +281,97 @@ func (d *frameDec) reset(br byteBuffer) error { // next will start decoding the next block from stream. func (d *frameDec) next(block *blockDec) error { if debugDecoder { - printf("decoding new block %p:%p", block, block.data) + println("decoding new block") } err := block.reset(d.rawInput, d.WindowSize) if err != nil { println("block error:", err) // Signal the frame decoder we have a problem. - d.sendErr(block, err) + block.sendErr(err) return err } - block.input <- struct{}{} - if debugDecoder { - println("next block:", block) - } - d.asyncRunningMu.Lock() - defer d.asyncRunningMu.Unlock() - if !d.asyncRunning { - return nil - } - if block.Last { - // We indicate the frame is done by sending io.EOF - d.decoding <- block - return io.EOF - } - d.decoding <- block return nil } -// sendEOF will queue an error block on the frame. -// This will cause the frame decoder to return when it encounters the block. -// Returns true if the decoder was added. -func (d *frameDec) sendErr(block *blockDec, err error) bool { - d.asyncRunningMu.Lock() - defer d.asyncRunningMu.Unlock() - if !d.asyncRunning { - return false - } - - println("sending error", err.Error()) - block.sendErr(err) - d.decoding <- block - return true -} - // checkCRC will check the checksum if the frame has one. // Will return ErrCRCMismatch if crc check failed, otherwise nil. func (d *frameDec) checkCRC() error { if !d.HasCheckSum { return nil } - var tmp [4]byte - got := d.crc.Sum64() - // Flip to match file order. - tmp[0] = byte(got >> 0) - tmp[1] = byte(got >> 8) - tmp[2] = byte(got >> 16) - tmp[3] = byte(got >> 24) // We can overwrite upper tmp now - want, err := d.rawInput.readSmall(4) + buf, err := d.rawInput.readSmall(4) if err != nil { println("CRC missing?", err) return err } - if !bytes.Equal(tmp[:], want) { + if d.o.ignoreChecksum { + return nil + } + + want := binary.LittleEndian.Uint32(buf[:4]) + got := uint32(d.crc.Sum64()) + + if got != want { if debugDecoder { - println("CRC Check Failed:", tmp[:], "!=", want) + printf("CRC check failed: got %08x, want %08x\n", got, want) } return ErrCRCMismatch } if debugDecoder { - println("CRC ok", tmp[:]) + printf("CRC ok %08x\n", got) } return nil } -func (d *frameDec) initAsync() { - if !d.o.lowMem && !d.SingleSegment { - // set max extra size history to 2MB. - d.history.maxSize = d.history.windowSize + maxBlockSize - } - // re-alloc if more than one extra block size. - if d.o.lowMem && cap(d.history.b) > d.history.maxSize+maxBlockSize { - d.history.b = make([]byte, 0, d.history.maxSize) - } - if cap(d.history.b) < d.history.maxSize { - d.history.b = make([]byte, 0, d.history.maxSize) - } - if cap(d.decoding) < d.o.concurrent { - d.decoding = make(chan *blockDec, d.o.concurrent) - } - if debugDecoder { - h := d.history - printf("history init. len: %d, cap: %d", len(h.b), cap(h.b)) - } - d.asyncRunningMu.Lock() - d.asyncRunning = true - d.asyncRunningMu.Unlock() -} - -// startDecoder will start decoding blocks and write them to the writer. -// The decoder will stop as soon as an error occurs or at end of frame. -// When the frame has finished decoding the *bufio.Reader -// containing the remaining input will be sent on frameDec.frameDone. -func (d *frameDec) startDecoder(output chan decodeOutput) { - written := int64(0) - - defer func() { - d.asyncRunningMu.Lock() - d.asyncRunning = false - d.asyncRunningMu.Unlock() - - // Drain the currently decoding. - d.history.error = true - flushdone: - for { - select { - case b := <-d.decoding: - b.history <- &d.history - output <- <-b.result - default: - break flushdone - } - } - println("frame decoder done, signalling done") - d.frameDone.Done() - }() - // Get decoder for first block. - block := <-d.decoding - block.history <- &d.history - for { - var next *blockDec - // Get result - r := <-block.result - if r.err != nil { - println("Result contained error", r.err) - output <- r - return - } - if debugDecoder { - println("got result, from ", d.offset, "to", d.offset+int64(len(r.b))) - d.offset += int64(len(r.b)) - } - if !block.Last { - // Send history to next block - select { - case next = <-d.decoding: - if debugDecoder { - println("Sending ", len(d.history.b), "bytes as history") - } - next.history <- &d.history - default: - // Wait until we have sent the block, so - // other decoders can potentially get the decoder. - next = nil - } - } - - // Add checksum, async to decoding. - if d.HasCheckSum { - n, err := d.crc.Write(r.b) - if err != nil { - r.err = err - if n != len(r.b) { - r.err = io.ErrShortWrite - } - output <- r - return - } - } - written += int64(len(r.b)) - if d.SingleSegment && uint64(written) > d.FrameContentSize { - println("runDecoder: single segment and", uint64(written), ">", d.FrameContentSize) - r.err = ErrFrameSizeExceeded - output <- r - return - } - if block.Last { - r.err = d.checkCRC() - output <- r - return - } - output <- r - if next == nil { - // There was no decoder available, we wait for one now that we have sent to the writer. - if debugDecoder { - println("Sending ", len(d.history.b), " bytes as history") - } - next = <-d.decoding - next.history <- &d.history +// consumeCRC reads the checksum data if the frame has one. +func (d *frameDec) consumeCRC() error { + if d.HasCheckSum { + _, err := d.rawInput.readSmall(4) + if err != nil { + println("CRC missing?", err) + return err } - block = next } + + return nil } -// runDecoder will create a sync decoder that will decode a block of data. +// runDecoder will run the decoder for the remainder of the frame. func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { saved := d.history.b // We use the history for output to avoid copying it. d.history.b = dst + d.history.ignoreBuffer = len(dst) // Store input length, so we only check new data. crcStart := len(dst) + d.history.decoders.maxSyncLen = 0 + if d.o.limitToCap { + d.history.decoders.maxSyncLen = uint64(cap(dst) - len(dst)) + } + if d.FrameContentSize != fcsUnknown { + if !d.o.limitToCap || d.FrameContentSize+uint64(len(dst)) < d.history.decoders.maxSyncLen { + d.history.decoders.maxSyncLen = d.FrameContentSize + uint64(len(dst)) + } + if d.history.decoders.maxSyncLen > d.o.maxDecodedSize { + if debugDecoder { + println("maxSyncLen:", d.history.decoders.maxSyncLen, "> maxDecodedSize:", d.o.maxDecodedSize) + } + return dst, ErrDecoderSizeExceeded + } + if debugDecoder { + println("maxSyncLen:", d.history.decoders.maxSyncLen) + } + if !d.o.limitToCap && uint64(cap(dst)) < d.history.decoders.maxSyncLen { + // Alloc for output + dst2 := make([]byte, len(dst), d.history.decoders.maxSyncLen+compressedBlockOverAlloc) + copy(dst2, dst) + dst = dst2 + } + } var err error for { err = dec.reset(d.rawInput, d.WindowSize) @@ -489,29 +382,47 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { println("next block:", dec) } err = dec.decodeBuf(&d.history) - if err != nil || dec.Last { + if err != nil { + break + } + if uint64(len(d.history.b)-crcStart) > d.o.maxDecodedSize { + println("runDecoder: maxDecodedSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.o.maxDecodedSize) + err = ErrDecoderSizeExceeded break } - if uint64(len(d.history.b)) > d.o.maxDecodedSize { + if d.o.limitToCap && len(d.history.b) > cap(dst) { + println("runDecoder: cap exceeded", uint64(len(d.history.b)), ">", cap(dst)) err = ErrDecoderSizeExceeded break } - if d.SingleSegment && uint64(len(d.history.b)) > d.o.maxDecodedSize { - println("runDecoder: single segment and", uint64(len(d.history.b)), ">", d.o.maxDecodedSize) + if uint64(len(d.history.b)-crcStart) > d.FrameContentSize { + println("runDecoder: FrameContentSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.FrameContentSize) err = ErrFrameSizeExceeded break } + if dec.Last { + break + } + if debugDecoder { + println("runDecoder: FrameContentSize", uint64(len(d.history.b)-crcStart), "<=", d.FrameContentSize) + } } dst = d.history.b if err == nil { - if d.HasCheckSum { - var n int - n, err = d.crc.Write(dst[crcStart:]) - if err == nil { - if n != len(dst)-crcStart { - err = io.ErrShortWrite - } else { - err = d.checkCRC() + if d.FrameContentSize != fcsUnknown && uint64(len(d.history.b)-crcStart) != d.FrameContentSize { + err = ErrFrameSizeMismatch + } else if d.HasCheckSum { + if d.o.ignoreChecksum { + err = d.consumeCRC() + } else { + var n int + n, err = d.crc.Write(dst[crcStart:]) + if err == nil { + if n != len(dst)-crcStart { + err = io.ErrShortWrite + } else { + err = d.checkCRC() + } } } } diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go index e6d3d49b3..2f8860a72 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go @@ -5,8 +5,10 @@ package zstd import ( + "encoding/binary" "errors" "fmt" + "io" ) const ( @@ -178,10 +180,32 @@ func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { return fmt.Errorf("corruption detected (total %d != %d)", gotTotal, 1<> 3) - // println(s.norm[:s.symbolLen], s.symbolLen) return s.buildDtable() } +func (s *fseDecoder) mustReadFrom(r io.Reader) { + fatalErr := func(err error) { + if err != nil { + panic(err) + } + } + // dt [maxTablesize]decSymbol // Decompression table. + // symbolLen uint16 // Length of active part of the symbol table. + // actualTableLog uint8 // Selected tablelog. + // maxBits uint8 // Maximum number of additional bits + // // used for table creation to avoid allocations. + // stateTable [256]uint16 + // norm [maxSymbolValue + 1]int16 + // preDefined bool + fatalErr(binary.Read(r, binary.LittleEndian, &s.dt)) + fatalErr(binary.Read(r, binary.LittleEndian, &s.symbolLen)) + fatalErr(binary.Read(r, binary.LittleEndian, &s.actualTableLog)) + fatalErr(binary.Read(r, binary.LittleEndian, &s.maxBits)) + fatalErr(binary.Read(r, binary.LittleEndian, &s.stateTable)) + fatalErr(binary.Read(r, binary.LittleEndian, &s.norm)) + fatalErr(binary.Read(r, binary.LittleEndian, &s.preDefined)) +} + // decSymbol contains information about a state entry, // Including the state offset base, the output symbol and // the number of bits to read for the low part of the destination state. @@ -204,18 +228,10 @@ func (d decSymbol) newState() uint16 { return uint16(d >> 16) } -func (d decSymbol) baseline() uint32 { - return uint32(d >> 32) -} - func (d decSymbol) baselineInt() int { return int(d >> 32) } -func (d *decSymbol) set(nbits, addBits uint8, newState uint16, baseline uint32) { - *d = decSymbol(nbits) | (decSymbol(addBits) << 8) | (decSymbol(newState) << 16) | (decSymbol(baseline) << 32) -} - func (d *decSymbol) setNBits(nBits uint8) { const mask = 0xffffffffffffff00 *d = (*d & mask) | decSymbol(nBits) @@ -231,11 +247,6 @@ func (d *decSymbol) setNewState(state uint16) { *d = (*d & mask) | decSymbol(state)<<16 } -func (d *decSymbol) setBaseline(baseline uint32) { - const mask = 0xffffffff - *d = (*d & mask) | decSymbol(baseline)<<32 -} - func (d *decSymbol) setExt(addBits uint8, baseline uint32) { const mask = 0xffff00ff *d = (*d & mask) | (decSymbol(addBits) << 8) | (decSymbol(baseline) << 32) @@ -257,68 +268,6 @@ func (s *fseDecoder) setRLE(symbol decSymbol) { s.dt[0] = symbol } -// buildDtable will build the decoding table. -func (s *fseDecoder) buildDtable() error { - tableSize := uint32(1 << s.actualTableLog) - highThreshold := tableSize - 1 - symbolNext := s.stateTable[:256] - - // Init, lay down lowprob symbols - { - for i, v := range s.norm[:s.symbolLen] { - if v == -1 { - s.dt[highThreshold].setAddBits(uint8(i)) - highThreshold-- - symbolNext[i] = 1 - } else { - symbolNext[i] = uint16(v) - } - } - } - // Spread symbols - { - tableMask := tableSize - 1 - step := tableStep(tableSize) - position := uint32(0) - for ss, v := range s.norm[:s.symbolLen] { - for i := 0; i < int(v); i++ { - s.dt[position].setAddBits(uint8(ss)) - position = (position + step) & tableMask - for position > highThreshold { - // lowprob area - position = (position + step) & tableMask - } - } - } - if position != 0 { - // position must reach all cells once, otherwise normalizedCounter is incorrect - return errors.New("corrupted input (position != 0)") - } - } - - // Build Decoding table - { - tableSize := uint16(1 << s.actualTableLog) - for u, v := range s.dt[:tableSize] { - symbol := v.addBits() - nextState := symbolNext[symbol] - symbolNext[symbol] = nextState + 1 - nBits := s.actualTableLog - byte(highBits(uint32(nextState))) - s.dt[u&maxTableMask].setNBits(nBits) - newState := (nextState << nBits) - tableSize - if newState > tableSize { - return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) - } - if newState == uint16(u) && nBits == 0 { - // Seems weird that this is possible with nbits > 0. - return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) - } - s.dt[u&maxTableMask].setNewState(newState) - } - } - return nil -} - // transform will transform the decoder table into a table usable for // decoding without having to apply the transformation while decoding. // The state will contain the base value and the number of bits to read. @@ -352,34 +301,7 @@ func (s *fseState) init(br *bitReader, tableLog uint8, dt []decSymbol) { s.state = dt[br.getBits(tableLog)] } -// next returns the current symbol and sets the next state. -// At least tablelog bits must be available in the bit reader. -func (s *fseState) next(br *bitReader) { - lowBits := uint16(br.getBits(s.state.nbBits())) - s.state = s.dt[s.state.newState()+lowBits] -} - -// finished returns true if all bits have been read from the bitstream -// and the next state would require reading bits from the input. -func (s *fseState) finished(br *bitReader) bool { - return br.finished() && s.state.nbBits() > 0 -} - -// final returns the current state symbol without decoding the next. -func (s *fseState) final() (int, uint8) { - return s.state.baselineInt(), s.state.addBits() -} - // final returns the current state symbol without decoding the next. func (s decSymbol) final() (int, uint8) { return s.baselineInt(), s.addBits() } - -// nextFast returns the next symbol and sets the next state. -// This can only be used if no symbols are 0 bits. -// At least tablelog bits must be available in the bit reader. -func (s *fseState) nextFast(br *bitReader) (uint32, uint8) { - lowBits := uint16(br.getBitsFast(s.state.nbBits())) - s.state = s.dt[s.state.newState()+lowBits] - return s.state.baseline(), s.state.addBits() -} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go new file mode 100644 index 000000000..d04a829b0 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go @@ -0,0 +1,65 @@ +//go:build amd64 && !appengine && !noasm && gc +// +build amd64,!appengine,!noasm,gc + +package zstd + +import ( + "fmt" +) + +type buildDtableAsmContext struct { + // inputs + stateTable *uint16 + norm *int16 + dt *uint64 + + // outputs --- set by the procedure in the case of error; + // for interpretation please see the error handling part below + errParam1 uint64 + errParam2 uint64 +} + +// buildDtable_asm is an x86 assembly implementation of fseDecoder.buildDtable. +// Function returns non-zero exit code on error. +// +//go:noescape +func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int + +// please keep in sync with _generate/gen_fse.go +const ( + errorCorruptedNormalizedCounter = 1 + errorNewStateTooBig = 2 + errorNewStateNoBits = 3 +) + +// buildDtable will build the decoding table. +func (s *fseDecoder) buildDtable() error { + ctx := buildDtableAsmContext{ + stateTable: &s.stateTable[0], + norm: &s.norm[0], + dt: (*uint64)(&s.dt[0]), + } + code := buildDtable_asm(s, &ctx) + + if code != 0 { + switch code { + case errorCorruptedNormalizedCounter: + position := ctx.errParam1 + return fmt.Errorf("corrupted input (position=%d, expected 0)", position) + + case errorNewStateTooBig: + newState := decSymbol(ctx.errParam1) + size := ctx.errParam2 + return fmt.Errorf("newState (%d) outside table size (%d)", newState, size) + + case errorNewStateNoBits: + newState := decSymbol(ctx.errParam1) + oldState := decSymbol(ctx.errParam2) + return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, oldState) + + default: + return fmt.Errorf("buildDtable_asm returned unhandled nonzero code = %d", code) + } + } + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s new file mode 100644 index 000000000..bcde39869 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s @@ -0,0 +1,126 @@ +// Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT. + +//go:build !appengine && !noasm && gc && !noasm + +// func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int +TEXT ·buildDtable_asm(SB), $0-24 + MOVQ ctx+8(FP), CX + MOVQ s+0(FP), DI + + // Load values + MOVBQZX 4098(DI), DX + XORQ AX, AX + BTSQ DX, AX + MOVQ (CX), BX + MOVQ 16(CX), SI + LEAQ -1(AX), R8 + MOVQ 8(CX), CX + MOVWQZX 4096(DI), DI + + // End load values + // Init, lay down lowprob symbols + XORQ R9, R9 + JMP init_main_loop_condition + +init_main_loop: + MOVWQSX (CX)(R9*2), R10 + CMPW R10, $-1 + JNE do_not_update_high_threshold + MOVB R9, 1(SI)(R8*8) + DECQ R8 + MOVQ $0x0000000000000001, R10 + +do_not_update_high_threshold: + MOVW R10, (BX)(R9*2) + INCQ R9 + +init_main_loop_condition: + CMPQ R9, DI + JL init_main_loop + + // Spread symbols + // Calculate table step + MOVQ AX, R9 + SHRQ $0x01, R9 + MOVQ AX, R10 + SHRQ $0x03, R10 + LEAQ 3(R9)(R10*1), R9 + + // Fill add bits values + LEAQ -1(AX), R10 + XORQ R11, R11 + XORQ R12, R12 + JMP spread_main_loop_condition + +spread_main_loop: + XORQ R13, R13 + MOVWQSX (CX)(R12*2), R14 + JMP spread_inner_loop_condition + +spread_inner_loop: + MOVB R12, 1(SI)(R11*8) + +adjust_position: + ADDQ R9, R11 + ANDQ R10, R11 + CMPQ R11, R8 + JG adjust_position + INCQ R13 + +spread_inner_loop_condition: + CMPQ R13, R14 + JL spread_inner_loop + INCQ R12 + +spread_main_loop_condition: + CMPQ R12, DI + JL spread_main_loop + TESTQ R11, R11 + JZ spread_check_ok + MOVQ ctx+8(FP), AX + MOVQ R11, 24(AX) + MOVQ $+1, ret+16(FP) + RET + +spread_check_ok: + // Build Decoding table + XORQ DI, DI + +build_table_main_table: + MOVBQZX 1(SI)(DI*8), CX + MOVWQZX (BX)(CX*2), R8 + LEAQ 1(R8), R9 + MOVW R9, (BX)(CX*2) + MOVQ R8, R9 + BSRQ R9, R9 + MOVQ DX, CX + SUBQ R9, CX + SHLQ CL, R8 + SUBQ AX, R8 + MOVB CL, (SI)(DI*8) + MOVW R8, 2(SI)(DI*8) + CMPQ R8, AX + JLE build_table_check1_ok + MOVQ ctx+8(FP), CX + MOVQ R8, 24(CX) + MOVQ AX, 32(CX) + MOVQ $+2, ret+16(FP) + RET + +build_table_check1_ok: + TESTB CL, CL + JNZ build_table_check2_ok + CMPW R8, DI + JNE build_table_check2_ok + MOVQ ctx+8(FP), AX + MOVQ R8, 24(AX) + MOVQ DI, 32(AX) + MOVQ $+3, ret+16(FP) + RET + +build_table_check2_ok: + INCQ DI + CMPQ DI, AX + JL build_table_main_table + MOVQ $+0, ret+16(FP) + RET diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go new file mode 100644 index 000000000..332e51fe4 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -0,0 +1,72 @@ +//go:build !amd64 || appengine || !gc || noasm +// +build !amd64 appengine !gc noasm + +package zstd + +import ( + "errors" + "fmt" +) + +// buildDtable will build the decoding table. +func (s *fseDecoder) buildDtable() error { + tableSize := uint32(1 << s.actualTableLog) + highThreshold := tableSize - 1 + symbolNext := s.stateTable[:256] + + // Init, lay down lowprob symbols + { + for i, v := range s.norm[:s.symbolLen] { + if v == -1 { + s.dt[highThreshold].setAddBits(uint8(i)) + highThreshold-- + symbolNext[i] = 1 + } else { + symbolNext[i] = uint16(v) + } + } + } + + // Spread symbols + { + tableMask := tableSize - 1 + step := tableStep(tableSize) + position := uint32(0) + for ss, v := range s.norm[:s.symbolLen] { + for i := 0; i < int(v); i++ { + s.dt[position].setAddBits(uint8(ss)) + position = (position + step) & tableMask + for position > highThreshold { + // lowprob area + position = (position + step) & tableMask + } + } + } + if position != 0 { + // position must reach all cells once, otherwise normalizedCounter is incorrect + return errors.New("corrupted input (position != 0)") + } + } + + // Build Decoding table + { + tableSize := uint16(1 << s.actualTableLog) + for u, v := range s.dt[:tableSize] { + symbol := v.addBits() + nextState := symbolNext[symbol] + symbolNext[symbol] = nextState + 1 + nBits := s.actualTableLog - byte(highBits(uint32(nextState))) + s.dt[u&maxTableMask].setNBits(nBits) + newState := (nextState << nBits) - tableSize + if newState > tableSize { + return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) + } + if newState == uint16(u) && nBits == 0 { + // Seems weird that this is possible with nbits > 0. + return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) + } + s.dt[u&maxTableMask].setNewState(newState) + } + } + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go index b4757ee3f..ab26326a8 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -62,9 +62,8 @@ func (s symbolTransform) String() string { // To indicate that you have populated the histogram call HistogramFinished // with the value of the highest populated symbol, as well as the number of entries // in the most populated entry. These are accepted at face value. -// The returned slice will always be length 256. -func (s *fseEncoder) Histogram() []uint32 { - return s.count[:] +func (s *fseEncoder) Histogram() *[256]uint32 { + return &s.count } // HistogramFinished can be called to indicate that the histogram has been populated. @@ -77,21 +76,6 @@ func (s *fseEncoder) HistogramFinished(maxSymbol uint8, maxCount int) { s.clearCount = maxCount != 0 } -// prepare will prepare and allocate scratch tables used for both compression and decompression. -func (s *fseEncoder) prepare() (*fseEncoder, error) { - if s == nil { - s = &fseEncoder{} - } - s.useRLE = false - if s.clearCount && s.maxCount == 0 { - for i := range s.count { - s.count[i] = 0 - } - s.clearCount = false - } - return s, nil -} - // allocCtable will allocate tables needed for compression. // If existing tables a re big enough, they are simply re-used. func (s *fseEncoder) allocCtable() { @@ -710,14 +694,6 @@ func (c *cState) init(bw *bitWriter, ct *cTable, first symbolTransform) { c.state = c.stateTable[lu] } -// encode the output symbol provided and write it to the bitstream. -func (c *cState) encode(symbolTT symbolTransform) { - nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16 - dstState := int32(c.state>>(nbBitsOut&15)) + int32(symbolTT.deltaFindState) - c.bw.addBits16NC(c.state, uint8(nbBitsOut)) - c.state = c.stateTable[dstState] -} - // flush will write the tablelog to the output and flush the remaining full bytes. func (c *cState) flush(tableLog uint8) { c.bw.flush32() diff --git a/vendor/github.com/klauspost/compress/zstd/hash.go b/vendor/github.com/klauspost/compress/zstd/hash.go index cf33f29a1..5d73c21eb 100644 --- a/vendor/github.com/klauspost/compress/zstd/hash.go +++ b/vendor/github.com/klauspost/compress/zstd/hash.go @@ -33,9 +33,3 @@ func hashLen(u uint64, length, mls uint8) uint32 { return (uint32(u) * prime4bytes) >> (32 - length) } } - -// hash3 returns the hash of the lower 3 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash3(u uint32, h uint8) uint32 { - return ((u << (32 - 24)) * prime3bytes) >> ((32 - h) & 31) -} diff --git a/vendor/github.com/klauspost/compress/zstd/history.go b/vendor/github.com/klauspost/compress/zstd/history.go index f783e32d2..09164856d 100644 --- a/vendor/github.com/klauspost/compress/zstd/history.go +++ b/vendor/github.com/klauspost/compress/zstd/history.go @@ -10,40 +10,48 @@ import ( // history contains the information transferred between blocks. type history struct { - b []byte - huffTree *huff0.Scratch - recentOffsets [3]int + // Literal decompression + huffTree *huff0.Scratch + + // Sequence decompression decoders sequenceDecs - windowSize int - maxSize int - error bool - dict *dict + recentOffsets [3]int + + // History buffer... + b []byte + + // ignoreBuffer is meant to ignore a number of bytes + // when checking for matches in history + ignoreBuffer int + + windowSize int + allocFrameBuffer int // needed? + error bool + dict *dict } // reset will reset the history to initial state of a frame. // The history must already have been initialized to the desired size. func (h *history) reset() { h.b = h.b[:0] + h.ignoreBuffer = 0 h.error = false h.recentOffsets = [3]int{1, 4, 8} - if f := h.decoders.litLengths.fse; f != nil && !f.preDefined { - fseDecoderPool.Put(f) - } - if f := h.decoders.offsets.fse; f != nil && !f.preDefined { - fseDecoderPool.Put(f) - } - if f := h.decoders.matchLengths.fse; f != nil && !f.preDefined { - fseDecoderPool.Put(f) - } - h.decoders = sequenceDecs{} + h.decoders.freeDecoders() + h.decoders = sequenceDecs{br: h.decoders.br} + h.freeHuffDecoder() + h.huffTree = nil + h.dict = nil + //printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b)) +} + +func (h *history) freeHuffDecoder() { if h.huffTree != nil { if h.dict == nil || h.dict.litEnc != h.huffTree { huffDecoderPool.Put(h.huffTree) + h.huffTree = nil } } - h.huffTree = nil - h.dict = nil - //printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b)) } func (h *history) setDict(dict *dict) { @@ -54,6 +62,7 @@ func (h *history) setDict(dict *dict) { h.decoders.litLengths = dict.llDec h.decoders.offsets = dict.ofDec h.decoders.matchLengths = dict.mlDec + h.decoders.dict = dict.content h.recentOffsets = dict.offsets h.huffTree = dict.litEnc } @@ -83,6 +92,24 @@ func (h *history) append(b []byte) { copy(h.b[h.windowSize-len(b):], b) } +// ensureBlock will ensure there is space for at least one block... +func (h *history) ensureBlock() { + if cap(h.b) < h.allocFrameBuffer { + h.b = make([]byte, 0, h.allocFrameBuffer) + return + } + + avail := cap(h.b) - len(h.b) + if avail >= h.windowSize || avail > maxCompressedBlockSize { + return + } + // Move data down so we only have window size left. + // We know we have less than window size in b at this point. + discard := len(h.b) - h.windowSize + copy(h.b, h.b[discard:]) + h.b = h.b[:h.windowSize] +} + // append bytes to history without ever discarding anything. func (h *history) appendKeep(b []byte) { h.b = append(h.b, b...) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md index 69aa3bb58..777290d44 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md @@ -2,12 +2,7 @@ VENDORED: Go to [github.com/cespare/xxhash](https://github.com/cespare/xxhash) for original package. - -[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash) -[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash) - -xxhash is a Go implementation of the 64-bit -[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a +xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a high-quality hashing algorithm that is much faster than anything in the Go standard library. @@ -28,31 +23,49 @@ func (*Digest) WriteString(string) (int, error) func (*Digest) Sum64() uint64 ``` -This implementation provides a fast pure-Go implementation and an even faster -assembly implementation for amd64. +The package is written with optimized pure Go and also contains even faster +assembly implementations for amd64 and arm64. If desired, the `purego` build tag +opts into using the Go code even on those architectures. + +[xxHash]: http://cyan4973.github.io/xxHash/ + +## Compatibility + +This package is in a module and the latest code is in version 2 of the module. +You need a version of Go with at least "minimal module compatibility" to use +github.com/cespare/xxhash/v2: + +* 1.9.7+ for Go 1.9 +* 1.10.3+ for Go 1.10 +* Go 1.11 or later + +I recommend using the latest release of Go. ## Benchmarks Here are some quick benchmarks comparing the pure-Go and assembly implementations of Sum64. -| input size | purego | asm | -| --- | --- | --- | -| 5 B | 979.66 MB/s | 1291.17 MB/s | -| 100 B | 7475.26 MB/s | 7973.40 MB/s | -| 4 KB | 17573.46 MB/s | 17602.65 MB/s | -| 10 MB | 17131.46 MB/s | 17142.16 MB/s | +| input size | purego | asm | +| ---------- | --------- | --------- | +| 4 B | 1.3 GB/s | 1.2 GB/s | +| 16 B | 2.9 GB/s | 3.5 GB/s | +| 100 B | 6.9 GB/s | 8.1 GB/s | +| 4 KB | 11.7 GB/s | 16.7 GB/s | +| 10 MB | 12.0 GB/s | 17.3 GB/s | -These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using -the following commands under Go 1.11.2: +These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C +CPU using the following commands under Go 1.19.2: ``` -$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes' -$ go test -benchtime 10s -bench '/xxhash,direct,bytes' +benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$') +benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$') ``` ## Projects using this package - [InfluxDB](https://github.com/influxdata/influxdb) - [Prometheus](https://github.com/prometheus/prometheus) +- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) - [FreeCache](https://github.com/coocood/freecache) +- [FastCache](https://github.com/VictoriaMetrics/fastcache) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go index 426b9cac7..fc40c8200 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go @@ -18,19 +18,11 @@ const ( prime5 uint64 = 2870177450012600261 ) -// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where -// possible in the Go code is worth a small (but measurable) performance boost -// by avoiding some MOVQs. Vars are needed for the asm and also are useful for -// convenience in the Go code in a few places where we need to intentionally -// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the -// result overflows a uint64). -var ( - prime1v = prime1 - prime2v = prime2 - prime3v = prime3 - prime4v = prime4 - prime5v = prime5 -) +// Store the primes in an array as well. +// +// The consts are used when possible in Go code to avoid MOVs but we need a +// contiguous array of the assembly code. +var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5} // Digest implements hash.Hash64. type Digest struct { @@ -52,10 +44,10 @@ func New() *Digest { // Reset clears the Digest's state so that it can be reused. func (d *Digest) Reset() { - d.v1 = prime1v + prime2 + d.v1 = primes[0] + prime2 d.v2 = prime2 d.v3 = 0 - d.v4 = -prime1v + d.v4 = -primes[0] d.total = 0 d.n = 0 } @@ -71,21 +63,23 @@ func (d *Digest) Write(b []byte) (n int, err error) { n = len(b) d.total += uint64(n) + memleft := d.mem[d.n&(len(d.mem)-1):] + if d.n+n < 32 { // This new data doesn't even fill the current block. - copy(d.mem[d.n:], b) + copy(memleft, b) d.n += n return } if d.n > 0 { // Finish off the partial block. - copy(d.mem[d.n:], b) + c := copy(memleft, b) d.v1 = round(d.v1, u64(d.mem[0:8])) d.v2 = round(d.v2, u64(d.mem[8:16])) d.v3 = round(d.v3, u64(d.mem[16:24])) d.v4 = round(d.v4, u64(d.mem[24:32])) - b = b[32-d.n:] + b = b[c:] d.n = 0 } @@ -135,21 +129,20 @@ func (d *Digest) Sum64() uint64 { h += d.total - i, end := 0, d.n - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(d.mem[i:i+8])) + b := d.mem[:d.n&(len(d.mem)-1)] + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(d.mem[i:i+4])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for i < end { - h ^= uint64(d.mem[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 - i++ } h ^= h >> 33 @@ -195,7 +188,6 @@ func (d *Digest) UnmarshalBinary(b []byte) error { b, d.v4 = consumeUint64(b) b, d.total = consumeUint64(b) copy(d.mem[:], b) - b = b[len(d.mem):] d.n = int(d.total % uint64(len(d.mem))) return nil } diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go deleted file mode 100644 index 3ddbd5c0b..000000000 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go +++ /dev/null @@ -1,12 +0,0 @@ -//go:build !appengine && gc && !purego -// +build !appengine,gc,!purego - -package xxhash - -// Sum64 computes the 64-bit xxHash digest of b. -// -//go:noescape -func Sum64(b []byte) uint64 - -//go:noescape -func writeBlocks(*Digest, []byte) int diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s index 2c9c5357a..ddb63aa91 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -1,215 +1,210 @@ +//go:build !appengine && gc && !purego && !noasm // +build !appengine // +build gc // +build !purego +// +build !noasm #include "textflag.h" -// Register allocation: -// AX h -// CX pointer to advance through b -// DX n -// BX loop end -// R8 v1, k1 -// R9 v2 -// R10 v3 -// R11 v4 -// R12 tmp -// R13 prime1v -// R14 prime2v -// R15 prime4v - -// round reads from and advances the buffer pointer in CX. -// It assumes that R13 has prime1v and R14 has prime2v. -#define round(r) \ - MOVQ (CX), R12 \ - ADDQ $8, CX \ - IMULQ R14, R12 \ - ADDQ R12, r \ - ROLQ $31, r \ - IMULQ R13, r - -// mergeRound applies a merge round on the two registers acc and val. -// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v. -#define mergeRound(acc, val) \ - IMULQ R14, val \ - ROLQ $31, val \ - IMULQ R13, val \ - XORQ val, acc \ - IMULQ R13, acc \ - ADDQ R15, acc +// Registers: +#define h AX +#define d AX +#define p SI // pointer to advance through b +#define n DX +#define end BX // loop end +#define v1 R8 +#define v2 R9 +#define v3 R10 +#define v4 R11 +#define x R12 +#define prime1 R13 +#define prime2 R14 +#define prime4 DI + +#define round(acc, x) \ + IMULQ prime2, x \ + ADDQ x, acc \ + ROLQ $31, acc \ + IMULQ prime1, acc + +// round0 performs the operation x = round(0, x). +#define round0(x) \ + IMULQ prime2, x \ + ROLQ $31, x \ + IMULQ prime1, x + +// mergeRound applies a merge round on the two registers acc and x. +// It assumes that prime1, prime2, and prime4 have been loaded. +#define mergeRound(acc, x) \ + round0(x) \ + XORQ x, acc \ + IMULQ prime1, acc \ + ADDQ prime4, acc + +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that there is at least one block +// to process. +#define blockLoop() \ +loop: \ + MOVQ +0(p), x \ + round(v1, x) \ + MOVQ +8(p), x \ + round(v2, x) \ + MOVQ +16(p), x \ + round(v3, x) \ + MOVQ +24(p), x \ + round(v4, x) \ + ADDQ $32, p \ + CMPQ p, end \ + JLE loop // func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOSPLIT, $0-32 +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 // Load fixed primes. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 - MOVQ ·prime4v(SB), R15 + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 + MOVQ ·primes+24(SB), prime4 // Load slice. - MOVQ b_base+0(FP), CX - MOVQ b_len+8(FP), DX - LEAQ (CX)(DX*1), BX + MOVQ b_base+0(FP), p + MOVQ b_len+8(FP), n + LEAQ (p)(n*1), end // The first loop limit will be len(b)-32. - SUBQ $32, BX + SUBQ $32, end // Check whether we have at least one block. - CMPQ DX, $32 + CMPQ n, $32 JLT noBlocks // Set up initial state (v1, v2, v3, v4). - MOVQ R13, R8 - ADDQ R14, R8 - MOVQ R14, R9 - XORQ R10, R10 - XORQ R11, R11 - SUBQ R13, R11 - - // Loop until CX > BX. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ CX, BX - JLE blockLoop - - MOVQ R8, AX - ROLQ $1, AX - MOVQ R9, R12 - ROLQ $7, R12 - ADDQ R12, AX - MOVQ R10, R12 - ROLQ $12, R12 - ADDQ R12, AX - MOVQ R11, R12 - ROLQ $18, R12 - ADDQ R12, AX - - mergeRound(AX, R8) - mergeRound(AX, R9) - mergeRound(AX, R10) - mergeRound(AX, R11) + MOVQ prime1, v1 + ADDQ prime2, v1 + MOVQ prime2, v2 + XORQ v3, v3 + XORQ v4, v4 + SUBQ prime1, v4 + + blockLoop() + + MOVQ v1, h + ROLQ $1, h + MOVQ v2, x + ROLQ $7, x + ADDQ x, h + MOVQ v3, x + ROLQ $12, x + ADDQ x, h + MOVQ v4, x + ROLQ $18, x + ADDQ x, h + + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) JMP afterBlocks noBlocks: - MOVQ ·prime5v(SB), AX + MOVQ ·primes+32(SB), h afterBlocks: - ADDQ DX, AX - - // Right now BX has len(b)-32, and we want to loop until CX > len(b)-8. - ADDQ $24, BX - - CMPQ CX, BX - JG fourByte - -wordLoop: - // Calculate k1. - MOVQ (CX), R8 - ADDQ $8, CX - IMULQ R14, R8 - ROLQ $31, R8 - IMULQ R13, R8 - - XORQ R8, AX - ROLQ $27, AX - IMULQ R13, AX - ADDQ R15, AX - - CMPQ CX, BX - JLE wordLoop - -fourByte: - ADDQ $4, BX - CMPQ CX, BX - JG singles - - MOVL (CX), R8 - ADDQ $4, CX - IMULQ R13, R8 - XORQ R8, AX - - ROLQ $23, AX - IMULQ R14, AX - ADDQ ·prime3v(SB), AX - -singles: - ADDQ $4, BX - CMPQ CX, BX + ADDQ n, h + + ADDQ $24, end + CMPQ p, end + JG try4 + +loop8: + MOVQ (p), x + ADDQ $8, p + round0(x) + XORQ x, h + ROLQ $27, h + IMULQ prime1, h + ADDQ prime4, h + + CMPQ p, end + JLE loop8 + +try4: + ADDQ $4, end + CMPQ p, end + JG try1 + + MOVL (p), x + ADDQ $4, p + IMULQ prime1, x + XORQ x, h + + ROLQ $23, h + IMULQ prime2, h + ADDQ ·primes+16(SB), h + +try1: + ADDQ $4, end + CMPQ p, end JGE finalize -singlesLoop: - MOVBQZX (CX), R12 - ADDQ $1, CX - IMULQ ·prime5v(SB), R12 - XORQ R12, AX +loop1: + MOVBQZX (p), x + ADDQ $1, p + IMULQ ·primes+32(SB), x + XORQ x, h + ROLQ $11, h + IMULQ prime1, h - ROLQ $11, AX - IMULQ R13, AX - - CMPQ CX, BX - JL singlesLoop + CMPQ p, end + JL loop1 finalize: - MOVQ AX, R12 - SHRQ $33, R12 - XORQ R12, AX - IMULQ R14, AX - MOVQ AX, R12 - SHRQ $29, R12 - XORQ R12, AX - IMULQ ·prime3v(SB), AX - MOVQ AX, R12 - SHRQ $32, R12 - XORQ R12, AX - - MOVQ AX, ret+24(FP) + MOVQ h, x + SHRQ $33, x + XORQ x, h + IMULQ prime2, h + MOVQ h, x + SHRQ $29, x + XORQ x, h + IMULQ ·primes+16(SB), h + MOVQ h, x + SHRQ $32, x + XORQ x, h + + MOVQ h, ret+24(FP) RET -// writeBlocks uses the same registers as above except that it uses AX to store -// the d pointer. - // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT, $0-40 +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 // Load fixed primes needed for round. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 // Load slice. - MOVQ arg1_base+8(FP), CX - MOVQ arg1_len+16(FP), DX - LEAQ (CX)(DX*1), BX - SUBQ $32, BX + MOVQ b_base+8(FP), p + MOVQ b_len+16(FP), n + LEAQ (p)(n*1), end + SUBQ $32, end // Load vN from d. - MOVQ arg+0(FP), AX - MOVQ 0(AX), R8 // v1 - MOVQ 8(AX), R9 // v2 - MOVQ 16(AX), R10 // v3 - MOVQ 24(AX), R11 // v4 + MOVQ s+0(FP), d + MOVQ 0(d), v1 + MOVQ 8(d), v2 + MOVQ 16(d), v3 + MOVQ 24(d), v4 // We don't need to check the loop condition here; this function is // always called with at least one block of data to process. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ CX, BX - JLE blockLoop + blockLoop() // Copy vN back to d. - MOVQ R8, 0(AX) - MOVQ R9, 8(AX) - MOVQ R10, 16(AX) - MOVQ R11, 24(AX) - - // The number of bytes written is CX minus the old base pointer. - SUBQ arg1_base+8(FP), CX - MOVQ CX, ret+32(FP) + MOVQ v1, 0(d) + MOVQ v2, 8(d) + MOVQ v3, 16(d) + MOVQ v4, 24(d) + + // The number of bytes written is p minus the old base pointer. + SUBQ b_base+8(FP), p + MOVQ p, ret+32(FP) RET diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s new file mode 100644 index 000000000..17901e080 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -0,0 +1,184 @@ +//go:build !appengine && gc && !purego && !noasm +// +build !appengine +// +build gc +// +build !purego +// +build !noasm + +#include "textflag.h" + +// Registers: +#define digest R1 +#define h R2 // return value +#define p R3 // input pointer +#define n R4 // input length +#define nblocks R5 // n / 32 +#define prime1 R7 +#define prime2 R8 +#define prime3 R9 +#define prime4 R10 +#define prime5 R11 +#define v1 R12 +#define v2 R13 +#define v3 R14 +#define v4 R15 +#define x1 R20 +#define x2 R21 +#define x3 R22 +#define x4 R23 + +#define round(acc, x) \ + MADD prime2, acc, x, acc \ + ROR $64-31, acc \ + MUL prime1, acc + +// round0 performs the operation x = round(0, x). +#define round0(x) \ + MUL prime2, x \ + ROR $64-31, x \ + MUL prime1, x + +#define mergeRound(acc, x) \ + round0(x) \ + EOR x, acc \ + MADD acc, prime4, prime1, acc + +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that n >= 32. +#define blockLoop() \ + LSR $5, n, nblocks \ + PCALIGN $16 \ + loop: \ + LDP.P 16(p), (x1, x2) \ + LDP.P 16(p), (x3, x4) \ + round(v1, x1) \ + round(v2, x2) \ + round(v3, x3) \ + round(v4, x4) \ + SUB $1, nblocks \ + CBNZ nblocks, loop + +// func Sum64(b []byte) uint64 +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 + LDP b_base+0(FP), (p, n) + + LDP ·primes+0(SB), (prime1, prime2) + LDP ·primes+16(SB), (prime3, prime4) + MOVD ·primes+32(SB), prime5 + + CMP $32, n + CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 } + BLT afterLoop + + ADD prime1, prime2, v1 + MOVD prime2, v2 + MOVD $0, v3 + NEG prime1, v4 + + blockLoop() + + ROR $64-1, v1, x1 + ROR $64-7, v2, x2 + ADD x1, x2 + ROR $64-12, v3, x3 + ROR $64-18, v4, x4 + ADD x3, x4 + ADD x2, x4, h + + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) + +afterLoop: + ADD n, h + + TBZ $4, n, try8 + LDP.P 16(p), (x1, x2) + + round0(x1) + + // NOTE: here and below, sequencing the EOR after the ROR (using a + // rotated register) is worth a small but measurable speedup for small + // inputs. + ROR $64-27, h + EOR x1 @> 64-27, h, h + MADD h, prime4, prime1, h + + round0(x2) + ROR $64-27, h + EOR x2 @> 64-27, h, h + MADD h, prime4, prime1, h + +try8: + TBZ $3, n, try4 + MOVD.P 8(p), x1 + + round0(x1) + ROR $64-27, h + EOR x1 @> 64-27, h, h + MADD h, prime4, prime1, h + +try4: + TBZ $2, n, try2 + MOVWU.P 4(p), x2 + + MUL prime1, x2 + ROR $64-23, h + EOR x2 @> 64-23, h, h + MADD h, prime3, prime2, h + +try2: + TBZ $1, n, try1 + MOVHU.P 2(p), x3 + AND $255, x3, x1 + LSR $8, x3, x2 + + MUL prime5, x1 + ROR $64-11, h + EOR x1 @> 64-11, h, h + MUL prime1, h + + MUL prime5, x2 + ROR $64-11, h + EOR x2 @> 64-11, h, h + MUL prime1, h + +try1: + TBZ $0, n, finalize + MOVBU (p), x4 + + MUL prime5, x4 + ROR $64-11, h + EOR x4 @> 64-11, h, h + MUL prime1, h + +finalize: + EOR h >> 33, h + MUL prime2, h + EOR h >> 29, h + MUL prime3, h + EOR h >> 32, h + + MOVD h, ret+24(FP) + RET + +// func writeBlocks(d *Digest, b []byte) int +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 + LDP ·primes+0(SB), (prime1, prime2) + + // Load state. Assume v[1-4] are stored contiguously. + MOVD d+0(FP), digest + LDP 0(digest), (v1, v2) + LDP 16(digest), (v3, v4) + + LDP b_base+8(FP), (p, n) + + blockLoop() + + // Store updated state. + STP (v1, v2), 0(digest) + STP (v3, v4), 16(digest) + + BIC $31, n + MOVD n, ret+32(FP) + RET diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go new file mode 100644 index 000000000..d4221edf4 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go @@ -0,0 +1,16 @@ +//go:build (amd64 || arm64) && !appengine && gc && !purego && !noasm +// +build amd64 arm64 +// +build !appengine +// +build gc +// +build !purego +// +build !noasm + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +// +//go:noescape +func Sum64(b []byte) uint64 + +//go:noescape +func writeBlocks(s *Digest, b []byte) int diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go index 1f52f296e..0be16cefc 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go @@ -1,5 +1,5 @@ -//go:build !amd64 || appengine || !gc || purego -// +build !amd64 appengine !gc purego +//go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm +// +build !amd64,!arm64 appengine !gc purego noasm package xxhash @@ -15,10 +15,10 @@ func Sum64(b []byte) uint64 { var h uint64 if n >= 32 { - v1 := prime1v + prime2 + v1 := primes[0] + prime2 v2 := prime2 v3 := uint64(0) - v4 := -prime1v + v4 := -primes[0] for len(b) >= 32 { v1 = round(v1, u64(b[0:8:len(b)])) v2 = round(v2, u64(b[8:16:len(b)])) @@ -37,19 +37,18 @@ func Sum64(b []byte) uint64 { h += uint64(n) - i, end := 0, len(b) - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(b[i:i+8:len(b)])) + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for ; i < end; i++ { - h ^= uint64(b[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 } diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 1dd39e63b..f833d1541 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -20,6 +20,10 @@ type seq struct { llCode, mlCode, ofCode uint8 } +type seqVals struct { + ll, ml, mo int +} + func (s seq) String() string { if s.offset <= 3 { if s.offset == 0 { @@ -61,16 +65,19 @@ type sequenceDecs struct { offsets sequenceDec matchLengths sequenceDec prevOffset [3]int - hist []byte dict []byte literals []byte out []byte + nSeqs int + br *bitReader + seqSize int windowSize int maxBits uint8 + maxSyncLen uint64 } // initialize all 3 decoders from the stream input. -func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out []byte) error { +func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) error { if err := s.litLengths.init(br); err != nil { return errors.New("litLengths:" + err.Error()) } @@ -80,8 +87,7 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out [] if err := s.matchLengths.init(br); err != nil { return errors.New("matchLengths:" + err.Error()) } - s.literals = literals - s.hist = hist.b + s.br = br s.prevOffset = hist.recentOffsets s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits s.windowSize = hist.windowSize @@ -93,12 +99,142 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out [] return nil } +func (s *sequenceDecs) freeDecoders() { + if f := s.litLengths.fse; f != nil && !f.preDefined { + fseDecoderPool.Put(f) + s.litLengths.fse = nil + } + if f := s.offsets.fse; f != nil && !f.preDefined { + fseDecoderPool.Put(f) + s.offsets.fse = nil + } + if f := s.matchLengths.fse; f != nil && !f.preDefined { + fseDecoderPool.Put(f) + s.matchLengths.fse = nil + } +} + +// execute will execute the decoded sequence with the provided history. +// The sequence must be evaluated before being sent. +func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error { + if len(s.dict) == 0 { + return s.executeSimple(seqs, hist) + } + + // Ensure we have enough output size... + if len(s.out)+s.seqSize > cap(s.out) { + addBytes := s.seqSize + len(s.out) + s.out = append(s.out, make([]byte, addBytes)...) + s.out = s.out[:len(s.out)-addBytes] + } + + if debugDecoder { + printf("Execute %d seqs with hist %d, dict %d, literals: %d into %d bytes\n", len(seqs), len(hist), len(s.dict), len(s.literals), s.seqSize) + } + + var t = len(s.out) + out := s.out[:t+s.seqSize] + + for _, seq := range seqs { + // Add literals + copy(out[t:], s.literals[:seq.ll]) + t += seq.ll + s.literals = s.literals[seq.ll:] + + // Copy from dictionary... + if seq.mo > t+len(hist) || seq.mo > s.windowSize { + if len(s.dict) == 0 { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", seq.mo, t+len(hist)) + } + + // we may be in dictionary. + dictO := len(s.dict) - (seq.mo - (t + len(hist))) + if dictO < 0 || dictO >= len(s.dict) { + return fmt.Errorf("match offset (%d) bigger than current history+dict (%d)", seq.mo, t+len(hist)+len(s.dict)) + } + end := dictO + seq.ml + if end > len(s.dict) { + n := len(s.dict) - dictO + copy(out[t:], s.dict[dictO:]) + t += n + seq.ml -= n + } else { + copy(out[t:], s.dict[dictO:end]) + t += end - dictO + continue + } + } + + // Copy from history. + if v := seq.mo - t; v > 0 { + // v is the start position in history from end. + start := len(hist) - v + if seq.ml > v { + // Some goes into current block. + // Copy remainder of history + copy(out[t:], hist[start:]) + t += v + seq.ml -= v + } else { + copy(out[t:], hist[start:start+seq.ml]) + t += seq.ml + continue + } + } + // We must be in current buffer now + if seq.ml > 0 { + start := t - seq.mo + if seq.ml <= t-start { + // No overlap + copy(out[t:], out[start:start+seq.ml]) + t += seq.ml + continue + } else { + // Overlapping copy + // Extend destination slice and copy one byte at the time. + src := out[start : start+seq.ml] + dst := out[t:] + dst = dst[:len(src)] + t += len(src) + // Destination is the space we just added. + for i := range src { + dst[i] = src[i] + } + } + } + } + + // Add final literals + copy(out[t:], s.literals) + if debugDecoder { + t += len(s.literals) + if t != len(out) { + panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) + } + } + s.out = out + + return nil +} + // decode sequences from the stream with the provided history. -func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { +func (s *sequenceDecs) decodeSync(hist []byte) error { + supported, err := s.decodeSyncSimple(hist) + if supported { + return err + } + + br := s.br + seqs := s.nSeqs startSize := len(s.out) // Grab full sizes tables, to avoid bounds checks. llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize] llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state + out := s.out + maxBlockSize := maxCompressedBlockSize + if s.windowSize < maxBlockSize { + maxBlockSize = s.windowSize + } for i := seqs - 1; i >= 0; i-- { if br.overread() { @@ -151,7 +287,7 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { if temp == 0 { // 0 is not valid; input is corrupted; force offset to 1 - println("temp was 0") + println("WARNING: temp was 0") temp = 1 } @@ -176,51 +312,52 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { if ll > len(s.literals) { return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, len(s.literals)) } - size := ll + ml + len(s.out) + size := ll + ml + len(out) if size-startSize > maxBlockSize { - return fmt.Errorf("output (%d) bigger than max block size", size) + if size-startSize == 424242 { + panic("here") + } + return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) } - if size > cap(s.out) { + if size > cap(out) { // Not enough size, which can happen under high volume block streaming conditions // but could be if destination slice is too small for sync operations. // over-allocating here can create a large amount of GC pressure so we try to keep // it as contained as possible - used := len(s.out) - startSize + used := len(out) - startSize addBytes := 256 + ll + ml + used>>2 // Clamp to max block size. if used+addBytes > maxBlockSize { addBytes = maxBlockSize - used } - s.out = append(s.out, make([]byte, addBytes)...) - s.out = s.out[:len(s.out)-addBytes] + out = append(out, make([]byte, addBytes)...) + out = out[:len(out)-addBytes] } if ml > maxMatchLen { return fmt.Errorf("match len (%d) bigger than max allowed length", ml) } // Add literals - s.out = append(s.out, s.literals[:ll]...) + out = append(out, s.literals[:ll]...) s.literals = s.literals[ll:] - out := s.out if mo == 0 && ml > 0 { return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) } - if mo > len(s.out)+len(hist) || mo > s.windowSize { + if mo > len(out)+len(hist) || mo > s.windowSize { if len(s.dict) == 0 { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize) } // we may be in dictionary. - dictO := len(s.dict) - (mo - (len(s.out) + len(hist))) + dictO := len(s.dict) - (mo - (len(out) + len(hist))) if dictO < 0 || dictO >= len(s.dict) { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize) } end := dictO + ml if end > len(s.dict) { out = append(out, s.dict[dictO:]...) - mo -= len(s.dict) - dictO ml -= len(s.dict) - dictO } else { out = append(out, s.dict[dictO:end]...) @@ -231,26 +368,25 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { // Copy from history. // TODO: Blocks without history could be made to ignore this completely. - if v := mo - len(s.out); v > 0 { + if v := mo - len(out); v > 0 { // v is the start position in history from end. - start := len(s.hist) - v + start := len(hist) - v if ml > v { // Some goes into current block. // Copy remainder of history - out = append(out, s.hist[start:]...) - mo -= v + out = append(out, hist[start:]...) ml -= v } else { - out = append(out, s.hist[start:start+ml]...) + out = append(out, hist[start:start+ml]...) ml = 0 } } // We must be in current buffer now if ml > 0 { - start := len(s.out) - mo - if ml <= len(s.out)-start { + start := len(out) - mo + if ml <= len(out)-start { // No overlap - out = append(out, s.out[start:start+ml]...) + out = append(out, out[start:start+ml]...) } else { // Overlapping copy // Extend destination slice and copy one byte at the time. @@ -264,7 +400,6 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { } } } - s.out = out if i == 0 { // This is the last sequence, so we shouldn't update state. break @@ -278,7 +413,8 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { mlState = mlTable[mlState.newState()&maxTableMask] ofState = ofTable[ofState.newState()&maxTableMask] } else { - bits := br.getBitsFast(nBits) + bits := br.get32BitsFast(nBits) + lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31)) llState = llTable[(llState.newState()+lowBits)&maxTableMask] @@ -291,19 +427,14 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { } } - // Add final literals - s.out = append(s.out, s.literals...) - return nil -} + // Check if space for literals + if size := len(s.literals) + len(s.out) - startSize; size > maxBlockSize { + return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + } -// update states, at least 27 bits must be available. -func (s *sequenceDecs) update(br *bitReader) { - // Max 8 bits - s.litLengths.state.next(br) - // Max 9 bits - s.matchLengths.state.next(br) - // Max 8 bits - s.offsets.state.next(br) + // Add final literals + s.out = append(out, s.literals...) + return br.close() } var bitMask [16]uint16 @@ -314,87 +445,6 @@ func init() { } } -// update states, at least 27 bits must be available. -func (s *sequenceDecs) updateAlt(br *bitReader) { - // Update all 3 states at once. Approx 20% faster. - a, b, c := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state - - nBits := a.nbBits() + b.nbBits() + c.nbBits() - if nBits == 0 { - s.litLengths.state.state = s.litLengths.state.dt[a.newState()] - s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()] - s.offsets.state.state = s.offsets.state.dt[c.newState()] - return - } - bits := br.getBitsFast(nBits) - lowBits := uint16(bits >> ((c.nbBits() + b.nbBits()) & 31)) - s.litLengths.state.state = s.litLengths.state.dt[a.newState()+lowBits] - - lowBits = uint16(bits >> (c.nbBits() & 31)) - lowBits &= bitMask[b.nbBits()&15] - s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()+lowBits] - - lowBits = uint16(bits) & bitMask[c.nbBits()&15] - s.offsets.state.state = s.offsets.state.dt[c.newState()+lowBits] -} - -// nextFast will return new states when there are at least 4 unused bytes left on the stream when done. -func (s *sequenceDecs) nextFast(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) { - // Final will not read from stream. - ll, llB := llState.final() - ml, mlB := mlState.final() - mo, moB := ofState.final() - - // extra bits are stored in reverse order. - br.fillFast() - mo += br.getBits(moB) - if s.maxBits > 32 { - br.fillFast() - } - ml += br.getBits(mlB) - ll += br.getBits(llB) - - if moB > 1 { - s.prevOffset[2] = s.prevOffset[1] - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = mo - return - } - // mo = s.adjustOffset(mo, ll, moB) - // Inlined for rather big speedup - if ll == 0 { - // There is an exception though, when current sequence's literals_length = 0. - // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, - // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. - mo++ - } - - if mo == 0 { - mo = s.prevOffset[0] - return - } - var temp int - if mo == 3 { - temp = s.prevOffset[0] - 1 - } else { - temp = s.prevOffset[mo] - } - - if temp == 0 { - // 0 is not valid; input is corrupted; force offset to 1 - println("temp was 0") - temp = 1 - } - - if mo != 1 { - s.prevOffset[2] = s.prevOffset[1] - } - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = temp - mo = temp - return -} - func (s *sequenceDecs) next(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) { // Final will not read from stream. ll, llB := llState.final() @@ -457,36 +507,3 @@ func (s *sequenceDecs) adjustOffset(offset, litLen int, offsetB uint8) int { s.prevOffset[0] = temp return temp } - -// mergeHistory will merge history. -func (s *sequenceDecs) mergeHistory(hist *sequenceDecs) (*sequenceDecs, error) { - for i := uint(0); i < 3; i++ { - var sNew, sHist *sequenceDec - switch i { - default: - // same as "case 0": - sNew = &s.litLengths - sHist = &hist.litLengths - case 1: - sNew = &s.offsets - sHist = &hist.offsets - case 2: - sNew = &s.matchLengths - sHist = &hist.matchLengths - } - if sNew.repeat { - if sHist.fse == nil { - return nil, fmt.Errorf("sequence stream %d, repeat requested, but no history", i) - } - continue - } - if sNew.fse == nil { - return nil, fmt.Errorf("sequence stream %d, no fse found", i) - } - if sHist.fse != nil && !sHist.fse.preDefined { - fseDecoderPool.Put(sHist.fse) - } - sHist.fse = sNew.fse - } - return hist, nil -} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go new file mode 100644 index 000000000..191384adf --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -0,0 +1,379 @@ +//go:build amd64 && !appengine && !noasm && gc +// +build amd64,!appengine,!noasm,gc + +package zstd + +import ( + "fmt" + + "github.com/klauspost/compress/internal/cpuinfo" +) + +type decodeSyncAsmContext struct { + llTable []decSymbol + mlTable []decSymbol + ofTable []decSymbol + llState uint64 + mlState uint64 + ofState uint64 + iteration int + litRemain int + out []byte + outPosition int + literals []byte + litPosition int + history []byte + windowSize int + ll int // set on error (not for all errors, please refer to _generate/gen.go) + ml int // set on error (not for all errors, please refer to _generate/gen.go) + mo int // set on error (not for all errors, please refer to _generate/gen.go) +} + +// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. +// +// Please refer to seqdec_generic.go for the reference implementation. +// +//go:noescape +func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int + +// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. +// +//go:noescape +func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int + +// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. +// +//go:noescape +func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int + +// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. +// +//go:noescape +func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int + +// decode sequences from the stream with the provided history but without a dictionary. +func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { + if len(s.dict) > 0 { + return false, nil + } + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize { + return false, nil + } + + // FIXME: Using unsafe memory copies leads to rare, random crashes + // with fuzz testing. It is therefore disabled for now. + const useSafe = true + /* + useSafe := false + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { + useSafe = true + } + if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { + useSafe = true + } + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + useSafe = true + } + */ + + br := s.br + + maxBlockSize := maxCompressedBlockSize + if s.windowSize < maxBlockSize { + maxBlockSize = s.windowSize + } + + ctx := decodeSyncAsmContext{ + llTable: s.litLengths.fse.dt[:maxTablesize], + mlTable: s.matchLengths.fse.dt[:maxTablesize], + ofTable: s.offsets.fse.dt[:maxTablesize], + llState: uint64(s.litLengths.state.state), + mlState: uint64(s.matchLengths.state.state), + ofState: uint64(s.offsets.state.state), + iteration: s.nSeqs - 1, + litRemain: len(s.literals), + out: s.out, + outPosition: len(s.out), + literals: s.literals, + windowSize: s.windowSize, + history: hist, + } + + s.seqSize = 0 + startSize := len(s.out) + + var errCode int + if cpuinfo.HasBMI2() { + if useSafe { + errCode = sequenceDecs_decodeSync_safe_bmi2(s, br, &ctx) + } else { + errCode = sequenceDecs_decodeSync_bmi2(s, br, &ctx) + } + } else { + if useSafe { + errCode = sequenceDecs_decodeSync_safe_amd64(s, br, &ctx) + } else { + errCode = sequenceDecs_decodeSync_amd64(s, br, &ctx) + } + } + switch errCode { + case noError: + break + + case errorMatchLenOfsMismatch: + return true, fmt.Errorf("zero matchoff and matchlen (%d) > 0", ctx.ml) + + case errorMatchLenTooBig: + return true, fmt.Errorf("match len (%d) bigger than max allowed length", ctx.ml) + + case errorMatchOffTooBig: + return true, fmt.Errorf("match offset (%d) bigger than current history (%d)", + ctx.mo, ctx.outPosition+len(hist)-startSize) + + case errorNotEnoughLiterals: + return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", + ctx.ll, ctx.litRemain+ctx.ll) + + case errorNotEnoughSpace: + size := ctx.outPosition + ctx.ll + ctx.ml + if debugDecoder { + println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize) + } + return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + + default: + return true, fmt.Errorf("sequenceDecs_decode returned erronous code %d", errCode) + } + + s.seqSize += ctx.litRemain + if s.seqSize > maxBlockSize { + return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + + } + err := br.close() + if err != nil { + printf("Closing sequences: %v, %+v\n", err, *br) + return true, err + } + + s.literals = s.literals[ctx.litPosition:] + t := ctx.outPosition + s.out = s.out[:t] + + // Add final literals + s.out = append(s.out, s.literals...) + if debugDecoder { + t += len(s.literals) + if t != len(s.out) { + panic(fmt.Errorf("length mismatch, want %d, got %d", len(s.out), t)) + } + } + + return true, nil +} + +// -------------------------------------------------------------------------------- + +type decodeAsmContext struct { + llTable []decSymbol + mlTable []decSymbol + ofTable []decSymbol + llState uint64 + mlState uint64 + ofState uint64 + iteration int + seqs []seqVals + litRemain int +} + +const noError = 0 + +// error reported when mo == 0 && ml > 0 +const errorMatchLenOfsMismatch = 1 + +// error reported when ml > maxMatchLen +const errorMatchLenTooBig = 2 + +// error reported when mo > available history or mo > s.windowSize +const errorMatchOffTooBig = 3 + +// error reported when the sum of literal lengths exeeceds the literal buffer size +const errorNotEnoughLiterals = 4 + +// error reported when capacity of `out` is too small +const errorNotEnoughSpace = 5 + +// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. +// +// Please refer to seqdec_generic.go for the reference implementation. +// +//go:noescape +func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int + +// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. +// +// Please refer to seqdec_generic.go for the reference implementation. +// +//go:noescape +func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int + +// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// +//go:noescape +func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int + +// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// +//go:noescape +func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int + +// decode sequences from the stream without the provided history. +func (s *sequenceDecs) decode(seqs []seqVals) error { + br := s.br + + maxBlockSize := maxCompressedBlockSize + if s.windowSize < maxBlockSize { + maxBlockSize = s.windowSize + } + + ctx := decodeAsmContext{ + llTable: s.litLengths.fse.dt[:maxTablesize], + mlTable: s.matchLengths.fse.dt[:maxTablesize], + ofTable: s.offsets.fse.dt[:maxTablesize], + llState: uint64(s.litLengths.state.state), + mlState: uint64(s.matchLengths.state.state), + ofState: uint64(s.offsets.state.state), + seqs: seqs, + iteration: len(seqs) - 1, + litRemain: len(s.literals), + } + + s.seqSize = 0 + lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56 + var errCode int + if cpuinfo.HasBMI2() { + if lte56bits { + errCode = sequenceDecs_decode_56_bmi2(s, br, &ctx) + } else { + errCode = sequenceDecs_decode_bmi2(s, br, &ctx) + } + } else { + if lte56bits { + errCode = sequenceDecs_decode_56_amd64(s, br, &ctx) + } else { + errCode = sequenceDecs_decode_amd64(s, br, &ctx) + } + } + if errCode != 0 { + i := len(seqs) - ctx.iteration - 1 + switch errCode { + case errorMatchLenOfsMismatch: + ml := ctx.seqs[i].ml + return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) + + case errorMatchLenTooBig: + ml := ctx.seqs[i].ml + return fmt.Errorf("match len (%d) bigger than max allowed length", ml) + + case errorNotEnoughLiterals: + ll := ctx.seqs[i].ll + return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll) + } + + return fmt.Errorf("sequenceDecs_decode_amd64 returned erronous code %d", errCode) + } + + if ctx.litRemain < 0 { + return fmt.Errorf("literal count is too big: total available %d, total requested %d", + len(s.literals), len(s.literals)-ctx.litRemain) + } + + s.seqSize += ctx.litRemain + if s.seqSize > maxBlockSize { + return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + } + err := br.close() + if err != nil { + printf("Closing sequences: %v, %+v\n", err, *br) + } + return err +} + +// -------------------------------------------------------------------------------- + +type executeAsmContext struct { + seqs []seqVals + seqIndex int + out []byte + history []byte + literals []byte + outPosition int + litPosition int + windowSize int +} + +// sequenceDecs_executeSimple_amd64 implements the main loop of sequenceDecs.executeSimple in x86 asm. +// +// Returns false if a match offset is too big. +// +// Please refer to seqdec_generic.go for the reference implementation. +// +//go:noescape +func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool + +// Same as above, but with safe memcopies +// +//go:noescape +func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool + +// executeSimple handles cases when dictionary is not used. +func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { + // Ensure we have enough output size... + if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) { + addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc + s.out = append(s.out, make([]byte, addBytes)...) + s.out = s.out[:len(s.out)-addBytes] + } + + if debugDecoder { + printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize) + } + + var t = len(s.out) + out := s.out[:t+s.seqSize] + + ctx := executeAsmContext{ + seqs: seqs, + seqIndex: 0, + out: out, + history: hist, + outPosition: t, + litPosition: 0, + literals: s.literals, + windowSize: s.windowSize, + } + var ok bool + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + ok = sequenceDecs_executeSimple_safe_amd64(&ctx) + } else { + ok = sequenceDecs_executeSimple_amd64(&ctx) + } + if !ok { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", + seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist)) + } + s.literals = s.literals[ctx.litPosition:] + t = ctx.outPosition + + // Add final literals + copy(out[t:], s.literals) + if debugDecoder { + t += len(s.literals) + if t != len(out) { + panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) + } + } + s.out = out + + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s new file mode 100644 index 000000000..b94993a07 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -0,0 +1,4079 @@ +// Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT. + +//go:build !appengine && !noasm && gc && !noasm + +// func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +// Requires: CMOV +TEXT ·sequenceDecs_decode_amd64(SB), $8-32 + MOVQ br+8(FP), AX + MOVQ 32(AX), DX + MOVBQZX 40(AX), BX + MOVQ 24(AX), SI + MOVQ (AX), AX + ADDQ SI, AX + MOVQ AX, (SP) + MOVQ ctx+16(FP), AX + MOVQ 72(AX), DI + MOVQ 80(AX), R8 + MOVQ 88(AX), R9 + MOVQ 104(AX), R10 + MOVQ s+0(FP), AX + MOVQ 144(AX), R11 + MOVQ 152(AX), R12 + MOVQ 160(AX), R13 + +sequenceDecs_decode_amd64_main_loop: + MOVQ (SP), R14 + + // Fill bitreader to have enough for the offset and match length. + CMPQ SI, $0x08 + JL sequenceDecs_decode_amd64_fill_byte_by_byte + MOVQ BX, AX + SHRQ $0x03, AX + SUBQ AX, R14 + MOVQ (R14), DX + SUBQ AX, SI + ANDQ $0x07, BX + JMP sequenceDecs_decode_amd64_fill_end + +sequenceDecs_decode_amd64_fill_byte_by_byte: + CMPQ SI, $0x00 + JLE sequenceDecs_decode_amd64_fill_end + CMPQ BX, $0x07 + JLE sequenceDecs_decode_amd64_fill_end + SHLQ $0x08, DX + SUBQ $0x01, R14 + SUBQ $0x01, SI + SUBQ $0x08, BX + MOVBQZX (R14), AX + ORQ AX, DX + JMP sequenceDecs_decode_amd64_fill_byte_by_byte + +sequenceDecs_decode_amd64_fill_end: + // Update offset + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_amd64_of_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX + +sequenceDecs_decode_amd64_of_update_zero: + MOVQ AX, 16(R10) + + // Update match length + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX + +sequenceDecs_decode_amd64_ml_update_zero: + MOVQ AX, 8(R10) + + // Fill bitreader to have enough for the remaining + CMPQ SI, $0x08 + JL sequenceDecs_decode_amd64_fill_2_byte_by_byte + MOVQ BX, AX + SHRQ $0x03, AX + SUBQ AX, R14 + MOVQ (R14), DX + SUBQ AX, SI + ANDQ $0x07, BX + JMP sequenceDecs_decode_amd64_fill_2_end + +sequenceDecs_decode_amd64_fill_2_byte_by_byte: + CMPQ SI, $0x00 + JLE sequenceDecs_decode_amd64_fill_2_end + CMPQ BX, $0x07 + JLE sequenceDecs_decode_amd64_fill_2_end + SHLQ $0x08, DX + SUBQ $0x01, R14 + SUBQ $0x01, SI + SUBQ $0x08, BX + MOVBQZX (R14), AX + ORQ AX, DX + JMP sequenceDecs_decode_amd64_fill_2_byte_by_byte + +sequenceDecs_decode_amd64_fill_2_end: + // Update literal length + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX + +sequenceDecs_decode_amd64_ll_update_zero: + MOVQ AX, (R10) + + // Fill bitreader for state updates + MOVQ R14, (SP) + MOVQ R9, AX + SHRQ $0x08, AX + MOVBQZX AL, AX + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decode_amd64_skip_update + + // Update Literal Length State + MOVBQZX DI, R14 + SHRQ $0x10, DI + MOVWQZX DI, DI + LEAQ (BX)(R14*1), CX + MOVQ DX, R15 + MOVQ CX, BX + ROLQ CL, R15 + MOVL $0x00000001, BP + MOVB R14, CL + SHLL CL, BP + DECL BP + ANDQ BP, R15 + ADDQ R15, DI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(DI*8), DI + + // Update Match Length State + MOVBQZX R8, R14 + SHRQ $0x10, R8 + MOVWQZX R8, R8 + LEAQ (BX)(R14*1), CX + MOVQ DX, R15 + MOVQ CX, BX + ROLQ CL, R15 + MOVL $0x00000001, BP + MOVB R14, CL + SHLL CL, BP + DECL BP + ANDQ BP, R15 + ADDQ R15, R8 + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Offset State + MOVBQZX R9, R14 + SHRQ $0x10, R9 + MOVWQZX R9, R9 + LEAQ (BX)(R14*1), CX + MOVQ DX, R15 + MOVQ CX, BX + ROLQ CL, R15 + MOVL $0x00000001, BP + MOVB R14, CL + SHLL CL, BP + DECL BP + ANDQ BP, R15 + ADDQ R15, R9 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R9*8), R9 + +sequenceDecs_decode_amd64_skip_update: + // Adjust offset + MOVQ 16(R10), CX + CMPQ AX, $0x01 + JBE sequenceDecs_decode_amd64_adjust_offsetB_1_or_0 + MOVQ R12, R13 + MOVQ R11, R12 + MOVQ CX, R11 + JMP sequenceDecs_decode_amd64_after_adjust + +sequenceDecs_decode_amd64_adjust_offsetB_1_or_0: + CMPQ (R10), $0x00000000 + JNE sequenceDecs_decode_amd64_adjust_offset_maybezero + INCQ CX + JMP sequenceDecs_decode_amd64_adjust_offset_nonzero + +sequenceDecs_decode_amd64_adjust_offset_maybezero: + TESTQ CX, CX + JNZ sequenceDecs_decode_amd64_adjust_offset_nonzero + MOVQ R11, CX + JMP sequenceDecs_decode_amd64_after_adjust + +sequenceDecs_decode_amd64_adjust_offset_nonzero: + CMPQ CX, $0x01 + JB sequenceDecs_decode_amd64_adjust_zero + JEQ sequenceDecs_decode_amd64_adjust_one + CMPQ CX, $0x02 + JA sequenceDecs_decode_amd64_adjust_three + JMP sequenceDecs_decode_amd64_adjust_two + +sequenceDecs_decode_amd64_adjust_zero: + MOVQ R11, AX + JMP sequenceDecs_decode_amd64_adjust_test_temp_valid + +sequenceDecs_decode_amd64_adjust_one: + MOVQ R12, AX + JMP sequenceDecs_decode_amd64_adjust_test_temp_valid + +sequenceDecs_decode_amd64_adjust_two: + MOVQ R13, AX + JMP sequenceDecs_decode_amd64_adjust_test_temp_valid + +sequenceDecs_decode_amd64_adjust_three: + LEAQ -1(R11), AX + +sequenceDecs_decode_amd64_adjust_test_temp_valid: + TESTQ AX, AX + JNZ sequenceDecs_decode_amd64_adjust_temp_valid + MOVQ $0x00000001, AX + +sequenceDecs_decode_amd64_adjust_temp_valid: + CMPQ CX, $0x01 + CMOVQNE R12, R13 + MOVQ R11, R12 + MOVQ AX, R11 + MOVQ AX, CX + +sequenceDecs_decode_amd64_after_adjust: + MOVQ CX, 16(R10) + + // Check values + MOVQ 8(R10), AX + MOVQ (R10), R14 + LEAQ (AX)(R14*1), R15 + MOVQ s+0(FP), BP + ADDQ R15, 256(BP) + MOVQ ctx+16(FP), R15 + SUBQ R14, 128(R15) + JS error_not_enough_literals + CMPQ AX, $0x00020002 + JA sequenceDecs_decode_amd64_error_match_len_too_big + TESTQ CX, CX + JNZ sequenceDecs_decode_amd64_match_len_ofs_ok + TESTQ AX, AX + JNZ sequenceDecs_decode_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decode_amd64_match_len_ofs_ok: + ADDQ $0x18, R10 + MOVQ ctx+16(FP), AX + DECQ 96(AX) + JNS sequenceDecs_decode_amd64_main_loop + MOVQ s+0(FP), AX + MOVQ R11, 144(AX) + MOVQ R12, 152(AX) + MOVQ R13, 160(AX) + MOVQ br+8(FP), AX + MOVQ DX, 32(AX) + MOVB BL, 40(AX) + MOVQ SI, 24(AX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decode_amd64_error_match_len_too_big: + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ $0x00000004, ret+24(FP) + RET + +// func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +// Requires: CMOV +TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 + MOVQ br+8(FP), AX + MOVQ 32(AX), DX + MOVBQZX 40(AX), BX + MOVQ 24(AX), SI + MOVQ (AX), AX + ADDQ SI, AX + MOVQ AX, (SP) + MOVQ ctx+16(FP), AX + MOVQ 72(AX), DI + MOVQ 80(AX), R8 + MOVQ 88(AX), R9 + MOVQ 104(AX), R10 + MOVQ s+0(FP), AX + MOVQ 144(AX), R11 + MOVQ 152(AX), R12 + MOVQ 160(AX), R13 + +sequenceDecs_decode_56_amd64_main_loop: + MOVQ (SP), R14 + + // Fill bitreader to have enough for the offset and match length. + CMPQ SI, $0x08 + JL sequenceDecs_decode_56_amd64_fill_byte_by_byte + MOVQ BX, AX + SHRQ $0x03, AX + SUBQ AX, R14 + MOVQ (R14), DX + SUBQ AX, SI + ANDQ $0x07, BX + JMP sequenceDecs_decode_56_amd64_fill_end + +sequenceDecs_decode_56_amd64_fill_byte_by_byte: + CMPQ SI, $0x00 + JLE sequenceDecs_decode_56_amd64_fill_end + CMPQ BX, $0x07 + JLE sequenceDecs_decode_56_amd64_fill_end + SHLQ $0x08, DX + SUBQ $0x01, R14 + SUBQ $0x01, SI + SUBQ $0x08, BX + MOVBQZX (R14), AX + ORQ AX, DX + JMP sequenceDecs_decode_56_amd64_fill_byte_by_byte + +sequenceDecs_decode_56_amd64_fill_end: + // Update offset + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_56_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_56_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_56_amd64_of_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX + +sequenceDecs_decode_56_amd64_of_update_zero: + MOVQ AX, 16(R10) + + // Update match length + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_56_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_56_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_56_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX + +sequenceDecs_decode_56_amd64_ml_update_zero: + MOVQ AX, 8(R10) + + // Update literal length + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decode_56_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decode_56_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decode_56_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R15 + ADDQ R15, AX + +sequenceDecs_decode_56_amd64_ll_update_zero: + MOVQ AX, (R10) + + // Fill bitreader for state updates + MOVQ R14, (SP) + MOVQ R9, AX + SHRQ $0x08, AX + MOVBQZX AL, AX + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decode_56_amd64_skip_update + + // Update Literal Length State + MOVBQZX DI, R14 + SHRQ $0x10, DI + MOVWQZX DI, DI + LEAQ (BX)(R14*1), CX + MOVQ DX, R15 + MOVQ CX, BX + ROLQ CL, R15 + MOVL $0x00000001, BP + MOVB R14, CL + SHLL CL, BP + DECL BP + ANDQ BP, R15 + ADDQ R15, DI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(DI*8), DI + + // Update Match Length State + MOVBQZX R8, R14 + SHRQ $0x10, R8 + MOVWQZX R8, R8 + LEAQ (BX)(R14*1), CX + MOVQ DX, R15 + MOVQ CX, BX + ROLQ CL, R15 + MOVL $0x00000001, BP + MOVB R14, CL + SHLL CL, BP + DECL BP + ANDQ BP, R15 + ADDQ R15, R8 + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Offset State + MOVBQZX R9, R14 + SHRQ $0x10, R9 + MOVWQZX R9, R9 + LEAQ (BX)(R14*1), CX + MOVQ DX, R15 + MOVQ CX, BX + ROLQ CL, R15 + MOVL $0x00000001, BP + MOVB R14, CL + SHLL CL, BP + DECL BP + ANDQ BP, R15 + ADDQ R15, R9 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R9*8), R9 + +sequenceDecs_decode_56_amd64_skip_update: + // Adjust offset + MOVQ 16(R10), CX + CMPQ AX, $0x01 + JBE sequenceDecs_decode_56_amd64_adjust_offsetB_1_or_0 + MOVQ R12, R13 + MOVQ R11, R12 + MOVQ CX, R11 + JMP sequenceDecs_decode_56_amd64_after_adjust + +sequenceDecs_decode_56_amd64_adjust_offsetB_1_or_0: + CMPQ (R10), $0x00000000 + JNE sequenceDecs_decode_56_amd64_adjust_offset_maybezero + INCQ CX + JMP sequenceDecs_decode_56_amd64_adjust_offset_nonzero + +sequenceDecs_decode_56_amd64_adjust_offset_maybezero: + TESTQ CX, CX + JNZ sequenceDecs_decode_56_amd64_adjust_offset_nonzero + MOVQ R11, CX + JMP sequenceDecs_decode_56_amd64_after_adjust + +sequenceDecs_decode_56_amd64_adjust_offset_nonzero: + CMPQ CX, $0x01 + JB sequenceDecs_decode_56_amd64_adjust_zero + JEQ sequenceDecs_decode_56_amd64_adjust_one + CMPQ CX, $0x02 + JA sequenceDecs_decode_56_amd64_adjust_three + JMP sequenceDecs_decode_56_amd64_adjust_two + +sequenceDecs_decode_56_amd64_adjust_zero: + MOVQ R11, AX + JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid + +sequenceDecs_decode_56_amd64_adjust_one: + MOVQ R12, AX + JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid + +sequenceDecs_decode_56_amd64_adjust_two: + MOVQ R13, AX + JMP sequenceDecs_decode_56_amd64_adjust_test_temp_valid + +sequenceDecs_decode_56_amd64_adjust_three: + LEAQ -1(R11), AX + +sequenceDecs_decode_56_amd64_adjust_test_temp_valid: + TESTQ AX, AX + JNZ sequenceDecs_decode_56_amd64_adjust_temp_valid + MOVQ $0x00000001, AX + +sequenceDecs_decode_56_amd64_adjust_temp_valid: + CMPQ CX, $0x01 + CMOVQNE R12, R13 + MOVQ R11, R12 + MOVQ AX, R11 + MOVQ AX, CX + +sequenceDecs_decode_56_amd64_after_adjust: + MOVQ CX, 16(R10) + + // Check values + MOVQ 8(R10), AX + MOVQ (R10), R14 + LEAQ (AX)(R14*1), R15 + MOVQ s+0(FP), BP + ADDQ R15, 256(BP) + MOVQ ctx+16(FP), R15 + SUBQ R14, 128(R15) + JS error_not_enough_literals + CMPQ AX, $0x00020002 + JA sequenceDecs_decode_56_amd64_error_match_len_too_big + TESTQ CX, CX + JNZ sequenceDecs_decode_56_amd64_match_len_ofs_ok + TESTQ AX, AX + JNZ sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decode_56_amd64_match_len_ofs_ok: + ADDQ $0x18, R10 + MOVQ ctx+16(FP), AX + DECQ 96(AX) + JNS sequenceDecs_decode_56_amd64_main_loop + MOVQ s+0(FP), AX + MOVQ R11, 144(AX) + MOVQ R12, 152(AX) + MOVQ R13, 160(AX) + MOVQ br+8(FP), AX + MOVQ DX, 32(AX) + MOVB BL, 40(AX) + MOVQ SI, 24(AX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ $0x00000004, ret+24(FP) + RET + +// func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +// Requires: BMI, BMI2, CMOV +TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 + MOVQ br+8(FP), CX + MOVQ 32(CX), AX + MOVBQZX 40(CX), DX + MOVQ 24(CX), BX + MOVQ (CX), CX + ADDQ BX, CX + MOVQ CX, (SP) + MOVQ ctx+16(FP), CX + MOVQ 72(CX), SI + MOVQ 80(CX), DI + MOVQ 88(CX), R8 + MOVQ 104(CX), R9 + MOVQ s+0(FP), CX + MOVQ 144(CX), R10 + MOVQ 152(CX), R11 + MOVQ 160(CX), R12 + +sequenceDecs_decode_bmi2_main_loop: + MOVQ (SP), R13 + + // Fill bitreader to have enough for the offset and match length. + CMPQ BX, $0x08 + JL sequenceDecs_decode_bmi2_fill_byte_by_byte + MOVQ DX, CX + SHRQ $0x03, CX + SUBQ CX, R13 + MOVQ (R13), AX + SUBQ CX, BX + ANDQ $0x07, DX + JMP sequenceDecs_decode_bmi2_fill_end + +sequenceDecs_decode_bmi2_fill_byte_by_byte: + CMPQ BX, $0x00 + JLE sequenceDecs_decode_bmi2_fill_end + CMPQ DX, $0x07 + JLE sequenceDecs_decode_bmi2_fill_end + SHLQ $0x08, AX + SUBQ $0x01, R13 + SUBQ $0x01, BX + SUBQ $0x08, DX + MOVBQZX (R13), CX + ORQ CX, AX + JMP sequenceDecs_decode_bmi2_fill_byte_by_byte + +sequenceDecs_decode_bmi2_fill_end: + // Update offset + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R14 + MOVQ AX, R15 + LEAQ (DX)(R14*1), CX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + MOVQ CX, DX + MOVQ R8, CX + SHRQ $0x20, CX + ADDQ R15, CX + MOVQ CX, 16(R9) + + // Update match length + MOVQ $0x00000808, CX + BEXTRQ CX, DI, R14 + MOVQ AX, R15 + LEAQ (DX)(R14*1), CX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + MOVQ CX, DX + MOVQ DI, CX + SHRQ $0x20, CX + ADDQ R15, CX + MOVQ CX, 8(R9) + + // Fill bitreader to have enough for the remaining + CMPQ BX, $0x08 + JL sequenceDecs_decode_bmi2_fill_2_byte_by_byte + MOVQ DX, CX + SHRQ $0x03, CX + SUBQ CX, R13 + MOVQ (R13), AX + SUBQ CX, BX + ANDQ $0x07, DX + JMP sequenceDecs_decode_bmi2_fill_2_end + +sequenceDecs_decode_bmi2_fill_2_byte_by_byte: + CMPQ BX, $0x00 + JLE sequenceDecs_decode_bmi2_fill_2_end + CMPQ DX, $0x07 + JLE sequenceDecs_decode_bmi2_fill_2_end + SHLQ $0x08, AX + SUBQ $0x01, R13 + SUBQ $0x01, BX + SUBQ $0x08, DX + MOVBQZX (R13), CX + ORQ CX, AX + JMP sequenceDecs_decode_bmi2_fill_2_byte_by_byte + +sequenceDecs_decode_bmi2_fill_2_end: + // Update literal length + MOVQ $0x00000808, CX + BEXTRQ CX, SI, R14 + MOVQ AX, R15 + LEAQ (DX)(R14*1), CX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + MOVQ CX, DX + MOVQ SI, CX + SHRQ $0x20, CX + ADDQ R15, CX + MOVQ CX, (R9) + + // Fill bitreader for state updates + MOVQ R13, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R13 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decode_bmi2_skip_update + LEAQ (SI)(DI*1), R14 + ADDQ R8, R14 + MOVBQZX R14, R14 + LEAQ (DX)(R14*1), CX + MOVQ AX, R15 + MOVQ CX, DX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + + // Update Offset State + BZHIQ R8, R15, CX + SHRXQ R8, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Match Length State + BZHIQ DI, R15, CX + SHRXQ DI, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, DI, DI + ADDQ CX, DI + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(DI*8), DI + + // Update Literal Length State + BZHIQ SI, R15, CX + MOVQ $0x00001010, R14 + BEXTRQ R14, SI, SI + ADDQ CX, SI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI + +sequenceDecs_decode_bmi2_skip_update: + // Adjust offset + MOVQ 16(R9), CX + CMPQ R13, $0x01 + JBE sequenceDecs_decode_bmi2_adjust_offsetB_1_or_0 + MOVQ R11, R12 + MOVQ R10, R11 + MOVQ CX, R10 + JMP sequenceDecs_decode_bmi2_after_adjust + +sequenceDecs_decode_bmi2_adjust_offsetB_1_or_0: + CMPQ (R9), $0x00000000 + JNE sequenceDecs_decode_bmi2_adjust_offset_maybezero + INCQ CX + JMP sequenceDecs_decode_bmi2_adjust_offset_nonzero + +sequenceDecs_decode_bmi2_adjust_offset_maybezero: + TESTQ CX, CX + JNZ sequenceDecs_decode_bmi2_adjust_offset_nonzero + MOVQ R10, CX + JMP sequenceDecs_decode_bmi2_after_adjust + +sequenceDecs_decode_bmi2_adjust_offset_nonzero: + CMPQ CX, $0x01 + JB sequenceDecs_decode_bmi2_adjust_zero + JEQ sequenceDecs_decode_bmi2_adjust_one + CMPQ CX, $0x02 + JA sequenceDecs_decode_bmi2_adjust_three + JMP sequenceDecs_decode_bmi2_adjust_two + +sequenceDecs_decode_bmi2_adjust_zero: + MOVQ R10, R13 + JMP sequenceDecs_decode_bmi2_adjust_test_temp_valid + +sequenceDecs_decode_bmi2_adjust_one: + MOVQ R11, R13 + JMP sequenceDecs_decode_bmi2_adjust_test_temp_valid + +sequenceDecs_decode_bmi2_adjust_two: + MOVQ R12, R13 + JMP sequenceDecs_decode_bmi2_adjust_test_temp_valid + +sequenceDecs_decode_bmi2_adjust_three: + LEAQ -1(R10), R13 + +sequenceDecs_decode_bmi2_adjust_test_temp_valid: + TESTQ R13, R13 + JNZ sequenceDecs_decode_bmi2_adjust_temp_valid + MOVQ $0x00000001, R13 + +sequenceDecs_decode_bmi2_adjust_temp_valid: + CMPQ CX, $0x01 + CMOVQNE R11, R12 + MOVQ R10, R11 + MOVQ R13, R10 + MOVQ R13, CX + +sequenceDecs_decode_bmi2_after_adjust: + MOVQ CX, 16(R9) + + // Check values + MOVQ 8(R9), R13 + MOVQ (R9), R14 + LEAQ (R13)(R14*1), R15 + MOVQ s+0(FP), BP + ADDQ R15, 256(BP) + MOVQ ctx+16(FP), R15 + SUBQ R14, 128(R15) + JS error_not_enough_literals + CMPQ R13, $0x00020002 + JA sequenceDecs_decode_bmi2_error_match_len_too_big + TESTQ CX, CX + JNZ sequenceDecs_decode_bmi2_match_len_ofs_ok + TESTQ R13, R13 + JNZ sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch + +sequenceDecs_decode_bmi2_match_len_ofs_ok: + ADDQ $0x18, R9 + MOVQ ctx+16(FP), CX + DECQ 96(CX) + JNS sequenceDecs_decode_bmi2_main_loop + MOVQ s+0(FP), CX + MOVQ R10, 144(CX) + MOVQ R11, 152(CX) + MOVQ R12, 160(CX) + MOVQ br+8(FP), CX + MOVQ AX, 32(CX) + MOVB DL, 40(CX) + MOVQ BX, 24(CX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch: + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decode_bmi2_error_match_len_too_big: + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ $0x00000004, ret+24(FP) + RET + +// func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +// Requires: BMI, BMI2, CMOV +TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 + MOVQ br+8(FP), CX + MOVQ 32(CX), AX + MOVBQZX 40(CX), DX + MOVQ 24(CX), BX + MOVQ (CX), CX + ADDQ BX, CX + MOVQ CX, (SP) + MOVQ ctx+16(FP), CX + MOVQ 72(CX), SI + MOVQ 80(CX), DI + MOVQ 88(CX), R8 + MOVQ 104(CX), R9 + MOVQ s+0(FP), CX + MOVQ 144(CX), R10 + MOVQ 152(CX), R11 + MOVQ 160(CX), R12 + +sequenceDecs_decode_56_bmi2_main_loop: + MOVQ (SP), R13 + + // Fill bitreader to have enough for the offset and match length. + CMPQ BX, $0x08 + JL sequenceDecs_decode_56_bmi2_fill_byte_by_byte + MOVQ DX, CX + SHRQ $0x03, CX + SUBQ CX, R13 + MOVQ (R13), AX + SUBQ CX, BX + ANDQ $0x07, DX + JMP sequenceDecs_decode_56_bmi2_fill_end + +sequenceDecs_decode_56_bmi2_fill_byte_by_byte: + CMPQ BX, $0x00 + JLE sequenceDecs_decode_56_bmi2_fill_end + CMPQ DX, $0x07 + JLE sequenceDecs_decode_56_bmi2_fill_end + SHLQ $0x08, AX + SUBQ $0x01, R13 + SUBQ $0x01, BX + SUBQ $0x08, DX + MOVBQZX (R13), CX + ORQ CX, AX + JMP sequenceDecs_decode_56_bmi2_fill_byte_by_byte + +sequenceDecs_decode_56_bmi2_fill_end: + // Update offset + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R14 + MOVQ AX, R15 + LEAQ (DX)(R14*1), CX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + MOVQ CX, DX + MOVQ R8, CX + SHRQ $0x20, CX + ADDQ R15, CX + MOVQ CX, 16(R9) + + // Update match length + MOVQ $0x00000808, CX + BEXTRQ CX, DI, R14 + MOVQ AX, R15 + LEAQ (DX)(R14*1), CX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + MOVQ CX, DX + MOVQ DI, CX + SHRQ $0x20, CX + ADDQ R15, CX + MOVQ CX, 8(R9) + + // Update literal length + MOVQ $0x00000808, CX + BEXTRQ CX, SI, R14 + MOVQ AX, R15 + LEAQ (DX)(R14*1), CX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + MOVQ CX, DX + MOVQ SI, CX + SHRQ $0x20, CX + ADDQ R15, CX + MOVQ CX, (R9) + + // Fill bitreader for state updates + MOVQ R13, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R13 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decode_56_bmi2_skip_update + LEAQ (SI)(DI*1), R14 + ADDQ R8, R14 + MOVBQZX R14, R14 + LEAQ (DX)(R14*1), CX + MOVQ AX, R15 + MOVQ CX, DX + ROLQ CL, R15 + BZHIQ R14, R15, R15 + + // Update Offset State + BZHIQ R8, R15, CX + SHRXQ R8, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Match Length State + BZHIQ DI, R15, CX + SHRXQ DI, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, DI, DI + ADDQ CX, DI + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(DI*8), DI + + // Update Literal Length State + BZHIQ SI, R15, CX + MOVQ $0x00001010, R14 + BEXTRQ R14, SI, SI + ADDQ CX, SI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI + +sequenceDecs_decode_56_bmi2_skip_update: + // Adjust offset + MOVQ 16(R9), CX + CMPQ R13, $0x01 + JBE sequenceDecs_decode_56_bmi2_adjust_offsetB_1_or_0 + MOVQ R11, R12 + MOVQ R10, R11 + MOVQ CX, R10 + JMP sequenceDecs_decode_56_bmi2_after_adjust + +sequenceDecs_decode_56_bmi2_adjust_offsetB_1_or_0: + CMPQ (R9), $0x00000000 + JNE sequenceDecs_decode_56_bmi2_adjust_offset_maybezero + INCQ CX + JMP sequenceDecs_decode_56_bmi2_adjust_offset_nonzero + +sequenceDecs_decode_56_bmi2_adjust_offset_maybezero: + TESTQ CX, CX + JNZ sequenceDecs_decode_56_bmi2_adjust_offset_nonzero + MOVQ R10, CX + JMP sequenceDecs_decode_56_bmi2_after_adjust + +sequenceDecs_decode_56_bmi2_adjust_offset_nonzero: + CMPQ CX, $0x01 + JB sequenceDecs_decode_56_bmi2_adjust_zero + JEQ sequenceDecs_decode_56_bmi2_adjust_one + CMPQ CX, $0x02 + JA sequenceDecs_decode_56_bmi2_adjust_three + JMP sequenceDecs_decode_56_bmi2_adjust_two + +sequenceDecs_decode_56_bmi2_adjust_zero: + MOVQ R10, R13 + JMP sequenceDecs_decode_56_bmi2_adjust_test_temp_valid + +sequenceDecs_decode_56_bmi2_adjust_one: + MOVQ R11, R13 + JMP sequenceDecs_decode_56_bmi2_adjust_test_temp_valid + +sequenceDecs_decode_56_bmi2_adjust_two: + MOVQ R12, R13 + JMP sequenceDecs_decode_56_bmi2_adjust_test_temp_valid + +sequenceDecs_decode_56_bmi2_adjust_three: + LEAQ -1(R10), R13 + +sequenceDecs_decode_56_bmi2_adjust_test_temp_valid: + TESTQ R13, R13 + JNZ sequenceDecs_decode_56_bmi2_adjust_temp_valid + MOVQ $0x00000001, R13 + +sequenceDecs_decode_56_bmi2_adjust_temp_valid: + CMPQ CX, $0x01 + CMOVQNE R11, R12 + MOVQ R10, R11 + MOVQ R13, R10 + MOVQ R13, CX + +sequenceDecs_decode_56_bmi2_after_adjust: + MOVQ CX, 16(R9) + + // Check values + MOVQ 8(R9), R13 + MOVQ (R9), R14 + LEAQ (R13)(R14*1), R15 + MOVQ s+0(FP), BP + ADDQ R15, 256(BP) + MOVQ ctx+16(FP), R15 + SUBQ R14, 128(R15) + JS error_not_enough_literals + CMPQ R13, $0x00020002 + JA sequenceDecs_decode_56_bmi2_error_match_len_too_big + TESTQ CX, CX + JNZ sequenceDecs_decode_56_bmi2_match_len_ofs_ok + TESTQ R13, R13 + JNZ sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch + +sequenceDecs_decode_56_bmi2_match_len_ofs_ok: + ADDQ $0x18, R9 + MOVQ ctx+16(FP), CX + DECQ 96(CX) + JNS sequenceDecs_decode_56_bmi2_main_loop + MOVQ s+0(FP), CX + MOVQ R10, 144(CX) + MOVQ R11, 152(CX) + MOVQ R12, 160(CX) + MOVQ br+8(FP), CX + MOVQ AX, 32(CX) + MOVB DL, 40(CX) + MOVQ BX, 24(CX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch: + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decode_56_bmi2_error_match_len_too_big: + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ $0x00000004, ret+24(FP) + RET + +// func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool +// Requires: SSE +TEXT ·sequenceDecs_executeSimple_amd64(SB), $8-9 + MOVQ ctx+0(FP), R10 + MOVQ 8(R10), CX + TESTQ CX, CX + JZ empty_seqs + MOVQ (R10), AX + MOVQ 24(R10), DX + MOVQ 32(R10), BX + MOVQ 80(R10), SI + MOVQ 104(R10), DI + MOVQ 120(R10), R8 + MOVQ 56(R10), R9 + MOVQ 64(R10), R10 + ADDQ R10, R9 + + // seqsBase += 24 * seqIndex + LEAQ (DX)(DX*2), R11 + SHLQ $0x03, R11 + ADDQ R11, AX + + // outBase += outPosition + ADDQ DI, BX + +main_loop: + MOVQ (AX), R11 + MOVQ 16(AX), R12 + MOVQ 8(AX), R13 + + // Copy literals + TESTQ R11, R11 + JZ check_offset + XORQ R14, R14 + +copy_1: + MOVUPS (SI)(R14*1), X0 + MOVUPS X0, (BX)(R14*1) + ADDQ $0x10, R14 + CMPQ R14, R11 + JB copy_1 + ADDQ R11, SI + ADDQ R11, BX + ADDQ R11, DI + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + LEAQ (DI)(R10*1), R11 + CMPQ R12, R11 + JG error_match_off_too_big + CMPQ R12, R8 + JG error_match_off_too_big + + // Copy match from history + MOVQ R12, R11 + SUBQ DI, R11 + JLS copy_match + MOVQ R9, R14 + SUBQ R11, R14 + CMPQ R13, R11 + JG copy_all_from_history + MOVQ R13, R11 + SUBQ $0x10, R11 + JB copy_4_small + +copy_4_loop: + MOVUPS (R14), X0 + MOVUPS X0, (BX) + ADDQ $0x10, R14 + ADDQ $0x10, BX + SUBQ $0x10, R11 + JAE copy_4_loop + LEAQ 16(R14)(R11*1), R14 + LEAQ 16(BX)(R11*1), BX + MOVUPS -16(R14), X0 + MOVUPS X0, -16(BX) + JMP copy_4_end + +copy_4_small: + CMPQ R13, $0x03 + JE copy_4_move_3 + CMPQ R13, $0x08 + JB copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVW (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end + +copy_4_move_4through7: + MOVL (R14), R11 + MOVL -4(R14)(R13*1), R12 + MOVL R11, (BX) + MOVL R12, -4(BX)(R13*1) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end + +copy_4_move_8through16: + MOVQ (R14), R11 + MOVQ -8(R14)(R13*1), R12 + MOVQ R11, (BX) + MOVQ R12, -8(BX)(R13*1) + ADDQ R13, R14 + ADDQ R13, BX + +copy_4_end: + ADDQ R13, DI + ADDQ $0x18, AX + INCQ DX + CMPQ DX, CX + JB main_loop + JMP loop_finished + +copy_all_from_history: + MOVQ R11, R15 + SUBQ $0x10, R15 + JB copy_5_small + +copy_5_loop: + MOVUPS (R14), X0 + MOVUPS X0, (BX) + ADDQ $0x10, R14 + ADDQ $0x10, BX + SUBQ $0x10, R15 + JAE copy_5_loop + LEAQ 16(R14)(R15*1), R14 + LEAQ 16(BX)(R15*1), BX + MOVUPS -16(R14), X0 + MOVUPS X0, -16(BX) + JMP copy_5_end + +copy_5_small: + CMPQ R11, $0x03 + JE copy_5_move_3 + JB copy_5_move_1or2 + CMPQ R11, $0x08 + JB copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVB (R14), R15 + MOVB -1(R14)(R11*1), BP + MOVB R15, (BX) + MOVB BP, -1(BX)(R11*1) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end + +copy_5_move_3: + MOVW (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end + +copy_5_move_4through7: + MOVL (R14), R15 + MOVL -4(R14)(R11*1), BP + MOVL R15, (BX) + MOVL BP, -4(BX)(R11*1) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end + +copy_5_move_8through16: + MOVQ (R14), R15 + MOVQ -8(R14)(R11*1), BP + MOVQ R15, (BX) + MOVQ BP, -8(BX)(R11*1) + ADDQ R11, R14 + ADDQ R11, BX + +copy_5_end: + ADDQ R11, DI + SUBQ R11, R13 + + // Copy match from the current buffer +copy_match: + MOVQ BX, R11 + SUBQ R12, R11 + + // ml <= mo + CMPQ R13, R12 + JA copy_overlapping_match + + // Copy non-overlapping match + ADDQ R13, DI + MOVQ BX, R12 + ADDQ R13, BX + +copy_2: + MOVUPS (R11), X0 + MOVUPS X0, (R12) + ADDQ $0x10, R11 + ADDQ $0x10, R12 + SUBQ $0x10, R13 + JHI copy_2 + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADDQ R13, DI + +copy_slow_3: + MOVB (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 + +handle_loop: + ADDQ $0x18, AX + INCQ DX + CMPQ DX, CX + JB main_loop + +loop_finished: + // Return value + MOVB $0x01, ret+8(FP) + + // Update the context + MOVQ ctx+0(FP), AX + MOVQ DX, 24(AX) + MOVQ DI, 104(AX) + SUBQ 80(AX), SI + MOVQ SI, 112(AX) + RET + +error_match_off_too_big: + // Return value + MOVB $0x00, ret+8(FP) + + // Update the context + MOVQ ctx+0(FP), AX + MOVQ DX, 24(AX) + MOVQ DI, 104(AX) + SUBQ 80(AX), SI + MOVQ SI, 112(AX) + RET + +empty_seqs: + // Return value + MOVB $0x01, ret+8(FP) + RET + +// func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool +// Requires: SSE +TEXT ·sequenceDecs_executeSimple_safe_amd64(SB), $8-9 + MOVQ ctx+0(FP), R10 + MOVQ 8(R10), CX + TESTQ CX, CX + JZ empty_seqs + MOVQ (R10), AX + MOVQ 24(R10), DX + MOVQ 32(R10), BX + MOVQ 80(R10), SI + MOVQ 104(R10), DI + MOVQ 120(R10), R8 + MOVQ 56(R10), R9 + MOVQ 64(R10), R10 + ADDQ R10, R9 + + // seqsBase += 24 * seqIndex + LEAQ (DX)(DX*2), R11 + SHLQ $0x03, R11 + ADDQ R11, AX + + // outBase += outPosition + ADDQ DI, BX + +main_loop: + MOVQ (AX), R11 + MOVQ 16(AX), R12 + MOVQ 8(AX), R13 + + // Copy literals + TESTQ R11, R11 + JZ check_offset + MOVQ R11, R14 + SUBQ $0x10, R14 + JB copy_1_small + +copy_1_loop: + MOVUPS (SI), X0 + MOVUPS X0, (BX) + ADDQ $0x10, SI + ADDQ $0x10, BX + SUBQ $0x10, R14 + JAE copy_1_loop + LEAQ 16(SI)(R14*1), SI + LEAQ 16(BX)(R14*1), BX + MOVUPS -16(SI), X0 + MOVUPS X0, -16(BX) + JMP copy_1_end + +copy_1_small: + CMPQ R11, $0x03 + JE copy_1_move_3 + JB copy_1_move_1or2 + CMPQ R11, $0x08 + JB copy_1_move_4through7 + JMP copy_1_move_8through16 + +copy_1_move_1or2: + MOVB (SI), R14 + MOVB -1(SI)(R11*1), R15 + MOVB R14, (BX) + MOVB R15, -1(BX)(R11*1) + ADDQ R11, SI + ADDQ R11, BX + JMP copy_1_end + +copy_1_move_3: + MOVW (SI), R14 + MOVB 2(SI), R15 + MOVW R14, (BX) + MOVB R15, 2(BX) + ADDQ R11, SI + ADDQ R11, BX + JMP copy_1_end + +copy_1_move_4through7: + MOVL (SI), R14 + MOVL -4(SI)(R11*1), R15 + MOVL R14, (BX) + MOVL R15, -4(BX)(R11*1) + ADDQ R11, SI + ADDQ R11, BX + JMP copy_1_end + +copy_1_move_8through16: + MOVQ (SI), R14 + MOVQ -8(SI)(R11*1), R15 + MOVQ R14, (BX) + MOVQ R15, -8(BX)(R11*1) + ADDQ R11, SI + ADDQ R11, BX + +copy_1_end: + ADDQ R11, DI + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + LEAQ (DI)(R10*1), R11 + CMPQ R12, R11 + JG error_match_off_too_big + CMPQ R12, R8 + JG error_match_off_too_big + + // Copy match from history + MOVQ R12, R11 + SUBQ DI, R11 + JLS copy_match + MOVQ R9, R14 + SUBQ R11, R14 + CMPQ R13, R11 + JG copy_all_from_history + MOVQ R13, R11 + SUBQ $0x10, R11 + JB copy_4_small + +copy_4_loop: + MOVUPS (R14), X0 + MOVUPS X0, (BX) + ADDQ $0x10, R14 + ADDQ $0x10, BX + SUBQ $0x10, R11 + JAE copy_4_loop + LEAQ 16(R14)(R11*1), R14 + LEAQ 16(BX)(R11*1), BX + MOVUPS -16(R14), X0 + MOVUPS X0, -16(BX) + JMP copy_4_end + +copy_4_small: + CMPQ R13, $0x03 + JE copy_4_move_3 + CMPQ R13, $0x08 + JB copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVW (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end + +copy_4_move_4through7: + MOVL (R14), R11 + MOVL -4(R14)(R13*1), R12 + MOVL R11, (BX) + MOVL R12, -4(BX)(R13*1) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end + +copy_4_move_8through16: + MOVQ (R14), R11 + MOVQ -8(R14)(R13*1), R12 + MOVQ R11, (BX) + MOVQ R12, -8(BX)(R13*1) + ADDQ R13, R14 + ADDQ R13, BX + +copy_4_end: + ADDQ R13, DI + ADDQ $0x18, AX + INCQ DX + CMPQ DX, CX + JB main_loop + JMP loop_finished + +copy_all_from_history: + MOVQ R11, R15 + SUBQ $0x10, R15 + JB copy_5_small + +copy_5_loop: + MOVUPS (R14), X0 + MOVUPS X0, (BX) + ADDQ $0x10, R14 + ADDQ $0x10, BX + SUBQ $0x10, R15 + JAE copy_5_loop + LEAQ 16(R14)(R15*1), R14 + LEAQ 16(BX)(R15*1), BX + MOVUPS -16(R14), X0 + MOVUPS X0, -16(BX) + JMP copy_5_end + +copy_5_small: + CMPQ R11, $0x03 + JE copy_5_move_3 + JB copy_5_move_1or2 + CMPQ R11, $0x08 + JB copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVB (R14), R15 + MOVB -1(R14)(R11*1), BP + MOVB R15, (BX) + MOVB BP, -1(BX)(R11*1) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end + +copy_5_move_3: + MOVW (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end + +copy_5_move_4through7: + MOVL (R14), R15 + MOVL -4(R14)(R11*1), BP + MOVL R15, (BX) + MOVL BP, -4(BX)(R11*1) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end + +copy_5_move_8through16: + MOVQ (R14), R15 + MOVQ -8(R14)(R11*1), BP + MOVQ R15, (BX) + MOVQ BP, -8(BX)(R11*1) + ADDQ R11, R14 + ADDQ R11, BX + +copy_5_end: + ADDQ R11, DI + SUBQ R11, R13 + + // Copy match from the current buffer +copy_match: + MOVQ BX, R11 + SUBQ R12, R11 + + // ml <= mo + CMPQ R13, R12 + JA copy_overlapping_match + + // Copy non-overlapping match + ADDQ R13, DI + MOVQ R13, R12 + SUBQ $0x10, R12 + JB copy_2_small + +copy_2_loop: + MOVUPS (R11), X0 + MOVUPS X0, (BX) + ADDQ $0x10, R11 + ADDQ $0x10, BX + SUBQ $0x10, R12 + JAE copy_2_loop + LEAQ 16(R11)(R12*1), R11 + LEAQ 16(BX)(R12*1), BX + MOVUPS -16(R11), X0 + MOVUPS X0, -16(BX) + JMP copy_2_end + +copy_2_small: + CMPQ R13, $0x03 + JE copy_2_move_3 + JB copy_2_move_1or2 + CMPQ R13, $0x08 + JB copy_2_move_4through7 + JMP copy_2_move_8through16 + +copy_2_move_1or2: + MOVB (R11), R12 + MOVB -1(R11)(R13*1), R14 + MOVB R12, (BX) + MOVB R14, -1(BX)(R13*1) + ADDQ R13, R11 + ADDQ R13, BX + JMP copy_2_end + +copy_2_move_3: + MOVW (R11), R12 + MOVB 2(R11), R14 + MOVW R12, (BX) + MOVB R14, 2(BX) + ADDQ R13, R11 + ADDQ R13, BX + JMP copy_2_end + +copy_2_move_4through7: + MOVL (R11), R12 + MOVL -4(R11)(R13*1), R14 + MOVL R12, (BX) + MOVL R14, -4(BX)(R13*1) + ADDQ R13, R11 + ADDQ R13, BX + JMP copy_2_end + +copy_2_move_8through16: + MOVQ (R11), R12 + MOVQ -8(R11)(R13*1), R14 + MOVQ R12, (BX) + MOVQ R14, -8(BX)(R13*1) + ADDQ R13, R11 + ADDQ R13, BX + +copy_2_end: + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADDQ R13, DI + +copy_slow_3: + MOVB (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 + +handle_loop: + ADDQ $0x18, AX + INCQ DX + CMPQ DX, CX + JB main_loop + +loop_finished: + // Return value + MOVB $0x01, ret+8(FP) + + // Update the context + MOVQ ctx+0(FP), AX + MOVQ DX, 24(AX) + MOVQ DI, 104(AX) + SUBQ 80(AX), SI + MOVQ SI, 112(AX) + RET + +error_match_off_too_big: + // Return value + MOVB $0x00, ret+8(FP) + + // Update the context + MOVQ ctx+0(FP), AX + MOVQ DX, 24(AX) + MOVQ DI, 104(AX) + SUBQ 80(AX), SI + MOVQ SI, 112(AX) + RET + +empty_seqs: + // Return value + MOVB $0x01, ret+8(FP) + RET + +// func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +// Requires: CMOV, SSE +TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 + MOVQ br+8(FP), AX + MOVQ 32(AX), DX + MOVBQZX 40(AX), BX + MOVQ 24(AX), SI + MOVQ (AX), AX + ADDQ SI, AX + MOVQ AX, (SP) + MOVQ ctx+16(FP), AX + MOVQ 72(AX), DI + MOVQ 80(AX), R8 + MOVQ 88(AX), R9 + XORQ CX, CX + MOVQ CX, 8(SP) + MOVQ CX, 16(SP) + MOVQ CX, 24(SP) + MOVQ 112(AX), R10 + MOVQ 128(AX), CX + MOVQ CX, 32(SP) + MOVQ 144(AX), R11 + MOVQ 136(AX), R12 + MOVQ 200(AX), CX + MOVQ CX, 56(SP) + MOVQ 176(AX), CX + MOVQ CX, 48(SP) + MOVQ 184(AX), AX + MOVQ AX, 40(SP) + MOVQ 40(SP), AX + ADDQ AX, 48(SP) + + // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + ADDQ R10, 32(SP) + + // outBase += outPosition + ADDQ R12, R10 + +sequenceDecs_decodeSync_amd64_main_loop: + MOVQ (SP), R13 + + // Fill bitreader to have enough for the offset and match length. + CMPQ SI, $0x08 + JL sequenceDecs_decodeSync_amd64_fill_byte_by_byte + MOVQ BX, AX + SHRQ $0x03, AX + SUBQ AX, R13 + MOVQ (R13), DX + SUBQ AX, SI + ANDQ $0x07, BX + JMP sequenceDecs_decodeSync_amd64_fill_end + +sequenceDecs_decodeSync_amd64_fill_byte_by_byte: + CMPQ SI, $0x00 + JLE sequenceDecs_decodeSync_amd64_fill_end + CMPQ BX, $0x07 + JLE sequenceDecs_decodeSync_amd64_fill_end + SHLQ $0x08, DX + SUBQ $0x01, R13 + SUBQ $0x01, SI + SUBQ $0x08, BX + MOVBQZX (R13), AX + ORQ AX, DX + JMP sequenceDecs_decodeSync_amd64_fill_byte_by_byte + +sequenceDecs_decodeSync_amd64_fill_end: + // Update offset + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_amd64_of_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX + +sequenceDecs_decodeSync_amd64_of_update_zero: + MOVQ AX, 8(SP) + + // Update match length + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX + +sequenceDecs_decodeSync_amd64_ml_update_zero: + MOVQ AX, 16(SP) + + // Fill bitreader to have enough for the remaining + CMPQ SI, $0x08 + JL sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte + MOVQ BX, AX + SHRQ $0x03, AX + SUBQ AX, R13 + MOVQ (R13), DX + SUBQ AX, SI + ANDQ $0x07, BX + JMP sequenceDecs_decodeSync_amd64_fill_2_end + +sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte: + CMPQ SI, $0x00 + JLE sequenceDecs_decodeSync_amd64_fill_2_end + CMPQ BX, $0x07 + JLE sequenceDecs_decodeSync_amd64_fill_2_end + SHLQ $0x08, DX + SUBQ $0x01, R13 + SUBQ $0x01, SI + SUBQ $0x08, BX + MOVBQZX (R13), AX + ORQ AX, DX + JMP sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte + +sequenceDecs_decodeSync_amd64_fill_2_end: + // Update literal length + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX + +sequenceDecs_decodeSync_amd64_ll_update_zero: + MOVQ AX, 24(SP) + + // Fill bitreader for state updates + MOVQ R13, (SP) + MOVQ R9, AX + SHRQ $0x08, AX + MOVBQZX AL, AX + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decodeSync_amd64_skip_update + + // Update Literal Length State + MOVBQZX DI, R13 + SHRQ $0x10, DI + MOVWQZX DI, DI + LEAQ (BX)(R13*1), CX + MOVQ DX, R14 + MOVQ CX, BX + ROLQ CL, R14 + MOVL $0x00000001, R15 + MOVB R13, CL + SHLL CL, R15 + DECL R15 + ANDQ R15, R14 + ADDQ R14, DI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(DI*8), DI + + // Update Match Length State + MOVBQZX R8, R13 + SHRQ $0x10, R8 + MOVWQZX R8, R8 + LEAQ (BX)(R13*1), CX + MOVQ DX, R14 + MOVQ CX, BX + ROLQ CL, R14 + MOVL $0x00000001, R15 + MOVB R13, CL + SHLL CL, R15 + DECL R15 + ANDQ R15, R14 + ADDQ R14, R8 + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Offset State + MOVBQZX R9, R13 + SHRQ $0x10, R9 + MOVWQZX R9, R9 + LEAQ (BX)(R13*1), CX + MOVQ DX, R14 + MOVQ CX, BX + ROLQ CL, R14 + MOVL $0x00000001, R15 + MOVB R13, CL + SHLL CL, R15 + DECL R15 + ANDQ R15, R14 + ADDQ R14, R9 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R9*8), R9 + +sequenceDecs_decodeSync_amd64_skip_update: + // Adjust offset + MOVQ s+0(FP), CX + MOVQ 8(SP), R13 + CMPQ AX, $0x01 + JBE sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 + MOVUPS 144(CX), X0 + MOVQ R13, 144(CX) + MOVUPS X0, 152(CX) + JMP sequenceDecs_decodeSync_amd64_after_adjust + +sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: + CMPQ 24(SP), $0x00000000 + JNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero + INCQ R13 + JMP sequenceDecs_decodeSync_amd64_adjust_offset_nonzero + +sequenceDecs_decodeSync_amd64_adjust_offset_maybezero: + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_amd64_adjust_offset_nonzero + MOVQ 144(CX), R13 + JMP sequenceDecs_decodeSync_amd64_after_adjust + +sequenceDecs_decodeSync_amd64_adjust_offset_nonzero: + MOVQ R13, AX + XORQ R14, R14 + MOVQ $-1, R15 + CMPQ R13, $0x03 + CMOVQEQ R14, AX + CMOVQEQ R15, R14 + ADDQ 144(CX)(AX*8), R14 + JNZ sequenceDecs_decodeSync_amd64_adjust_temp_valid + MOVQ $0x00000001, R14 + +sequenceDecs_decodeSync_amd64_adjust_temp_valid: + CMPQ R13, $0x01 + JZ sequenceDecs_decodeSync_amd64_adjust_skip + MOVQ 152(CX), AX + MOVQ AX, 160(CX) + +sequenceDecs_decodeSync_amd64_adjust_skip: + MOVQ 144(CX), AX + MOVQ AX, 152(CX) + MOVQ R14, 144(CX) + MOVQ R14, R13 + +sequenceDecs_decodeSync_amd64_after_adjust: + MOVQ R13, 8(SP) + + // Check values + MOVQ 16(SP), AX + MOVQ 24(SP), CX + LEAQ (AX)(CX*1), R14 + MOVQ s+0(FP), R15 + ADDQ R14, 256(R15) + MOVQ ctx+16(FP), R14 + SUBQ CX, 104(R14) + JS error_not_enough_literals + CMPQ AX, $0x00020002 + JA sequenceDecs_decodeSync_amd64_error_match_len_too_big + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_amd64_match_len_ofs_ok + TESTQ AX, AX + JNZ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decodeSync_amd64_match_len_ofs_ok: + MOVQ 24(SP), AX + MOVQ 8(SP), CX + MOVQ 16(SP), R13 + + // Check if we have enough space in s.out + LEAQ (AX)(R13*1), R14 + ADDQ R10, R14 + CMPQ R14, 32(SP) + JA error_not_enough_space + + // Copy literals + TESTQ AX, AX + JZ check_offset + XORQ R14, R14 + +copy_1: + MOVUPS (R11)(R14*1), X0 + MOVUPS X0, (R10)(R14*1) + ADDQ $0x10, R14 + CMPQ R14, AX + JB copy_1 + ADDQ AX, R11 + ADDQ AX, R10 + ADDQ AX, R12 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + MOVQ R12, AX + ADDQ 40(SP), AX + CMPQ CX, AX + JG error_match_off_too_big + CMPQ CX, 56(SP) + JG error_match_off_too_big + + // Copy match from history + MOVQ CX, AX + SUBQ R12, AX + JLS copy_match + MOVQ 48(SP), R14 + SUBQ AX, R14 + CMPQ R13, AX + JG copy_all_from_history + MOVQ R13, AX + SUBQ $0x10, AX + JB copy_4_small + +copy_4_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R10) + ADDQ $0x10, R14 + ADDQ $0x10, R10 + SUBQ $0x10, AX + JAE copy_4_loop + LEAQ 16(R14)(AX*1), R14 + LEAQ 16(R10)(AX*1), R10 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R10) + JMP copy_4_end + +copy_4_small: + CMPQ R13, $0x03 + JE copy_4_move_3 + CMPQ R13, $0x08 + JB copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVW (R14), AX + MOVB 2(R14), CL + MOVW AX, (R10) + MOVB CL, 2(R10) + ADDQ R13, R14 + ADDQ R13, R10 + JMP copy_4_end + +copy_4_move_4through7: + MOVL (R14), AX + MOVL -4(R14)(R13*1), CX + MOVL AX, (R10) + MOVL CX, -4(R10)(R13*1) + ADDQ R13, R14 + ADDQ R13, R10 + JMP copy_4_end + +copy_4_move_8through16: + MOVQ (R14), AX + MOVQ -8(R14)(R13*1), CX + MOVQ AX, (R10) + MOVQ CX, -8(R10)(R13*1) + ADDQ R13, R14 + ADDQ R13, R10 + +copy_4_end: + ADDQ R13, R12 + JMP handle_loop + JMP loop_finished + +copy_all_from_history: + MOVQ AX, R15 + SUBQ $0x10, R15 + JB copy_5_small + +copy_5_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R10) + ADDQ $0x10, R14 + ADDQ $0x10, R10 + SUBQ $0x10, R15 + JAE copy_5_loop + LEAQ 16(R14)(R15*1), R14 + LEAQ 16(R10)(R15*1), R10 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R10) + JMP copy_5_end + +copy_5_small: + CMPQ AX, $0x03 + JE copy_5_move_3 + JB copy_5_move_1or2 + CMPQ AX, $0x08 + JB copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVB (R14), R15 + MOVB -1(R14)(AX*1), BP + MOVB R15, (R10) + MOVB BP, -1(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end + +copy_5_move_3: + MOVW (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end + +copy_5_move_4through7: + MOVL (R14), R15 + MOVL -4(R14)(AX*1), BP + MOVL R15, (R10) + MOVL BP, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end + +copy_5_move_8through16: + MOVQ (R14), R15 + MOVQ -8(R14)(AX*1), BP + MOVQ R15, (R10) + MOVQ BP, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 + +copy_5_end: + ADDQ AX, R12 + SUBQ AX, R13 + + // Copy match from the current buffer +copy_match: + MOVQ R10, AX + SUBQ CX, AX + + // ml <= mo + CMPQ R13, CX + JA copy_overlapping_match + + // Copy non-overlapping match + ADDQ R13, R12 + MOVQ R10, CX + ADDQ R13, R10 + +copy_2: + MOVUPS (AX), X0 + MOVUPS X0, (CX) + ADDQ $0x10, AX + ADDQ $0x10, CX + SUBQ $0x10, R13 + JHI copy_2 + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADDQ R13, R12 + +copy_slow_3: + MOVB (AX), CL + MOVB CL, (R10) + INCQ AX + INCQ R10 + DECQ R13 + JNZ copy_slow_3 + +handle_loop: + MOVQ ctx+16(FP), AX + DECQ 96(AX) + JNS sequenceDecs_decodeSync_amd64_main_loop + +loop_finished: + MOVQ br+8(FP), AX + MOVQ DX, 32(AX) + MOVB BL, 40(AX) + MOVQ SI, 24(AX) + + // Update the context + MOVQ ctx+16(FP), AX + MOVQ R12, 136(AX) + MOVQ 144(AX), CX + SUBQ CX, R11 + MOVQ R11, 168(AX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: + MOVQ 16(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 216(CX) + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decodeSync_amd64_error_match_len_too_big: + MOVQ ctx+16(FP), AX + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error +error_match_off_too_big: + MOVQ ctx+16(FP), AX + MOVQ 8(SP), CX + MOVQ CX, 224(AX) + MOVQ R12, 136(AX) + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ $0x00000004, ret+24(FP) + RET + + // Return with not enough output space error +error_not_enough_space: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ R12, 136(AX) + MOVQ $0x00000005, ret+24(FP) + RET + +// func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +// Requires: BMI, BMI2, CMOV, SSE +TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 + MOVQ br+8(FP), CX + MOVQ 32(CX), AX + MOVBQZX 40(CX), DX + MOVQ 24(CX), BX + MOVQ (CX), CX + ADDQ BX, CX + MOVQ CX, (SP) + MOVQ ctx+16(FP), CX + MOVQ 72(CX), SI + MOVQ 80(CX), DI + MOVQ 88(CX), R8 + XORQ R9, R9 + MOVQ R9, 8(SP) + MOVQ R9, 16(SP) + MOVQ R9, 24(SP) + MOVQ 112(CX), R9 + MOVQ 128(CX), R10 + MOVQ R10, 32(SP) + MOVQ 144(CX), R10 + MOVQ 136(CX), R11 + MOVQ 200(CX), R12 + MOVQ R12, 56(SP) + MOVQ 176(CX), R12 + MOVQ R12, 48(SP) + MOVQ 184(CX), CX + MOVQ CX, 40(SP) + MOVQ 40(SP), CX + ADDQ CX, 48(SP) + + // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + ADDQ R9, 32(SP) + + // outBase += outPosition + ADDQ R11, R9 + +sequenceDecs_decodeSync_bmi2_main_loop: + MOVQ (SP), R12 + + // Fill bitreader to have enough for the offset and match length. + CMPQ BX, $0x08 + JL sequenceDecs_decodeSync_bmi2_fill_byte_by_byte + MOVQ DX, CX + SHRQ $0x03, CX + SUBQ CX, R12 + MOVQ (R12), AX + SUBQ CX, BX + ANDQ $0x07, DX + JMP sequenceDecs_decodeSync_bmi2_fill_end + +sequenceDecs_decodeSync_bmi2_fill_byte_by_byte: + CMPQ BX, $0x00 + JLE sequenceDecs_decodeSync_bmi2_fill_end + CMPQ DX, $0x07 + JLE sequenceDecs_decodeSync_bmi2_fill_end + SHLQ $0x08, AX + SUBQ $0x01, R12 + SUBQ $0x01, BX + SUBQ $0x08, DX + MOVBQZX (R12), CX + ORQ CX, AX + JMP sequenceDecs_decodeSync_bmi2_fill_byte_by_byte + +sequenceDecs_decodeSync_bmi2_fill_end: + // Update offset + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R13 + MOVQ AX, R14 + LEAQ (DX)(R13*1), CX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + MOVQ CX, DX + MOVQ R8, CX + SHRQ $0x20, CX + ADDQ R14, CX + MOVQ CX, 8(SP) + + // Update match length + MOVQ $0x00000808, CX + BEXTRQ CX, DI, R13 + MOVQ AX, R14 + LEAQ (DX)(R13*1), CX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + MOVQ CX, DX + MOVQ DI, CX + SHRQ $0x20, CX + ADDQ R14, CX + MOVQ CX, 16(SP) + + // Fill bitreader to have enough for the remaining + CMPQ BX, $0x08 + JL sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte + MOVQ DX, CX + SHRQ $0x03, CX + SUBQ CX, R12 + MOVQ (R12), AX + SUBQ CX, BX + ANDQ $0x07, DX + JMP sequenceDecs_decodeSync_bmi2_fill_2_end + +sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte: + CMPQ BX, $0x00 + JLE sequenceDecs_decodeSync_bmi2_fill_2_end + CMPQ DX, $0x07 + JLE sequenceDecs_decodeSync_bmi2_fill_2_end + SHLQ $0x08, AX + SUBQ $0x01, R12 + SUBQ $0x01, BX + SUBQ $0x08, DX + MOVBQZX (R12), CX + ORQ CX, AX + JMP sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte + +sequenceDecs_decodeSync_bmi2_fill_2_end: + // Update literal length + MOVQ $0x00000808, CX + BEXTRQ CX, SI, R13 + MOVQ AX, R14 + LEAQ (DX)(R13*1), CX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + MOVQ CX, DX + MOVQ SI, CX + SHRQ $0x20, CX + ADDQ R14, CX + MOVQ CX, 24(SP) + + // Fill bitreader for state updates + MOVQ R12, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R12 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decodeSync_bmi2_skip_update + LEAQ (SI)(DI*1), R13 + ADDQ R8, R13 + MOVBQZX R13, R13 + LEAQ (DX)(R13*1), CX + MOVQ AX, R14 + MOVQ CX, DX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + + // Update Offset State + BZHIQ R8, R14, CX + SHRXQ R8, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Match Length State + BZHIQ DI, R14, CX + SHRXQ DI, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, DI, DI + ADDQ CX, DI + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(DI*8), DI + + // Update Literal Length State + BZHIQ SI, R14, CX + MOVQ $0x00001010, R13 + BEXTRQ R13, SI, SI + ADDQ CX, SI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI + +sequenceDecs_decodeSync_bmi2_skip_update: + // Adjust offset + MOVQ s+0(FP), CX + MOVQ 8(SP), R13 + CMPQ R12, $0x01 + JBE sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0 + MOVUPS 144(CX), X0 + MOVQ R13, 144(CX) + MOVUPS X0, 152(CX) + JMP sequenceDecs_decodeSync_bmi2_after_adjust + +sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0: + CMPQ 24(SP), $0x00000000 + JNE sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero + INCQ R13 + JMP sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero + +sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero: + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero + MOVQ 144(CX), R13 + JMP sequenceDecs_decodeSync_bmi2_after_adjust + +sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero: + MOVQ R13, R12 + XORQ R14, R14 + MOVQ $-1, R15 + CMPQ R13, $0x03 + CMOVQEQ R14, R12 + CMOVQEQ R15, R14 + ADDQ 144(CX)(R12*8), R14 + JNZ sequenceDecs_decodeSync_bmi2_adjust_temp_valid + MOVQ $0x00000001, R14 + +sequenceDecs_decodeSync_bmi2_adjust_temp_valid: + CMPQ R13, $0x01 + JZ sequenceDecs_decodeSync_bmi2_adjust_skip + MOVQ 152(CX), R12 + MOVQ R12, 160(CX) + +sequenceDecs_decodeSync_bmi2_adjust_skip: + MOVQ 144(CX), R12 + MOVQ R12, 152(CX) + MOVQ R14, 144(CX) + MOVQ R14, R13 + +sequenceDecs_decodeSync_bmi2_after_adjust: + MOVQ R13, 8(SP) + + // Check values + MOVQ 16(SP), CX + MOVQ 24(SP), R12 + LEAQ (CX)(R12*1), R14 + MOVQ s+0(FP), R15 + ADDQ R14, 256(R15) + MOVQ ctx+16(FP), R14 + SUBQ R12, 104(R14) + JS error_not_enough_literals + CMPQ CX, $0x00020002 + JA sequenceDecs_decodeSync_bmi2_error_match_len_too_big + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_bmi2_match_len_ofs_ok + TESTQ CX, CX + JNZ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch + +sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: + MOVQ 24(SP), CX + MOVQ 8(SP), R12 + MOVQ 16(SP), R13 + + // Check if we have enough space in s.out + LEAQ (CX)(R13*1), R14 + ADDQ R9, R14 + CMPQ R14, 32(SP) + JA error_not_enough_space + + // Copy literals + TESTQ CX, CX + JZ check_offset + XORQ R14, R14 + +copy_1: + MOVUPS (R10)(R14*1), X0 + MOVUPS X0, (R9)(R14*1) + ADDQ $0x10, R14 + CMPQ R14, CX + JB copy_1 + ADDQ CX, R10 + ADDQ CX, R9 + ADDQ CX, R11 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + MOVQ R11, CX + ADDQ 40(SP), CX + CMPQ R12, CX + JG error_match_off_too_big + CMPQ R12, 56(SP) + JG error_match_off_too_big + + // Copy match from history + MOVQ R12, CX + SUBQ R11, CX + JLS copy_match + MOVQ 48(SP), R14 + SUBQ CX, R14 + CMPQ R13, CX + JG copy_all_from_history + MOVQ R13, CX + SUBQ $0x10, CX + JB copy_4_small + +copy_4_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R9) + ADDQ $0x10, R14 + ADDQ $0x10, R9 + SUBQ $0x10, CX + JAE copy_4_loop + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R9)(CX*1), R9 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R9) + JMP copy_4_end + +copy_4_small: + CMPQ R13, $0x03 + JE copy_4_move_3 + CMPQ R13, $0x08 + JB copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVW (R14), CX + MOVB 2(R14), R12 + MOVW CX, (R9) + MOVB R12, 2(R9) + ADDQ R13, R14 + ADDQ R13, R9 + JMP copy_4_end + +copy_4_move_4through7: + MOVL (R14), CX + MOVL -4(R14)(R13*1), R12 + MOVL CX, (R9) + MOVL R12, -4(R9)(R13*1) + ADDQ R13, R14 + ADDQ R13, R9 + JMP copy_4_end + +copy_4_move_8through16: + MOVQ (R14), CX + MOVQ -8(R14)(R13*1), R12 + MOVQ CX, (R9) + MOVQ R12, -8(R9)(R13*1) + ADDQ R13, R14 + ADDQ R13, R9 + +copy_4_end: + ADDQ R13, R11 + JMP handle_loop + JMP loop_finished + +copy_all_from_history: + MOVQ CX, R15 + SUBQ $0x10, R15 + JB copy_5_small + +copy_5_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R9) + ADDQ $0x10, R14 + ADDQ $0x10, R9 + SUBQ $0x10, R15 + JAE copy_5_loop + LEAQ 16(R14)(R15*1), R14 + LEAQ 16(R9)(R15*1), R9 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R9) + JMP copy_5_end + +copy_5_small: + CMPQ CX, $0x03 + JE copy_5_move_3 + JB copy_5_move_1or2 + CMPQ CX, $0x08 + JB copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVB (R14), R15 + MOVB -1(R14)(CX*1), BP + MOVB R15, (R9) + MOVB BP, -1(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end + +copy_5_move_3: + MOVW (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end + +copy_5_move_4through7: + MOVL (R14), R15 + MOVL -4(R14)(CX*1), BP + MOVL R15, (R9) + MOVL BP, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end + +copy_5_move_8through16: + MOVQ (R14), R15 + MOVQ -8(R14)(CX*1), BP + MOVQ R15, (R9) + MOVQ BP, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 + +copy_5_end: + ADDQ CX, R11 + SUBQ CX, R13 + + // Copy match from the current buffer +copy_match: + MOVQ R9, CX + SUBQ R12, CX + + // ml <= mo + CMPQ R13, R12 + JA copy_overlapping_match + + // Copy non-overlapping match + ADDQ R13, R11 + MOVQ R9, R12 + ADDQ R13, R9 + +copy_2: + MOVUPS (CX), X0 + MOVUPS X0, (R12) + ADDQ $0x10, CX + ADDQ $0x10, R12 + SUBQ $0x10, R13 + JHI copy_2 + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADDQ R13, R11 + +copy_slow_3: + MOVB (CX), R12 + MOVB R12, (R9) + INCQ CX + INCQ R9 + DECQ R13 + JNZ copy_slow_3 + +handle_loop: + MOVQ ctx+16(FP), CX + DECQ 96(CX) + JNS sequenceDecs_decodeSync_bmi2_main_loop + +loop_finished: + MOVQ br+8(FP), CX + MOVQ AX, 32(CX) + MOVB DL, 40(CX) + MOVQ BX, 24(CX) + + // Update the context + MOVQ ctx+16(FP), AX + MOVQ R11, 136(AX) + MOVQ 144(AX), CX + SUBQ CX, R10 + MOVQ R10, 168(AX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: + MOVQ 16(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 216(CX) + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decodeSync_bmi2_error_match_len_too_big: + MOVQ ctx+16(FP), AX + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error +error_match_off_too_big: + MOVQ ctx+16(FP), AX + MOVQ 8(SP), CX + MOVQ CX, 224(AX) + MOVQ R11, 136(AX) + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ $0x00000004, ret+24(FP) + RET + + // Return with not enough output space error +error_not_enough_space: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ R11, 136(AX) + MOVQ $0x00000005, ret+24(FP) + RET + +// func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +// Requires: CMOV, SSE +TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 + MOVQ br+8(FP), AX + MOVQ 32(AX), DX + MOVBQZX 40(AX), BX + MOVQ 24(AX), SI + MOVQ (AX), AX + ADDQ SI, AX + MOVQ AX, (SP) + MOVQ ctx+16(FP), AX + MOVQ 72(AX), DI + MOVQ 80(AX), R8 + MOVQ 88(AX), R9 + XORQ CX, CX + MOVQ CX, 8(SP) + MOVQ CX, 16(SP) + MOVQ CX, 24(SP) + MOVQ 112(AX), R10 + MOVQ 128(AX), CX + MOVQ CX, 32(SP) + MOVQ 144(AX), R11 + MOVQ 136(AX), R12 + MOVQ 200(AX), CX + MOVQ CX, 56(SP) + MOVQ 176(AX), CX + MOVQ CX, 48(SP) + MOVQ 184(AX), AX + MOVQ AX, 40(SP) + MOVQ 40(SP), AX + ADDQ AX, 48(SP) + + // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + ADDQ R10, 32(SP) + + // outBase += outPosition + ADDQ R12, R10 + +sequenceDecs_decodeSync_safe_amd64_main_loop: + MOVQ (SP), R13 + + // Fill bitreader to have enough for the offset and match length. + CMPQ SI, $0x08 + JL sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte + MOVQ BX, AX + SHRQ $0x03, AX + SUBQ AX, R13 + MOVQ (R13), DX + SUBQ AX, SI + ANDQ $0x07, BX + JMP sequenceDecs_decodeSync_safe_amd64_fill_end + +sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte: + CMPQ SI, $0x00 + JLE sequenceDecs_decodeSync_safe_amd64_fill_end + CMPQ BX, $0x07 + JLE sequenceDecs_decodeSync_safe_amd64_fill_end + SHLQ $0x08, DX + SUBQ $0x01, R13 + SUBQ $0x01, SI + SUBQ $0x08, BX + MOVBQZX (R13), AX + ORQ AX, DX + JMP sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte + +sequenceDecs_decodeSync_safe_amd64_fill_end: + // Update offset + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_safe_amd64_of_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX + +sequenceDecs_decodeSync_safe_amd64_of_update_zero: + MOVQ AX, 8(SP) + + // Update match length + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX + +sequenceDecs_decodeSync_safe_amd64_ml_update_zero: + MOVQ AX, 16(SP) + + // Fill bitreader to have enough for the remaining + CMPQ SI, $0x08 + JL sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte + MOVQ BX, AX + SHRQ $0x03, AX + SUBQ AX, R13 + MOVQ (R13), DX + SUBQ AX, SI + ANDQ $0x07, BX + JMP sequenceDecs_decodeSync_safe_amd64_fill_2_end + +sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte: + CMPQ SI, $0x00 + JLE sequenceDecs_decodeSync_safe_amd64_fill_2_end + CMPQ BX, $0x07 + JLE sequenceDecs_decodeSync_safe_amd64_fill_2_end + SHLQ $0x08, DX + SUBQ $0x01, R13 + SUBQ $0x01, SI + SUBQ $0x08, BX + MOVBQZX (R13), AX + ORQ AX, DX + JMP sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte + +sequenceDecs_decodeSync_safe_amd64_fill_2_end: + // Update literal length + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + MOVB AH, CL + SHRQ $0x20, AX + TESTQ CX, CX + JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero + ADDQ CX, BX + CMPQ BX, $0x40 + JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero + CMPQ CX, $0x40 + JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero + NEGQ CX + SHRQ CL, R14 + ADDQ R14, AX + +sequenceDecs_decodeSync_safe_amd64_ll_update_zero: + MOVQ AX, 24(SP) + + // Fill bitreader for state updates + MOVQ R13, (SP) + MOVQ R9, AX + SHRQ $0x08, AX + MOVBQZX AL, AX + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decodeSync_safe_amd64_skip_update + + // Update Literal Length State + MOVBQZX DI, R13 + SHRQ $0x10, DI + MOVWQZX DI, DI + LEAQ (BX)(R13*1), CX + MOVQ DX, R14 + MOVQ CX, BX + ROLQ CL, R14 + MOVL $0x00000001, R15 + MOVB R13, CL + SHLL CL, R15 + DECL R15 + ANDQ R15, R14 + ADDQ R14, DI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(DI*8), DI + + // Update Match Length State + MOVBQZX R8, R13 + SHRQ $0x10, R8 + MOVWQZX R8, R8 + LEAQ (BX)(R13*1), CX + MOVQ DX, R14 + MOVQ CX, BX + ROLQ CL, R14 + MOVL $0x00000001, R15 + MOVB R13, CL + SHLL CL, R15 + DECL R15 + ANDQ R15, R14 + ADDQ R14, R8 + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Offset State + MOVBQZX R9, R13 + SHRQ $0x10, R9 + MOVWQZX R9, R9 + LEAQ (BX)(R13*1), CX + MOVQ DX, R14 + MOVQ CX, BX + ROLQ CL, R14 + MOVL $0x00000001, R15 + MOVB R13, CL + SHLL CL, R15 + DECL R15 + ANDQ R15, R14 + ADDQ R14, R9 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R9*8), R9 + +sequenceDecs_decodeSync_safe_amd64_skip_update: + // Adjust offset + MOVQ s+0(FP), CX + MOVQ 8(SP), R13 + CMPQ AX, $0x01 + JBE sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 + MOVUPS 144(CX), X0 + MOVQ R13, 144(CX) + MOVUPS X0, 152(CX) + JMP sequenceDecs_decodeSync_safe_amd64_after_adjust + +sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: + CMPQ 24(SP), $0x00000000 + JNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero + INCQ R13 + JMP sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero + +sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero: + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero + MOVQ 144(CX), R13 + JMP sequenceDecs_decodeSync_safe_amd64_after_adjust + +sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero: + MOVQ R13, AX + XORQ R14, R14 + MOVQ $-1, R15 + CMPQ R13, $0x03 + CMOVQEQ R14, AX + CMOVQEQ R15, R14 + ADDQ 144(CX)(AX*8), R14 + JNZ sequenceDecs_decodeSync_safe_amd64_adjust_temp_valid + MOVQ $0x00000001, R14 + +sequenceDecs_decodeSync_safe_amd64_adjust_temp_valid: + CMPQ R13, $0x01 + JZ sequenceDecs_decodeSync_safe_amd64_adjust_skip + MOVQ 152(CX), AX + MOVQ AX, 160(CX) + +sequenceDecs_decodeSync_safe_amd64_adjust_skip: + MOVQ 144(CX), AX + MOVQ AX, 152(CX) + MOVQ R14, 144(CX) + MOVQ R14, R13 + +sequenceDecs_decodeSync_safe_amd64_after_adjust: + MOVQ R13, 8(SP) + + // Check values + MOVQ 16(SP), AX + MOVQ 24(SP), CX + LEAQ (AX)(CX*1), R14 + MOVQ s+0(FP), R15 + ADDQ R14, 256(R15) + MOVQ ctx+16(FP), R14 + SUBQ CX, 104(R14) + JS error_not_enough_literals + CMPQ AX, $0x00020002 + JA sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok + TESTQ AX, AX + JNZ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch + +sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: + MOVQ 24(SP), AX + MOVQ 8(SP), CX + MOVQ 16(SP), R13 + + // Check if we have enough space in s.out + LEAQ (AX)(R13*1), R14 + ADDQ R10, R14 + CMPQ R14, 32(SP) + JA error_not_enough_space + + // Copy literals + TESTQ AX, AX + JZ check_offset + MOVQ AX, R14 + SUBQ $0x10, R14 + JB copy_1_small + +copy_1_loop: + MOVUPS (R11), X0 + MOVUPS X0, (R10) + ADDQ $0x10, R11 + ADDQ $0x10, R10 + SUBQ $0x10, R14 + JAE copy_1_loop + LEAQ 16(R11)(R14*1), R11 + LEAQ 16(R10)(R14*1), R10 + MOVUPS -16(R11), X0 + MOVUPS X0, -16(R10) + JMP copy_1_end + +copy_1_small: + CMPQ AX, $0x03 + JE copy_1_move_3 + JB copy_1_move_1or2 + CMPQ AX, $0x08 + JB copy_1_move_4through7 + JMP copy_1_move_8through16 + +copy_1_move_1or2: + MOVB (R11), R14 + MOVB -1(R11)(AX*1), R15 + MOVB R14, (R10) + MOVB R15, -1(R10)(AX*1) + ADDQ AX, R11 + ADDQ AX, R10 + JMP copy_1_end + +copy_1_move_3: + MOVW (R11), R14 + MOVB 2(R11), R15 + MOVW R14, (R10) + MOVB R15, 2(R10) + ADDQ AX, R11 + ADDQ AX, R10 + JMP copy_1_end + +copy_1_move_4through7: + MOVL (R11), R14 + MOVL -4(R11)(AX*1), R15 + MOVL R14, (R10) + MOVL R15, -4(R10)(AX*1) + ADDQ AX, R11 + ADDQ AX, R10 + JMP copy_1_end + +copy_1_move_8through16: + MOVQ (R11), R14 + MOVQ -8(R11)(AX*1), R15 + MOVQ R14, (R10) + MOVQ R15, -8(R10)(AX*1) + ADDQ AX, R11 + ADDQ AX, R10 + +copy_1_end: + ADDQ AX, R12 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + MOVQ R12, AX + ADDQ 40(SP), AX + CMPQ CX, AX + JG error_match_off_too_big + CMPQ CX, 56(SP) + JG error_match_off_too_big + + // Copy match from history + MOVQ CX, AX + SUBQ R12, AX + JLS copy_match + MOVQ 48(SP), R14 + SUBQ AX, R14 + CMPQ R13, AX + JG copy_all_from_history + MOVQ R13, AX + SUBQ $0x10, AX + JB copy_4_small + +copy_4_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R10) + ADDQ $0x10, R14 + ADDQ $0x10, R10 + SUBQ $0x10, AX + JAE copy_4_loop + LEAQ 16(R14)(AX*1), R14 + LEAQ 16(R10)(AX*1), R10 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R10) + JMP copy_4_end + +copy_4_small: + CMPQ R13, $0x03 + JE copy_4_move_3 + CMPQ R13, $0x08 + JB copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVW (R14), AX + MOVB 2(R14), CL + MOVW AX, (R10) + MOVB CL, 2(R10) + ADDQ R13, R14 + ADDQ R13, R10 + JMP copy_4_end + +copy_4_move_4through7: + MOVL (R14), AX + MOVL -4(R14)(R13*1), CX + MOVL AX, (R10) + MOVL CX, -4(R10)(R13*1) + ADDQ R13, R14 + ADDQ R13, R10 + JMP copy_4_end + +copy_4_move_8through16: + MOVQ (R14), AX + MOVQ -8(R14)(R13*1), CX + MOVQ AX, (R10) + MOVQ CX, -8(R10)(R13*1) + ADDQ R13, R14 + ADDQ R13, R10 + +copy_4_end: + ADDQ R13, R12 + JMP handle_loop + JMP loop_finished + +copy_all_from_history: + MOVQ AX, R15 + SUBQ $0x10, R15 + JB copy_5_small + +copy_5_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R10) + ADDQ $0x10, R14 + ADDQ $0x10, R10 + SUBQ $0x10, R15 + JAE copy_5_loop + LEAQ 16(R14)(R15*1), R14 + LEAQ 16(R10)(R15*1), R10 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R10) + JMP copy_5_end + +copy_5_small: + CMPQ AX, $0x03 + JE copy_5_move_3 + JB copy_5_move_1or2 + CMPQ AX, $0x08 + JB copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVB (R14), R15 + MOVB -1(R14)(AX*1), BP + MOVB R15, (R10) + MOVB BP, -1(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end + +copy_5_move_3: + MOVW (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end + +copy_5_move_4through7: + MOVL (R14), R15 + MOVL -4(R14)(AX*1), BP + MOVL R15, (R10) + MOVL BP, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_5_end + +copy_5_move_8through16: + MOVQ (R14), R15 + MOVQ -8(R14)(AX*1), BP + MOVQ R15, (R10) + MOVQ BP, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 + +copy_5_end: + ADDQ AX, R12 + SUBQ AX, R13 + + // Copy match from the current buffer +copy_match: + MOVQ R10, AX + SUBQ CX, AX + + // ml <= mo + CMPQ R13, CX + JA copy_overlapping_match + + // Copy non-overlapping match + ADDQ R13, R12 + MOVQ R13, CX + SUBQ $0x10, CX + JB copy_2_small + +copy_2_loop: + MOVUPS (AX), X0 + MOVUPS X0, (R10) + ADDQ $0x10, AX + ADDQ $0x10, R10 + SUBQ $0x10, CX + JAE copy_2_loop + LEAQ 16(AX)(CX*1), AX + LEAQ 16(R10)(CX*1), R10 + MOVUPS -16(AX), X0 + MOVUPS X0, -16(R10) + JMP copy_2_end + +copy_2_small: + CMPQ R13, $0x03 + JE copy_2_move_3 + JB copy_2_move_1or2 + CMPQ R13, $0x08 + JB copy_2_move_4through7 + JMP copy_2_move_8through16 + +copy_2_move_1or2: + MOVB (AX), CL + MOVB -1(AX)(R13*1), R14 + MOVB CL, (R10) + MOVB R14, -1(R10)(R13*1) + ADDQ R13, AX + ADDQ R13, R10 + JMP copy_2_end + +copy_2_move_3: + MOVW (AX), CX + MOVB 2(AX), R14 + MOVW CX, (R10) + MOVB R14, 2(R10) + ADDQ R13, AX + ADDQ R13, R10 + JMP copy_2_end + +copy_2_move_4through7: + MOVL (AX), CX + MOVL -4(AX)(R13*1), R14 + MOVL CX, (R10) + MOVL R14, -4(R10)(R13*1) + ADDQ R13, AX + ADDQ R13, R10 + JMP copy_2_end + +copy_2_move_8through16: + MOVQ (AX), CX + MOVQ -8(AX)(R13*1), R14 + MOVQ CX, (R10) + MOVQ R14, -8(R10)(R13*1) + ADDQ R13, AX + ADDQ R13, R10 + +copy_2_end: + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADDQ R13, R12 + +copy_slow_3: + MOVB (AX), CL + MOVB CL, (R10) + INCQ AX + INCQ R10 + DECQ R13 + JNZ copy_slow_3 + +handle_loop: + MOVQ ctx+16(FP), AX + DECQ 96(AX) + JNS sequenceDecs_decodeSync_safe_amd64_main_loop + +loop_finished: + MOVQ br+8(FP), AX + MOVQ DX, 32(AX) + MOVB BL, 40(AX) + MOVQ SI, 24(AX) + + // Update the context + MOVQ ctx+16(FP), AX + MOVQ R12, 136(AX) + MOVQ 144(AX), CX + SUBQ CX, R11 + MOVQ R11, 168(AX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: + MOVQ 16(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 216(CX) + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: + MOVQ ctx+16(FP), AX + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error +error_match_off_too_big: + MOVQ ctx+16(FP), AX + MOVQ 8(SP), CX + MOVQ CX, 224(AX) + MOVQ R12, 136(AX) + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ $0x00000004, ret+24(FP) + RET + + // Return with not enough output space error +error_not_enough_space: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ R12, 136(AX) + MOVQ $0x00000005, ret+24(FP) + RET + +// func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +// Requires: BMI, BMI2, CMOV, SSE +TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 + MOVQ br+8(FP), CX + MOVQ 32(CX), AX + MOVBQZX 40(CX), DX + MOVQ 24(CX), BX + MOVQ (CX), CX + ADDQ BX, CX + MOVQ CX, (SP) + MOVQ ctx+16(FP), CX + MOVQ 72(CX), SI + MOVQ 80(CX), DI + MOVQ 88(CX), R8 + XORQ R9, R9 + MOVQ R9, 8(SP) + MOVQ R9, 16(SP) + MOVQ R9, 24(SP) + MOVQ 112(CX), R9 + MOVQ 128(CX), R10 + MOVQ R10, 32(SP) + MOVQ 144(CX), R10 + MOVQ 136(CX), R11 + MOVQ 200(CX), R12 + MOVQ R12, 56(SP) + MOVQ 176(CX), R12 + MOVQ R12, 48(SP) + MOVQ 184(CX), CX + MOVQ CX, 40(SP) + MOVQ 40(SP), CX + ADDQ CX, 48(SP) + + // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + ADDQ R9, 32(SP) + + // outBase += outPosition + ADDQ R11, R9 + +sequenceDecs_decodeSync_safe_bmi2_main_loop: + MOVQ (SP), R12 + + // Fill bitreader to have enough for the offset and match length. + CMPQ BX, $0x08 + JL sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte + MOVQ DX, CX + SHRQ $0x03, CX + SUBQ CX, R12 + MOVQ (R12), AX + SUBQ CX, BX + ANDQ $0x07, DX + JMP sequenceDecs_decodeSync_safe_bmi2_fill_end + +sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte: + CMPQ BX, $0x00 + JLE sequenceDecs_decodeSync_safe_bmi2_fill_end + CMPQ DX, $0x07 + JLE sequenceDecs_decodeSync_safe_bmi2_fill_end + SHLQ $0x08, AX + SUBQ $0x01, R12 + SUBQ $0x01, BX + SUBQ $0x08, DX + MOVBQZX (R12), CX + ORQ CX, AX + JMP sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte + +sequenceDecs_decodeSync_safe_bmi2_fill_end: + // Update offset + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R13 + MOVQ AX, R14 + LEAQ (DX)(R13*1), CX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + MOVQ CX, DX + MOVQ R8, CX + SHRQ $0x20, CX + ADDQ R14, CX + MOVQ CX, 8(SP) + + // Update match length + MOVQ $0x00000808, CX + BEXTRQ CX, DI, R13 + MOVQ AX, R14 + LEAQ (DX)(R13*1), CX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + MOVQ CX, DX + MOVQ DI, CX + SHRQ $0x20, CX + ADDQ R14, CX + MOVQ CX, 16(SP) + + // Fill bitreader to have enough for the remaining + CMPQ BX, $0x08 + JL sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte + MOVQ DX, CX + SHRQ $0x03, CX + SUBQ CX, R12 + MOVQ (R12), AX + SUBQ CX, BX + ANDQ $0x07, DX + JMP sequenceDecs_decodeSync_safe_bmi2_fill_2_end + +sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte: + CMPQ BX, $0x00 + JLE sequenceDecs_decodeSync_safe_bmi2_fill_2_end + CMPQ DX, $0x07 + JLE sequenceDecs_decodeSync_safe_bmi2_fill_2_end + SHLQ $0x08, AX + SUBQ $0x01, R12 + SUBQ $0x01, BX + SUBQ $0x08, DX + MOVBQZX (R12), CX + ORQ CX, AX + JMP sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte + +sequenceDecs_decodeSync_safe_bmi2_fill_2_end: + // Update literal length + MOVQ $0x00000808, CX + BEXTRQ CX, SI, R13 + MOVQ AX, R14 + LEAQ (DX)(R13*1), CX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + MOVQ CX, DX + MOVQ SI, CX + SHRQ $0x20, CX + ADDQ R14, CX + MOVQ CX, 24(SP) + + // Fill bitreader for state updates + MOVQ R12, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R12 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decodeSync_safe_bmi2_skip_update + LEAQ (SI)(DI*1), R13 + ADDQ R8, R13 + MOVBQZX R13, R13 + LEAQ (DX)(R13*1), CX + MOVQ AX, R14 + MOVQ CX, DX + ROLQ CL, R14 + BZHIQ R13, R14, R14 + + // Update Offset State + BZHIQ R8, R14, CX + SHRXQ R8, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable + MOVQ ctx+16(FP), CX + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 + + // Update Match Length State + BZHIQ DI, R14, CX + SHRXQ DI, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, DI, DI + ADDQ CX, DI + + // Load ctx.mlTable + MOVQ ctx+16(FP), CX + MOVQ 24(CX), CX + MOVQ (CX)(DI*8), DI + + // Update Literal Length State + BZHIQ SI, R14, CX + MOVQ $0x00001010, R13 + BEXTRQ R13, SI, SI + ADDQ CX, SI + + // Load ctx.llTable + MOVQ ctx+16(FP), CX + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI + +sequenceDecs_decodeSync_safe_bmi2_skip_update: + // Adjust offset + MOVQ s+0(FP), CX + MOVQ 8(SP), R13 + CMPQ R12, $0x01 + JBE sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0 + MOVUPS 144(CX), X0 + MOVQ R13, 144(CX) + MOVUPS X0, 152(CX) + JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust + +sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0: + CMPQ 24(SP), $0x00000000 + JNE sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero + INCQ R13 + JMP sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero + +sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero: + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero + MOVQ 144(CX), R13 + JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust + +sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero: + MOVQ R13, R12 + XORQ R14, R14 + MOVQ $-1, R15 + CMPQ R13, $0x03 + CMOVQEQ R14, R12 + CMOVQEQ R15, R14 + ADDQ 144(CX)(R12*8), R14 + JNZ sequenceDecs_decodeSync_safe_bmi2_adjust_temp_valid + MOVQ $0x00000001, R14 + +sequenceDecs_decodeSync_safe_bmi2_adjust_temp_valid: + CMPQ R13, $0x01 + JZ sequenceDecs_decodeSync_safe_bmi2_adjust_skip + MOVQ 152(CX), R12 + MOVQ R12, 160(CX) + +sequenceDecs_decodeSync_safe_bmi2_adjust_skip: + MOVQ 144(CX), R12 + MOVQ R12, 152(CX) + MOVQ R14, 144(CX) + MOVQ R14, R13 + +sequenceDecs_decodeSync_safe_bmi2_after_adjust: + MOVQ R13, 8(SP) + + // Check values + MOVQ 16(SP), CX + MOVQ 24(SP), R12 + LEAQ (CX)(R12*1), R14 + MOVQ s+0(FP), R15 + ADDQ R14, 256(R15) + MOVQ ctx+16(FP), R14 + SUBQ R12, 104(R14) + JS error_not_enough_literals + CMPQ CX, $0x00020002 + JA sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big + TESTQ R13, R13 + JNZ sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok + TESTQ CX, CX + JNZ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch + +sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok: + MOVQ 24(SP), CX + MOVQ 8(SP), R12 + MOVQ 16(SP), R13 + + // Check if we have enough space in s.out + LEAQ (CX)(R13*1), R14 + ADDQ R9, R14 + CMPQ R14, 32(SP) + JA error_not_enough_space + + // Copy literals + TESTQ CX, CX + JZ check_offset + MOVQ CX, R14 + SUBQ $0x10, R14 + JB copy_1_small + +copy_1_loop: + MOVUPS (R10), X0 + MOVUPS X0, (R9) + ADDQ $0x10, R10 + ADDQ $0x10, R9 + SUBQ $0x10, R14 + JAE copy_1_loop + LEAQ 16(R10)(R14*1), R10 + LEAQ 16(R9)(R14*1), R9 + MOVUPS -16(R10), X0 + MOVUPS X0, -16(R9) + JMP copy_1_end + +copy_1_small: + CMPQ CX, $0x03 + JE copy_1_move_3 + JB copy_1_move_1or2 + CMPQ CX, $0x08 + JB copy_1_move_4through7 + JMP copy_1_move_8through16 + +copy_1_move_1or2: + MOVB (R10), R14 + MOVB -1(R10)(CX*1), R15 + MOVB R14, (R9) + MOVB R15, -1(R9)(CX*1) + ADDQ CX, R10 + ADDQ CX, R9 + JMP copy_1_end + +copy_1_move_3: + MOVW (R10), R14 + MOVB 2(R10), R15 + MOVW R14, (R9) + MOVB R15, 2(R9) + ADDQ CX, R10 + ADDQ CX, R9 + JMP copy_1_end + +copy_1_move_4through7: + MOVL (R10), R14 + MOVL -4(R10)(CX*1), R15 + MOVL R14, (R9) + MOVL R15, -4(R9)(CX*1) + ADDQ CX, R10 + ADDQ CX, R9 + JMP copy_1_end + +copy_1_move_8through16: + MOVQ (R10), R14 + MOVQ -8(R10)(CX*1), R15 + MOVQ R14, (R9) + MOVQ R15, -8(R9)(CX*1) + ADDQ CX, R10 + ADDQ CX, R9 + +copy_1_end: + ADDQ CX, R11 + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + MOVQ R11, CX + ADDQ 40(SP), CX + CMPQ R12, CX + JG error_match_off_too_big + CMPQ R12, 56(SP) + JG error_match_off_too_big + + // Copy match from history + MOVQ R12, CX + SUBQ R11, CX + JLS copy_match + MOVQ 48(SP), R14 + SUBQ CX, R14 + CMPQ R13, CX + JG copy_all_from_history + MOVQ R13, CX + SUBQ $0x10, CX + JB copy_4_small + +copy_4_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R9) + ADDQ $0x10, R14 + ADDQ $0x10, R9 + SUBQ $0x10, CX + JAE copy_4_loop + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R9)(CX*1), R9 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R9) + JMP copy_4_end + +copy_4_small: + CMPQ R13, $0x03 + JE copy_4_move_3 + CMPQ R13, $0x08 + JB copy_4_move_4through7 + JMP copy_4_move_8through16 + +copy_4_move_3: + MOVW (R14), CX + MOVB 2(R14), R12 + MOVW CX, (R9) + MOVB R12, 2(R9) + ADDQ R13, R14 + ADDQ R13, R9 + JMP copy_4_end + +copy_4_move_4through7: + MOVL (R14), CX + MOVL -4(R14)(R13*1), R12 + MOVL CX, (R9) + MOVL R12, -4(R9)(R13*1) + ADDQ R13, R14 + ADDQ R13, R9 + JMP copy_4_end + +copy_4_move_8through16: + MOVQ (R14), CX + MOVQ -8(R14)(R13*1), R12 + MOVQ CX, (R9) + MOVQ R12, -8(R9)(R13*1) + ADDQ R13, R14 + ADDQ R13, R9 + +copy_4_end: + ADDQ R13, R11 + JMP handle_loop + JMP loop_finished + +copy_all_from_history: + MOVQ CX, R15 + SUBQ $0x10, R15 + JB copy_5_small + +copy_5_loop: + MOVUPS (R14), X0 + MOVUPS X0, (R9) + ADDQ $0x10, R14 + ADDQ $0x10, R9 + SUBQ $0x10, R15 + JAE copy_5_loop + LEAQ 16(R14)(R15*1), R14 + LEAQ 16(R9)(R15*1), R9 + MOVUPS -16(R14), X0 + MOVUPS X0, -16(R9) + JMP copy_5_end + +copy_5_small: + CMPQ CX, $0x03 + JE copy_5_move_3 + JB copy_5_move_1or2 + CMPQ CX, $0x08 + JB copy_5_move_4through7 + JMP copy_5_move_8through16 + +copy_5_move_1or2: + MOVB (R14), R15 + MOVB -1(R14)(CX*1), BP + MOVB R15, (R9) + MOVB BP, -1(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end + +copy_5_move_3: + MOVW (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end + +copy_5_move_4through7: + MOVL (R14), R15 + MOVL -4(R14)(CX*1), BP + MOVL R15, (R9) + MOVL BP, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_5_end + +copy_5_move_8through16: + MOVQ (R14), R15 + MOVQ -8(R14)(CX*1), BP + MOVQ R15, (R9) + MOVQ BP, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 + +copy_5_end: + ADDQ CX, R11 + SUBQ CX, R13 + + // Copy match from the current buffer +copy_match: + MOVQ R9, CX + SUBQ R12, CX + + // ml <= mo + CMPQ R13, R12 + JA copy_overlapping_match + + // Copy non-overlapping match + ADDQ R13, R11 + MOVQ R13, R12 + SUBQ $0x10, R12 + JB copy_2_small + +copy_2_loop: + MOVUPS (CX), X0 + MOVUPS X0, (R9) + ADDQ $0x10, CX + ADDQ $0x10, R9 + SUBQ $0x10, R12 + JAE copy_2_loop + LEAQ 16(CX)(R12*1), CX + LEAQ 16(R9)(R12*1), R9 + MOVUPS -16(CX), X0 + MOVUPS X0, -16(R9) + JMP copy_2_end + +copy_2_small: + CMPQ R13, $0x03 + JE copy_2_move_3 + JB copy_2_move_1or2 + CMPQ R13, $0x08 + JB copy_2_move_4through7 + JMP copy_2_move_8through16 + +copy_2_move_1or2: + MOVB (CX), R12 + MOVB -1(CX)(R13*1), R14 + MOVB R12, (R9) + MOVB R14, -1(R9)(R13*1) + ADDQ R13, CX + ADDQ R13, R9 + JMP copy_2_end + +copy_2_move_3: + MOVW (CX), R12 + MOVB 2(CX), R14 + MOVW R12, (R9) + MOVB R14, 2(R9) + ADDQ R13, CX + ADDQ R13, R9 + JMP copy_2_end + +copy_2_move_4through7: + MOVL (CX), R12 + MOVL -4(CX)(R13*1), R14 + MOVL R12, (R9) + MOVL R14, -4(R9)(R13*1) + ADDQ R13, CX + ADDQ R13, R9 + JMP copy_2_end + +copy_2_move_8through16: + MOVQ (CX), R12 + MOVQ -8(CX)(R13*1), R14 + MOVQ R12, (R9) + MOVQ R14, -8(R9)(R13*1) + ADDQ R13, CX + ADDQ R13, R9 + +copy_2_end: + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADDQ R13, R11 + +copy_slow_3: + MOVB (CX), R12 + MOVB R12, (R9) + INCQ CX + INCQ R9 + DECQ R13 + JNZ copy_slow_3 + +handle_loop: + MOVQ ctx+16(FP), CX + DECQ 96(CX) + JNS sequenceDecs_decodeSync_safe_bmi2_main_loop + +loop_finished: + MOVQ br+8(FP), CX + MOVQ AX, 32(CX) + MOVB DL, 40(CX) + MOVQ BX, 24(CX) + + // Update the context + MOVQ ctx+16(FP), AX + MOVQ R11, 136(AX) + MOVQ 144(AX), CX + SUBQ CX, R10 + MOVQ R10, 168(AX) + + // Return success + MOVQ $0x00000000, ret+24(FP) + RET + + // Return with match length error +sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: + MOVQ 16(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 216(CX) + MOVQ $0x00000001, ret+24(FP) + RET + + // Return with match too long error +sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: + MOVQ ctx+16(FP), AX + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ $0x00000002, ret+24(FP) + RET + + // Return with match offset too long error +error_match_off_too_big: + MOVQ ctx+16(FP), AX + MOVQ 8(SP), CX + MOVQ CX, 224(AX) + MOVQ R11, 136(AX) + MOVQ $0x00000003, ret+24(FP) + RET + + // Return with not enough literals error +error_not_enough_literals: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ $0x00000004, ret+24(FP) + RET + + // Return with not enough output space error +error_not_enough_space: + MOVQ ctx+16(FP), AX + MOVQ 24(SP), CX + MOVQ CX, 208(AX) + MOVQ 16(SP), CX + MOVQ CX, 216(AX) + MOVQ R11, 136(AX) + MOVQ $0x00000005, ret+24(FP) + RET diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go new file mode 100644 index 000000000..ac2a80d29 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go @@ -0,0 +1,237 @@ +//go:build !amd64 || appengine || !gc || noasm +// +build !amd64 appengine !gc noasm + +package zstd + +import ( + "fmt" + "io" +) + +// decode sequences from the stream with the provided history but without dictionary. +func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { + return false, nil +} + +// decode sequences from the stream without the provided history. +func (s *sequenceDecs) decode(seqs []seqVals) error { + br := s.br + + // Grab full sizes tables, to avoid bounds checks. + llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize] + llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state + s.seqSize = 0 + litRemain := len(s.literals) + + maxBlockSize := maxCompressedBlockSize + if s.windowSize < maxBlockSize { + maxBlockSize = s.windowSize + } + for i := range seqs { + var ll, mo, ml int + if br.off > 4+((maxOffsetBits+16+16)>>3) { + // inlined function: + // ll, mo, ml = s.nextFast(br, llState, mlState, ofState) + + // Final will not read from stream. + var llB, mlB, moB uint8 + ll, llB = llState.final() + ml, mlB = mlState.final() + mo, moB = ofState.final() + + // extra bits are stored in reverse order. + br.fillFast() + mo += br.getBits(moB) + if s.maxBits > 32 { + br.fillFast() + } + ml += br.getBits(mlB) + ll += br.getBits(llB) + + if moB > 1 { + s.prevOffset[2] = s.prevOffset[1] + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = mo + } else { + // mo = s.adjustOffset(mo, ll, moB) + // Inlined for rather big speedup + if ll == 0 { + // There is an exception though, when current sequence's literals_length = 0. + // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, + // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. + mo++ + } + + if mo == 0 { + mo = s.prevOffset[0] + } else { + var temp int + if mo == 3 { + temp = s.prevOffset[0] - 1 + } else { + temp = s.prevOffset[mo] + } + + if temp == 0 { + // 0 is not valid; input is corrupted; force offset to 1 + println("WARNING: temp was 0") + temp = 1 + } + + if mo != 1 { + s.prevOffset[2] = s.prevOffset[1] + } + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = temp + mo = temp + } + } + br.fillFast() + } else { + if br.overread() { + if debugDecoder { + printf("reading sequence %d, exceeded available data\n", i) + } + return io.ErrUnexpectedEOF + } + ll, mo, ml = s.next(br, llState, mlState, ofState) + br.fill() + } + + if debugSequences { + println("Seq", i, "Litlen:", ll, "mo:", mo, "(abs) ml:", ml) + } + // Evaluate. + // We might be doing this async, so do it early. + if mo == 0 && ml > 0 { + return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) + } + if ml > maxMatchLen { + return fmt.Errorf("match len (%d) bigger than max allowed length", ml) + } + s.seqSize += ll + ml + if s.seqSize > maxBlockSize { + return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + } + litRemain -= ll + if litRemain < 0 { + return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, litRemain+ll) + } + seqs[i] = seqVals{ + ll: ll, + ml: ml, + mo: mo, + } + if i == len(seqs)-1 { + // This is the last sequence, so we shouldn't update state. + break + } + + // Manually inlined, ~ 5-20% faster + // Update all 3 states at once. Approx 20% faster. + nBits := llState.nbBits() + mlState.nbBits() + ofState.nbBits() + if nBits == 0 { + llState = llTable[llState.newState()&maxTableMask] + mlState = mlTable[mlState.newState()&maxTableMask] + ofState = ofTable[ofState.newState()&maxTableMask] + } else { + bits := br.get32BitsFast(nBits) + lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31)) + llState = llTable[(llState.newState()+lowBits)&maxTableMask] + + lowBits = uint16(bits >> (ofState.nbBits() & 31)) + lowBits &= bitMask[mlState.nbBits()&15] + mlState = mlTable[(mlState.newState()+lowBits)&maxTableMask] + + lowBits = uint16(bits) & bitMask[ofState.nbBits()&15] + ofState = ofTable[(ofState.newState()+lowBits)&maxTableMask] + } + } + s.seqSize += litRemain + if s.seqSize > maxBlockSize { + return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + } + err := br.close() + if err != nil { + printf("Closing sequences: %v, %+v\n", err, *br) + } + return err +} + +// executeSimple handles cases when a dictionary is not used. +func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { + // Ensure we have enough output size... + if len(s.out)+s.seqSize > cap(s.out) { + addBytes := s.seqSize + len(s.out) + s.out = append(s.out, make([]byte, addBytes)...) + s.out = s.out[:len(s.out)-addBytes] + } + + if debugDecoder { + printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize) + } + + var t = len(s.out) + out := s.out[:t+s.seqSize] + + for _, seq := range seqs { + // Add literals + copy(out[t:], s.literals[:seq.ll]) + t += seq.ll + s.literals = s.literals[seq.ll:] + + // Malformed input + if seq.mo > t+len(hist) || seq.mo > s.windowSize { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", seq.mo, t+len(hist)) + } + + // Copy from history. + if v := seq.mo - t; v > 0 { + // v is the start position in history from end. + start := len(hist) - v + if seq.ml > v { + // Some goes into the current block. + // Copy remainder of history + copy(out[t:], hist[start:]) + t += v + seq.ml -= v + } else { + copy(out[t:], hist[start:start+seq.ml]) + t += seq.ml + continue + } + } + + // We must be in the current buffer now + if seq.ml > 0 { + start := t - seq.mo + if seq.ml <= t-start { + // No overlap + copy(out[t:], out[start:start+seq.ml]) + t += seq.ml + } else { + // Overlapping copy + // Extend destination slice and copy one byte at the time. + src := out[start : start+seq.ml] + dst := out[t:] + dst = dst[:len(src)] + t += len(src) + // Destination is the space we just added. + for i := range src { + dst[i] = src[i] + } + } + } + } + // Add final literals + copy(out[t:], s.literals) + if debugDecoder { + t += len(s.literals) + if t != len(out) { + panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) + } + } + s.out = out + + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/zip.go b/vendor/github.com/klauspost/compress/zstd/zip.go index 967f29b31..29c15c8c4 100644 --- a/vendor/github.com/klauspost/compress/zstd/zip.go +++ b/vendor/github.com/klauspost/compress/zstd/zip.go @@ -18,36 +18,58 @@ const ZipMethodWinZip = 93 // See https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT const ZipMethodPKWare = 20 -var zipReaderPool sync.Pool +// zipReaderPool is the default reader pool. +var zipReaderPool = sync.Pool{New: func() interface{} { + z, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderMaxWindow(128<<20), WithDecoderConcurrency(1)) + if err != nil { + panic(err) + } + return z +}} -// newZipReader cannot be used since we would leak goroutines... -func newZipReader(r io.Reader) io.ReadCloser { - dec, ok := zipReaderPool.Get().(*Decoder) - if ok { - dec.Reset(r) - } else { - d, err := NewReader(r, WithDecoderConcurrency(1), WithDecoderLowmem(true)) - if err != nil { - panic(err) +// newZipReader creates a pooled zip decompressor. +func newZipReader(opts ...DOption) func(r io.Reader) io.ReadCloser { + pool := &zipReaderPool + if len(opts) > 0 { + opts = append([]DOption{WithDecoderLowmem(true), WithDecoderMaxWindow(128 << 20)}, opts...) + // Force concurrency 1 + opts = append(opts, WithDecoderConcurrency(1)) + // Create our own pool + pool = &sync.Pool{} + } + return func(r io.Reader) io.ReadCloser { + dec, ok := pool.Get().(*Decoder) + if ok { + dec.Reset(r) + } else { + d, err := NewReader(r, opts...) + if err != nil { + panic(err) + } + dec = d } - dec = d + return &pooledZipReader{dec: dec, pool: pool} } - return &pooledZipReader{dec: dec} } type pooledZipReader struct { - mu sync.Mutex // guards Close and Read - dec *Decoder + mu sync.Mutex // guards Close and Read + pool *sync.Pool + dec *Decoder } func (r *pooledZipReader) Read(p []byte) (n int, err error) { r.mu.Lock() defer r.mu.Unlock() if r.dec == nil { - return 0, errors.New("Read after Close") + return 0, errors.New("read after close or EOF") } dec, err := r.dec.Read(p) - + if err == io.EOF { + r.dec.Reset(nil) + r.pool.Put(r.dec) + r.dec = nil + } return dec, err } @@ -57,7 +79,7 @@ func (r *pooledZipReader) Close() error { var err error if r.dec != nil { err = r.dec.Reset(nil) - zipReaderPool.Put(r.dec) + r.pool.Put(r.dec) r.dec = nil } return err @@ -111,12 +133,9 @@ func ZipCompressor(opts ...EOption) func(w io.Writer) (io.WriteCloser, error) { // ZipDecompressor returns a decompressor that can be registered with zip libraries. // See ZipCompressor for example. -func ZipDecompressor() func(r io.Reader) io.ReadCloser { - return func(r io.Reader) io.ReadCloser { - d, err := NewReader(r, WithDecoderConcurrency(1), WithDecoderLowmem(true)) - if err != nil { - panic(err) - } - return d.IOReadCloser() - } +// Options can be specified. WithDecoderConcurrency(1) is forced, +// and by default a 128MB maximum decompression window is specified. +// The window size can be overridden if required. +func ZipDecompressor(opts ...DOption) func(r io.Reader) io.ReadCloser { + return newZipReader(opts...) } diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index ef1d49a00..5ffa82f5a 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -36,8 +36,8 @@ const forcePreDef = false // zstdMinMatch is the minimum zstd match length. const zstdMinMatch = 3 -// Reset the buffer offset when reaching this. -const bufferReset = math.MaxInt32 - MaxWindowSize +// fcsUnknown is used for unknown frame content size. +const fcsUnknown = math.MaxUint64 var ( // ErrReservedBlockType is returned when a reserved block type is found. @@ -52,6 +52,10 @@ var ( // Typically returned on invalid input. ErrBlockTooSmall = errors.New("block too small") + // ErrUnexpectedBlockSize is returned when a block has unexpected size. + // Typically returned on invalid input. + ErrUnexpectedBlockSize = errors.New("unexpected block size") + // ErrMagicMismatch is returned when a "magic" number isn't what is expected. // Typically this indicates wrong or corrupted input. ErrMagicMismatch = errors.New("invalid input: magic number mismatch") @@ -68,13 +72,16 @@ var ( ErrDecoderSizeExceeded = errors.New("decompressed size exceeds configured limit") // ErrUnknownDictionary is returned if the dictionary ID is unknown. - // For the time being dictionaries are not supported. ErrUnknownDictionary = errors.New("unknown dictionary") // ErrFrameSizeExceeded is returned if the stated frame size is exceeded. // This is only returned if SingleSegment is specified on the frame. ErrFrameSizeExceeded = errors.New("frame size exceeded") + // ErrFrameSizeMismatch is returned if the stated frame size does not match the expected size. + // This is only returned if SingleSegment is specified on the frame. + ErrFrameSizeMismatch = errors.New("frame size does not match size on stream") + // ErrCRCMismatch is returned if CRC mismatches. ErrCRCMismatch = errors.New("CRC check failed") @@ -99,37 +106,25 @@ func printf(format string, a ...interface{}) { } } -// matchLenFast does matching, but will not match the last up to 7 bytes. -func matchLenFast(a, b []byte) int { - endI := len(a) & (math.MaxInt32 - 7) - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - return i + bits.TrailingZeros64(diff)>>3 - } - } - return endI -} - -// matchLen returns the maximum length. +// matchLen returns the maximum common prefix length of a and b. // a must be the shortest of the two. -// The function also returns whether all bytes matched. -func matchLen(a, b []byte) int { - b = b[:len(a)] - for i := 0; i < len(a)-7; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - return i + (bits.TrailingZeros64(diff) >> 3) +func matchLen(a, b []byte) (n int) { + for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] { + diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b) + if diff != 0 { + return n + bits.TrailingZeros64(diff)>>3 } + n += 8 } - checked := (len(a) >> 3) << 3 - a = a[checked:] - b = b[checked:] for i := range a { if a[i] != b[i] { - return i + checked + break } + n++ } - return len(a) + checked + return n + } func load3232(b []byte, i int32) uint32 { @@ -140,10 +135,6 @@ func load6432(b []byte, i int32) uint64 { return binary.LittleEndian.Uint64(b[i:]) } -func load64(b []byte, i int) uint64 { - return binary.LittleEndian.Uint64(b[i:]) -} - type byter interface { Bytes() []byte Len() int diff --git a/vendor/github.com/mattn/go-isatty/isatty_bsd.go b/vendor/github.com/mattn/go-isatty/isatty_bsd.go index 39bbcf00f..d569c0c94 100644 --- a/vendor/github.com/mattn/go-isatty/isatty_bsd.go +++ b/vendor/github.com/mattn/go-isatty/isatty_bsd.go @@ -1,5 +1,5 @@ -//go:build (darwin || freebsd || openbsd || netbsd || dragonfly) && !appengine -// +build darwin freebsd openbsd netbsd dragonfly +//go:build (darwin || freebsd || openbsd || netbsd || dragonfly || hurd) && !appengine +// +build darwin freebsd openbsd netbsd dragonfly hurd // +build !appengine package isatty diff --git a/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md b/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md index f79662bf5..9d901bd2f 100644 --- a/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md +++ b/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md @@ -1,3 +1,262 @@ +## 2.8.4 + +### Features +- Add OmitSuiteSetupNodes to JunitReportConfig (#1147) [979fbc2] +- Add a reference to ginkgolinter in docs.index.md (#1143) [8432589] + +### Fixes +- rename tools hack to see if it fixes things for downstream users [a8bb39a] + +### Maintenance +- Bump golang.org/x/text (#1144) [41b2a8a] +- Bump github.com/onsi/gomega from 1.27.0 to 1.27.1 (#1142) [7c4f583] + +## 2.8.3 + +Released to fix security issue in golang.org/x/net dependency + +### Maintenance + +- Bump golang.org/x/net from 0.6.0 to 0.7.0 (#1141) [fc1a02e] +- remove tools.go hack from documentation [0718693] + +## 2.8.2 + +Ginkgo now includes a `tools.go` file in the root directory of the `ginkgo` package. This should allow modules that simply `go get github.com/onsi/ginkgo/v2` to also pull in the CLI dependencies. This obviates the need for consumers of Ginkgo to have their own `tools.go` file and makes it simpler to ensure that the version of the `ginkgo` CLI being used matches the version of the library. You can simply run `go run github.com/onsi/ginkgo/v2/ginkgo` to run the version of the cli associated with your package go.mod. + +### Maintenance + +- Bump github.com/onsi/gomega from 1.26.0 to 1.27.0 (#1139) [5767b0a] +- Fix minor typos (#1138) [e1e9723] +- Fix link in V2 Migration Guide (#1137) [a588f60] + +## 2.8.1 + +### Fixes +- lock around default report output to avoid triggering the race detector when calling By from goroutines [2d5075a] +- don't run ReportEntries through sprintf [febbe38] + +### Maintenance +- Bump golang.org/x/tools from 0.5.0 to 0.6.0 (#1135) [11a4860] +- test: update matrix for Go 1.20 (#1130) [4890a62] +- Bump golang.org/x/sys from 0.4.0 to 0.5.0 (#1133) [a774638] +- Bump github.com/onsi/gomega from 1.25.0 to 1.26.0 (#1120) [3f233bd] +- Bump github-pages from 227 to 228 in /docs (#1131) [f9b8649] +- Bump activesupport from 6.0.6 to 6.0.6.1 in /docs (#1127) [6f8c042] +- Update index.md with instructions on how to upgrade Ginkgo [833a75e] + +## 2.8.0 + +### Features + +- Introduce GinkgoHelper() to track and exclude helper functions from potential CodeLocations [e19f556] + +Modeled after `testing.T.Helper()`. Now, rather than write code like: + +```go +func helper(model Model) { + Expect(model).WithOffset(1).To(BeValid()) + Expect(model.SerialNumber).WithOffset(1).To(MatchRegexp(/[a-f0-9]*/)) +} +``` + +you can stop tracking offsets (which makes nesting composing helpers nearly impossible) and simply write: + +```go +func helper(model Model) { + GinkgoHelper() + Expect(model).To(BeValid()) + Expect(model.SerialNumber).To(MatchRegexp(/[a-f0-9]*/)) +} +``` + +- Introduce GinkgoLabelFilter() and Label().MatchesLabelFilter() to make it possible to programmatically match filters (fixes #1119) [2f6597c] + +You can now write code like this: + +```go +BeforeSuite(func() { + if Label("slow").MatchesLabelFilter(GinkgoLabelFilter()) { + // do slow setup + } + + if Label("fast").MatchesLabelFilter(GinkgoLabelFilter()) { + // do fast setup + } +}) +``` + +to programmatically check whether a given set of labels will match the configured `--label-filter`. + +### Maintenance + +- Bump webrick from 1.7.0 to 1.8.1 in /docs (#1125) [ea4966e] +- cdeql: add ruby language (#1124) [9dd275b] +- dependabot: add bundler package-ecosystem for docs (#1123) [14e7bdd] + +## 2.7.1 + +### Fixes +- Bring back SuiteConfig.EmitSpecProgress to avoid compilation issue for consumers that set it manually [d2a1cb0] + +### Maintenance +- Bump github.com/onsi/gomega from 1.24.2 to 1.25.0 (#1118) [cafece6] +- Bump golang.org/x/tools from 0.4.0 to 0.5.0 (#1111) [eda66c2] +- Bump golang.org/x/sys from 0.3.0 to 0.4.0 (#1112) [ac5ccaa] +- Bump github.com/onsi/gomega from 1.24.1 to 1.24.2 (#1097) [eee6480] + +## 2.7.0 + +### Features +- Introduce ContinueOnFailure for Ordered containers [e0123ca] - Ordered containers that are also decorated with ContinueOnFailure will not stop running specs after the first spec fails. +- Support for bootstrap commands to use custom data for templates (#1110) [7a2b242] +- Support for labels and pending decorator in ginkgo outline output (#1113) [e6e3b98] +- Color aliases for custom color support (#1101) [49fab7a] + +### Fixes +- correctly ensure deterministic spec order, even if specs are generated by iterating over a map [89dda20] +- Fix a bug where timedout specs were not correctly treated as failures when determining whether or not to run AfterAlls in an Ordered container. +- Ensure go test coverprofile outputs to the expected location (#1105) [b0bd77b] + +## 2.6.1 + +### Features +- Override formatter colors from envvars - this is a new feature but an alternative approach involving config files might be taken in the future (#1095) [60240d1] + +### Fixes +- GinkgoRecover now supports ignoring panics that match a specific, hidden, interface [301f3e2] + +### Maintenance +- Bump github.com/onsi/gomega from 1.24.0 to 1.24.1 (#1077) [3643823] +- Bump golang.org/x/tools from 0.2.0 to 0.4.0 (#1090) [f9f856e] +- Bump nokogiri from 1.13.9 to 1.13.10 in /docs (#1091) [0d7087e] + +## 2.6.0 + +### Features +- `ReportBeforeSuite` provides access to the suite report before the suite begins. +- Add junit config option for omitting leafnodetype (#1088) [956e6d2] +- Add support to customize junit report config to omit spec labels (#1087) [de44005] + +### Fixes +- Fix stack trace pruning so that it has a chance of working on windows [2165648] + +## 2.5.1 + +### Fixes +- skipped tests only show as 'S' when running with -v [3ab38ae] +- Fix typo in docs/index.md (#1082) [55fc58d] +- Fix typo in docs/index.md (#1081) [8a14f1f] +- Fix link notation in docs/index.md (#1080) [2669612] +- Fix typo in `--progress` deprecation message (#1076) [b4b7edc] + +### Maintenance +- chore: Included githubactions in the dependabot config (#976) [baea341] +- Bump golang.org/x/sys from 0.1.0 to 0.2.0 (#1075) [9646297] + +## 2.5.0 + +### Ginkgo output now includes a timeline-view of the spec + +This commit changes Ginkgo's default output. Spec details are now +presented as a **timeline** that includes events that occur during the spec +lifecycle interleaved with any GinkgoWriter content. This makes is much easier +to understand the flow of a spec and where a given failure occurs. + +The --progress, --slow-spec-threshold, --always-emit-ginkgo-writer flags +and the SuppressProgressReporting decorator have all been deprecated. Instead +the existing -v and -vv flags better capture the level of verbosity to display. However, +a new --show-node-events flag is added to include node `> Enter` and `< Exit` events +in the spec timeline. + +In addition, JUnit reports now include the timeline (rendered with -vv) and custom JUnit +reports can be configured and generated using +`GenerateJUnitReportWithConfig(report types.Report, dst string, config JunitReportConfig)` + +Code should continue to work unchanged with this version of Ginkgo - however if you have tooling that +was relying on the specific output format of Ginkgo you _may_ run into issues. Ginkgo's console output is not guaranteed to be stable for tooling and automation purposes. You should, instead, use Ginkgo's JSON format +to build tooling on top of as it has stronger guarantees to be stable from version to version. + +### Features +- Provide details about which timeout expired [0f2fa27] + +### Fixes +- Add Support Policy to docs [c70867a] + +### Maintenance +- Bump github.com/onsi/gomega from 1.22.1 to 1.23.0 (#1070) [bb3b4e2] + +## 2.4.0 + +### Features + +- DeferCleanup supports functions with multiple-return values [5e33c75] +- Add GinkgoLogr (#1067) [bf78c28] +- Introduction of 'MustPassRepeatedly' decorator (#1051) [047c02f] + +### Fixes +- correcting some typos (#1064) [1403d3c] +- fix flaky internal_integration interrupt specs [2105ba3] +- Correct busted link in README [be6b5b9] + +### Maintenance +- Bump actions/checkout from 2 to 3 (#1062) [8a2f483] +- Bump golang.org/x/tools from 0.1.12 to 0.2.0 (#1065) [529c4e8] +- Bump github/codeql-action from 1 to 2 (#1061) [da09146] +- Bump actions/setup-go from 2 to 3 (#1060) [918040d] +- Bump github.com/onsi/gomega from 1.22.0 to 1.22.1 (#1053) [2098e4d] +- Bump nokogiri from 1.13.8 to 1.13.9 in /docs (#1066) [1d74122] +- Add GHA to dependabot config [4442772] + +## 2.3.1 + +## Fixes +Several users were invoking `ginkgo` by installing the latest version of the cli via `go install github.com/onsi/ginkgo/v2/ginkgo@latest`. When 2.3.0 was released this resulted in an influx of issues as CI systems failed due to a change in the internal contract between the Ginkgo CLI and the Ginkgo library. Ginkgo only supports running the same version of the library as the cli (which is why both are packaged in the same repository). + +With this patch release, the ginkgo CLI can now identify a version mismatch and emit a helpful error message. + +- Ginkgo cli can identify version mismatches and emit a helpful error message [bc4ae2f] +- further emphasize that a version match is required when running Ginkgo on CI and/or locally [2691dd8] + +### Maintenance +- bump gomega to v1.22.0 [822a937] + +## 2.3.0 + +### Interruptible Nodes and Timeouts + +Ginkgo now supports per-node and per-spec timeouts on interruptible nodes. Check out the [documentation for all the details](https://onsi.github.io/ginkgo/#spec-timeouts-and-interruptible-nodes) but the gist is you can now write specs like this: + +```go +It("is interruptible", func(ctx SpecContext) { // or context.Context instead of SpecContext, both are valid. + // do things until `ctx.Done()` is closed, for example: + req, err := http.NewRequestWithContext(ctx, "POST", "/build-widgets", nil) + Expect(err).NotTo(HaveOccured()) + _, err := http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccured()) + + Eventually(client.WidgetCount).WithContext(ctx).Should(Equal(17)) +}, NodeTimeout(time.Second*20), GracePeriod(5*time.Second)) +``` + +and have Ginkgo ensure that the node completes before the timeout elapses. If it does elapse, or if an external interrupt is received (e.g. `^C`) then Ginkgo will cancel the context and wait for the Grace Period for the node to exit before proceeding with any cleanup nodes associated with the spec. The `ctx` provided by Ginkgo can also be passed down to Gomega's `Eventually` to have all assertions within the node governed by a single deadline. + +### Features + +- Ginkgo now records any additional failures that occur during the cleanup of a failed spec. In prior versions this information was quietly discarded, but the introduction of a more rigorous approach to timeouts and interruptions allows Ginkgo to better track subsequent failures. +- `SpecContext` also provides a mechanism for third-party libraries to provide additional information when a Progress Report is generated. Gomega uses this to provide the current state of an `Eventually().WithContext()` assertion when a Progress Report is requested. +- DescribeTable now exits with an error if it is not passed any Entries [a4c9865] + +## Fixes +- fixes crashes on newer Ruby 3 installations by upgrading github-pages gem dependency [92c88d5] +- Make the outline command able to use the DSL import [1be2427] + +## Maintenance +- chore(docs): delete no meaning d [57c373c] +- chore(docs): Fix hyperlinks [30526d5] +- chore(docs): fix code blocks without language settings [cf611c4] +- fix intra-doc link [b541bcb] + ## 2.2.0 ### Generate real-time Progress Reports [f91377c] diff --git a/vendor/github.com/onsi/ginkgo/v2/README.md b/vendor/github.com/onsi/ginkgo/v2/README.md index 58507c36f..d0473a467 100644 --- a/vendor/github.com/onsi/ginkgo/v2/README.md +++ b/vendor/github.com/onsi/ginkgo/v2/README.md @@ -4,11 +4,7 @@ --- -# Ginkgo 2.0 is now Generally Available! - -You can learn more about 2.0 in the [Migration Guide](https://onsi.github.io/ginkgo/MIGRATING_TO_V2)! - ---- +# Ginkgo Ginkgo is a mature testing framework for Go designed to help you write expressive specs. Ginkgo builds on top of Go's `testing` foundation and is complemented by the [Gomega](https://github.com/onsi/gomega) matcher library. Together, Ginkgo and Gomega let you express the intent behind your specs clearly: @@ -33,53 +29,53 @@ Describe("Checking books out of the library", Label("library"), func() { }) When("the library has the book in question", func() { - BeforeEach(func() { - Expect(library.Store(book)).To(Succeed()) + BeforeEach(func(ctx SpecContext) { + Expect(library.Store(ctx, book)).To(Succeed()) }) Context("and the book is available", func() { - It("lends it to the reader", func() { - Expect(valjean.Checkout(library, "Les Miserables")).To(Succeed()) + It("lends it to the reader", func(ctx SpecContext) { + Expect(valjean.Checkout(ctx, library, "Les Miserables")).To(Succeed()) Expect(valjean.Books()).To(ContainElement(book)) - Expect(library.UserWithBook(book)).To(Equal(valjean)) - }) + Expect(library.UserWithBook(ctx, book)).To(Equal(valjean)) + }, SpecTimeout(time.Second * 5)) }) Context("but the book has already been checked out", func() { var javert *users.User - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { javert = users.NewUser("Javert") - Expect(javert.Checkout(library, "Les Miserables")).To(Succeed()) + Expect(javert.Checkout(ctx, library, "Les Miserables")).To(Succeed()) }) - It("tells the user", func() { - err := valjean.Checkout(library, "Les Miserables") + It("tells the user", func(ctx SpecContext) { + err := valjean.Checkout(ctx, library, "Les Miserables") Expect(error).To(MatchError("Les Miserables is currently checked out")) - }) + }, SpecTimeout(time.Second * 5)) - It("lets the user place a hold and get notified later", func() { - Expect(valjean.Hold(library, "Les Miserables")).To(Succeed()) - Expect(valjean.Holds()).To(ContainElement(book)) + It("lets the user place a hold and get notified later", func(ctx SpecContext) { + Expect(valjean.Hold(ctx, library, "Les Miserables")).To(Succeed()) + Expect(valjean.Holds(ctx)).To(ContainElement(book)) By("when Javert returns the book") - Expect(javert.Return(library, book)).To(Succeed()) + Expect(javert.Return(ctx, library, book)).To(Succeed()) By("it eventually informs Valjean") notification := "Les Miserables is ready for pick up" - Eventually(valjean.Notifications).Should(ContainElement(notification)) + Eventually(ctx, valjean.Notifications).Should(ContainElement(notification)) - Expect(valjean.Checkout(library, "Les Miserables")).To(Succeed()) - Expect(valjean.Books()).To(ContainElement(book)) - Expect(valjean.Holds()).To(BeEmpty()) - }) + Expect(valjean.Checkout(ctx, library, "Les Miserables")).To(Succeed()) + Expect(valjean.Books(ctx)).To(ContainElement(book)) + Expect(valjean.Holds(ctx)).To(BeEmpty()) + }, SpecTimeout(time.Second * 10)) }) }) When("the library does not have the book in question", func() { - It("tells the reader the book is unavailable", func() { - err := valjean.Checkout(library, "Les Miserables") + It("tells the reader the book is unavailable", func(ctx SpecContext) { + err := valjean.Checkout(ctx, library, "Les Miserables") Expect(error).To(MatchError("Les Miserables is not in the library catalog")) - }) + }, SpecTimeout(time.Second * 5)) }) }) ``` @@ -90,9 +86,9 @@ If you have a question, comment, bug report, feature request, etc. please open a ## Capabilities -Whether writing basic unit specs, complex integration specs, or even performance specs - Ginkgo gives you an expressive Domain-Specific Language (DSL) that will be familiar to users coming from frameworks such as [Quick](https://github.com/Quick/Quick), [RSpec](https://rspec.info), [Jasmine](https://jasmine.github.io), and [Busted](https://olivinelabs.com/busted/). This style of testing is sometimes referred to as "Behavior-Driven Development" (BDD) though Ginkgo's utility extends beyond acceptance-level testing. +Whether writing basic unit specs, complex integration specs, or even performance specs - Ginkgo gives you an expressive Domain-Specific Language (DSL) that will be familiar to users coming from frameworks such as [Quick](https://github.com/Quick/Quick), [RSpec](https://rspec.info), [Jasmine](https://jasmine.github.io), and [Busted](https://lunarmodules.github.io/busted/). This style of testing is sometimes referred to as "Behavior-Driven Development" (BDD) though Ginkgo's utility extends beyond acceptance-level testing. -With Ginkgo's DSL you can use nestable [`Describe`, `Context` and `When` container nodes](https://onsi.github.io/ginkgo/#organizing-specs-with-container-nodes) to help you organize your specs. [`BeforeEach` and `AfterEach` setup nodes](https://onsi.github.io/ginkgo/#extracting-common-setup-beforeeach) for setup and cleanup. [`It` and `Specify` subject nodes](https://onsi.github.io/ginkgo/#spec-subjects-it) that hold your assertions. [`BeforeSuite` and `AfterSuite` nodes](https://onsi.github.io/ginkgo/#suite-setup-and-cleanup-beforesuite-and-aftersuite) to prep for and cleanup after a suite... and [much more!](https://onsi.github.io/ginkgo/#writing-specs) +With Ginkgo's DSL you can use nestable [`Describe`, `Context` and `When` container nodes](https://onsi.github.io/ginkgo/#organizing-specs-with-container-nodes) to help you organize your specs. [`BeforeEach` and `AfterEach` setup nodes](https://onsi.github.io/ginkgo/#extracting-common-setup-beforeeach) for setup and cleanup. [`It` and `Specify` subject nodes](https://onsi.github.io/ginkgo/#spec-subjects-it) that hold your assertions. [`BeforeSuite` and `AfterSuite` nodes](https://onsi.github.io/ginkgo/#suite-setup-and-cleanup-beforesuite-and-aftersuite) to prep for and cleanup after a suite... and [much more!](https://onsi.github.io/ginkgo/#writing-specs). At runtime, Ginkgo can run your specs in reproducibly [random order](https://onsi.github.io/ginkgo/#spec-randomization) and has sophisticated support for [spec parallelization](https://onsi.github.io/ginkgo/#spec-parallelization). In fact, running specs in parallel is as easy as @@ -100,7 +96,7 @@ At runtime, Ginkgo can run your specs in reproducibly [random order](https://ons ginkgo -p ``` -By following [established patterns for writing parallel specs](https://onsi.github.io/ginkgo/#patterns-for-parallel-integration-specs) you can build even large, complex integration suites that parallelize cleanly and run performantly. +By following [established patterns for writing parallel specs](https://onsi.github.io/ginkgo/#patterns-for-parallel-integration-specs) you can build even large, complex integration suites that parallelize cleanly and run performantly. And you don't have to worry about your spec suite hanging or leaving a mess behind - Ginkgo provides a per-node `context.Context` and the capability to interrupt the spec after a set period of time - and then clean up. As your suites grow Ginkgo helps you keep your specs organized with [labels](https://onsi.github.io/ginkgo/#spec-labels) and lets you easily run [subsets of specs](https://onsi.github.io/ginkgo/#filtering-specs), either [programmatically](https://onsi.github.io/ginkgo/#focused-specs) or on the [command line](https://onsi.github.io/ginkgo/#combining-filters). And Ginkgo's reporting infrastructure generates machine-readable output in a [variety of formats](https://onsi.github.io/ginkgo/#generating-machine-readable-reports) _and_ allows you to build your own [custom reporting infrastructure](https://onsi.github.io/ginkgo/#generating-reports-programmatically). diff --git a/vendor/github.com/onsi/ginkgo/v2/RELEASING.md b/vendor/github.com/onsi/ginkgo/v2/RELEASING.md index 0c80f668d..363815d7c 100644 --- a/vendor/github.com/onsi/ginkgo/v2/RELEASING.md +++ b/vendor/github.com/onsi/ginkgo/v2/RELEASING.md @@ -1,7 +1,13 @@ A Ginkgo release is a tagged git sha and a GitHub release. To cut a release: 1. Ensure CHANGELOG.md is up to date. - - Use `git log --pretty=format:'- %s [%h]' HEAD...vX.X.X` to list all the commits since the last release + - Use + ```bash + LAST_VERSION=$(git tag --sort=version:refname | tail -n1) + CHANGES=$(git log --pretty=format:'- %s [%h]' HEAD...$LAST_VERSION) + echo -e "## NEXT\n\n$CHANGES\n\n### Features\n\n### Fixes\n\n### Maintenance\n\n$(cat CHANGELOG.md)" > CHANGELOG.md + ``` + to update the changelog - Categorize the changes into - Breaking Changes (requires a major version) - New Features (minor version) diff --git a/vendor/github.com/onsi/ginkgo/v2/core_dsl.go b/vendor/github.com/onsi/ginkgo/v2/core_dsl.go index ff8d2790b..0b13b0d22 100644 --- a/vendor/github.com/onsi/ginkgo/v2/core_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/core_dsl.go @@ -21,8 +21,8 @@ import ( "os" "path/filepath" "strings" - "time" + "github.com/go-logr/logr" "github.com/onsi/ginkgo/v2/formatter" "github.com/onsi/ginkgo/v2/internal" "github.com/onsi/ginkgo/v2/internal/global" @@ -46,7 +46,9 @@ func init() { var err error flagSet, err = types.BuildTestSuiteFlagSet(&suiteConfig, &reporterConfig) exitIfErr(err) - GinkgoWriter = internal.NewWriter(os.Stdout) + writer := internal.NewWriter(os.Stdout) + GinkgoWriter = writer + GinkgoLogr = internal.GinkgoLogrFunc(writer) } func exitIfErr(err error) { @@ -77,7 +79,7 @@ func exitIfErrors(errors []error) { } } -//The interface implemented by GinkgoWriter +// The interface implemented by GinkgoWriter type GinkgoWriterInterface interface { io.Writer @@ -89,6 +91,15 @@ type GinkgoWriterInterface interface { ClearTeeWriters() } +/* +SpecContext is the context object passed into nodes that are subject to a timeout or need to be notified of an interrupt. It implements the standard context.Context interface but also contains additional helpers to provide an extensibility point for Ginkgo. (As an example, Gomega's Eventually can use the methods defined on SpecContext to provide deeper integration with Ginkgo). + +You can do anything with SpecContext that you do with a typical context.Context including wrapping it with any of the context.With* methods. + +Ginkgo will cancel the SpecContext when a node is interrupted (e.g. by the user sending an interrupt signal) or when a node has exceeded its allowed run-time. Note, however, that even in cases where a node has a deadline, SpecContext will not return a deadline via .Deadline(). This is because Ginkgo does not use a WithDeadline() context to model node deadlines as Ginkgo needs control over the precise timing of the context cancellation to ensure it can provide an accurate progress report at the moment of cancellation. +*/ +type SpecContext = internal.SpecContext + /* GinkgoWriter implements a GinkgoWriterInterface and io.Writer @@ -103,7 +114,12 @@ You can learn more at https://onsi.github.io/ginkgo/#logging-output */ var GinkgoWriter GinkgoWriterInterface -//The interface by which Ginkgo receives *testing.T +/* +GinkgoLogr is a logr.Logger that writes to GinkgoWriter +*/ +var GinkgoLogr logr.Logger + +// The interface by which Ginkgo receives *testing.T type GinkgoTestingT interface { Fail() } @@ -147,6 +163,29 @@ func GinkgoParallelProcess() int { return suiteConfig.ParallelProcess } +/* +GinkgoHelper marks the function it's called in as a test helper. When a failure occurs inside a helper function, Ginkgo will skip the helper when analyzing the stack trace to identify where the failure occurred. + +This is an alternative, simpler, mechanism to passing in a skip offset when calling Fail or using Gomega. +*/ +func GinkgoHelper() { + types.MarkAsHelper(1) +} + +/* +GinkgoLabelFilter() returns the label filter configured for this suite via `--label-filter`. + +You can use this to manually check if a set of labels would satisfy the filter via: + + if (Label("cat", "dog").MatchesLabelFilter(GinkgoLabelFilter())) { + //... + } +*/ +func GinkgoLabelFilter() string { + suiteConfig, _ := GinkgoConfiguration() + return suiteConfig.LabelFilter +} + /* PauseOutputInterception() pauses Ginkgo's output interception. This is only relevant when running in parallel and output to stdout/stderr is being intercepted. You generally @@ -168,7 +207,7 @@ func PauseOutputInterception() { outputInterceptor.PauseIntercepting() } -//ResumeOutputInterception() - see docs for PauseOutputInterception() +// ResumeOutputInterception() - see docs for PauseOutputInterception() func ResumeOutputInterception() { if outputInterceptor == nil { return @@ -259,7 +298,7 @@ func RunSpecs(t GinkgoTestingT, description string, args ...interface{}) bool { } writer := GinkgoWriter.(*internal.Writer) - if reporterConfig.Verbose && suiteConfig.ParallelTotal == 1 { + if reporterConfig.Verbosity().GTE(types.VerbosityLevelVerbose) && suiteConfig.ParallelTotal == 1 { writer.SetMode(internal.WriterModeStreamAndBuffer) } else { writer.SetMode(internal.WriterModeBufferOnly) @@ -277,7 +316,7 @@ func RunSpecs(t GinkgoTestingT, description string, args ...interface{}) bool { suitePath, err = filepath.Abs(suitePath) exitIfErr(err) - passed, hasFocusedTests := global.Suite.Run(description, suiteLabels, suitePath, global.Failer, reporter, writer, outputInterceptor, interrupt_handler.NewInterruptHandler(suiteConfig.Timeout, client), client, internal.RegisterForProgressSignal, suiteConfig) + passed, hasFocusedTests := global.Suite.Run(description, suiteLabels, suitePath, global.Failer, reporter, writer, outputInterceptor, interrupt_handler.NewInterruptHandler(client), client, internal.RegisterForProgressSignal, suiteConfig) outputInterceptor.Shutdown() flagSet.ValidateDeprecations(deprecationTracker) @@ -353,6 +392,12 @@ func AbortSuite(message string, callerSkip ...int) { panic(types.GinkgoErrors.UncaughtGinkgoPanic(cl)) } +/* +ignorablePanic is used by Gomega to signal to GinkgoRecover that Goemga is handling +the error associated with this panic. It i used when Eventually/Consistently are passed a func(g Gomega) and the resulting function launches a goroutines that makes a failed assertion. That failed assertion is registered by Gomega and then panics. Ordinarily the panic is captured by Gomega. In the case of a goroutine Gomega can't capture the panic - so we piggy back on GinkgoRecover so users have a single defer GinkgoRecover() pattern to follow. To do that we need to tell Ginkgo to ignore this panic and not register it as a panic on the global Failer. +*/ +type ignorablePanic interface{ GinkgoRecoverShouldIgnoreThisPanic() } + /* GinkgoRecover should be deferred at the top of any spawned goroutine that (may) call `Fail` Since Gomega assertions call fail, you should throw a `defer GinkgoRecover()` at the top of any goroutine that @@ -368,6 +413,9 @@ You can learn more about how Ginkgo manages failures here: https://onsi.github.i func GinkgoRecover() { e := recover() if e != nil { + if _, ok := e.(ignorablePanic); ok { + return + } global.Failer.Panic(types.NewCodeLocationWithStackTrace(1), e) } } @@ -444,6 +492,8 @@ It nodes are Subject nodes that contain your spec code and assertions. Each It node corresponds to an individual Ginkgo spec. You cannot nest any other Ginkgo nodes within an It node's closure. +You can pass It nodes bare functions (func() {}) or functions that receive a SpecContext or context.Context: func(ctx SpecContext) {} and func (ctx context.Context) {}. If the function takes a context then the It is deemed interruptible and Ginkgo will cancel the context in the event of a timeout (configured via the SpecTimeout() or NodeTimeout() decorators) or of an interrupt signal. + You can learn more at https://onsi.github.io/ginkgo/#spec-subjects-it In addition, subject nodes can be decorated with a variety of decorators. You can learn more here: https://onsi.github.io/ginkgo/#decorator-reference */ @@ -494,31 +544,7 @@ Note that By does not generate a new Ginkgo node - rather it is simply synctacti You can learn more about By here: https://onsi.github.io/ginkgo/#documenting-complex-specs-by */ func By(text string, callback ...func()) { - if !global.Suite.InRunPhase() { - exitIfErr(types.GinkgoErrors.ByNotDuringRunPhase(types.NewCodeLocation(1))) - } - value := struct { - Text string - Duration time.Duration - }{ - Text: text, - } - t := time.Now() - global.Suite.SetProgressStepCursor(internal.ProgressStepCursor{ - Text: text, - CodeLocation: types.NewCodeLocation(1), - StartTime: t, - }) - AddReportEntry("By Step", ReportEntryVisibilityNever, Offset(1), &value, t) - formatter := formatter.NewWithNoColorBool(reporterConfig.NoColor) - GinkgoWriter.Println(formatter.F("{{bold}}STEP:{{/}} %s {{gray}}%s{{/}}", text, t.Format(types.GINKGO_TIME_FORMAT))) - if len(callback) == 1 { - callback[0]() - value.Duration = time.Since(t) - } - if len(callback) > 1 { - panic("just one callback per By, please") - } + exitIfErr(global.Suite.By(text, callback...)) } /* @@ -527,10 +553,12 @@ When running in parallel, each parallel process will call BeforeSuite. You may only register *one* BeforeSuite handler per test suite. You typically do so in your bootstrap file at the top level. +BeforeSuite can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within a BeforeSuite node's closure. You can learn more here: https://onsi.github.io/ginkgo/#suite-setup-and-cleanup-beforesuite-and-aftersuite */ -func BeforeSuite(body func(), args ...interface{}) bool { +func BeforeSuite(body interface{}, args ...interface{}) bool { combinedArgs := []interface{}{body} combinedArgs = append(combinedArgs, args...) return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeBeforeSuite, "", combinedArgs...)) @@ -544,10 +572,12 @@ When running in parallel, each parallel process will call AfterSuite. You may only register *one* AfterSuite handler per test suite. You typically do so in your bootstrap file at the top level. +AfterSuite can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within an AfterSuite node's closure. You can learn more here: https://onsi.github.io/ginkgo/#suite-setup-and-cleanup-beforesuite-and-aftersuite */ -func AfterSuite(body func(), args ...interface{}) bool { +func AfterSuite(body interface{}, args ...interface{}) bool { combinedArgs := []interface{}{body} combinedArgs = append(combinedArgs, args...) return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeAfterSuite, "", combinedArgs...)) @@ -561,18 +591,30 @@ information from that setup to all parallel processes. SynchronizedBeforeSuite accomplishes this by taking *two* function arguments and passing data between them. The first function is only run on parallel process #1. The second is run on all processes, but *only* after the first function completes successfully. The functions have the following signatures: -The first function (which only runs on process #1) has the signature: +The first function (which only runs on process #1) can have any of the following the signatures: + func() + func(ctx context.Context) + func(ctx SpecContext) func() []byte + func(ctx context.Context) []byte + func(ctx SpecContext) []byte -The byte array returned by the first function is then passed to the second function, which has the signature: +The byte array returned by the first function (if present) is then passed to the second function, which can have any of the following signature: + func() + func(ctx context.Context) + func(ctx SpecContext) func(data []byte) + func(ctx context.Context, data []byte) + func(ctx SpecContext, data []byte) + +If either function receives a context.Context/SpecContext it is considered interruptible. You cannot nest any other Ginkgo nodes within an SynchronizedBeforeSuite node's closure. You can learn more, and see some examples, here: https://onsi.github.io/ginkgo/#parallel-suite-setup-and-cleanup-synchronizedbeforesuite-and-synchronizedaftersuite */ -func SynchronizedBeforeSuite(process1Body func() []byte, allProcessBody func([]byte), args ...interface{}) bool { +func SynchronizedBeforeSuite(process1Body interface{}, allProcessBody interface{}, args ...interface{}) bool { combinedArgs := []interface{}{process1Body, allProcessBody} combinedArgs = append(combinedArgs, args...) @@ -585,14 +627,14 @@ and a piece that must only run once - on process #1. SynchronizedAfterSuite accomplishes this by taking *two* function arguments. The first runs on all processes. The second runs only on parallel process #1 and *only* after all other processes have finished and exited. This ensures that process #1, and any resources it is managing, remain alive until -all other processes are finished. +all other processes are finished. These two functions can be bare functions (func()) or interruptible (func(context.Context)/func(SpecContext)) Note that you can also use DeferCleanup() in SynchronizedBeforeSuite to accomplish similar results. You cannot nest any other Ginkgo nodes within an SynchronizedAfterSuite node's closure. You can learn more, and see some examples, here: https://onsi.github.io/ginkgo/#parallel-suite-setup-and-cleanup-synchronizedbeforesuite-and-synchronizedaftersuite */ -func SynchronizedAfterSuite(allProcessBody func(), process1Body func(), args ...interface{}) bool { +func SynchronizedAfterSuite(allProcessBody interface{}, process1Body interface{}, args ...interface{}) bool { combinedArgs := []interface{}{allProcessBody, process1Body} combinedArgs = append(combinedArgs, args...) @@ -603,6 +645,8 @@ func SynchronizedAfterSuite(allProcessBody func(), process1Body func(), args ... BeforeEach nodes are Setup nodes whose closures run before It node closures. When multiple BeforeEach nodes are defined in nested Container nodes the outermost BeforeEach node closures are run first. +BeforeEach can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within a BeforeEach node's closure. You can learn more here: https://onsi.github.io/ginkgo/#extracting-common-setup-beforeeach */ @@ -614,6 +658,8 @@ func BeforeEach(args ...interface{}) bool { JustBeforeEach nodes are similar to BeforeEach nodes, however they are guaranteed to run *after* all BeforeEach node closures - just before the It node closure. This can allow you to separate configuration from creation of resources for a spec. +JustBeforeEach can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within a JustBeforeEach node's closure. You can learn more and see some examples here: https://onsi.github.io/ginkgo/#separating-creation-and-configuration-justbeforeeach */ @@ -627,6 +673,8 @@ are defined in nested Container nodes the innermost AfterEach node closures are Note that you can also use DeferCleanup() in other Setup or Subject nodes to accomplish similar results. +AfterEach can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within an AfterEach node's closure. You can learn more here: https://onsi.github.io/ginkgo/#spec-cleanup-aftereach-and-defercleanup */ @@ -637,6 +685,8 @@ func AfterEach(args ...interface{}) bool { /* JustAfterEach nodes are similar to AfterEach nodes, however they are guaranteed to run *before* all AfterEach node closures - just after the It node closure. This can allow you to separate diagnostics collection from teardown for a spec. +JustAfterEach can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within a JustAfterEach node's closure. You can learn more and see some examples here: https://onsi.github.io/ginkgo/#separating-diagnostics-collection-and-teardown-justaftereach */ @@ -649,6 +699,8 @@ BeforeAll nodes are Setup nodes that can occur inside Ordered containers. They Multiple BeforeAll nodes can be defined in a given Ordered container however they cannot be nested inside any other container. +BeforeAll can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within a BeforeAll node's closure. You can learn more about Ordered Containers at: https://onsi.github.io/ginkgo/#ordered-containers And you can learn more about BeforeAll at: https://onsi.github.io/ginkgo/#setup-in-ordered-containers-beforeall-and-afterall @@ -664,6 +716,8 @@ Multiple AfterAll nodes can be defined in a given Ordered container however they Note that you can also use DeferCleanup() in a BeforeAll node to accomplish similar behavior. +AfterAll can take a func() body, or an interruptible func(SpecContext)/func(context.Context) body. + You cannot nest any other Ginkgo nodes within an AfterAll node's closure. You can learn more about Ordered Containers at: https://onsi.github.io/ginkgo/#ordered-containers And you can learn more about AfterAll at: https://onsi.github.io/ginkgo/#setup-in-ordered-containers-beforeall-and-afterall @@ -677,15 +731,32 @@ DeferCleanup can be called within any Setup or Subject node to register a cleanu DeferCleanup can be passed: 1. A function that takes no arguments and returns no values. -2. A function that returns an error (in which case it will assert that the returned error was nil, or it will fail the spec). -3. A function that takes arguments (and optionally returns an error) followed by a list of arguments to passe to the function. For example: +2. A function that returns multiple values. `DeferCleanup` will ignore all these return values except for the last one. If this last return value is a non-nil error `DeferCleanup` will fail the spec). +3. A function that takes a context.Context or SpecContext (and optionally returns multiple values). The resulting cleanup node is deemed interruptible and the passed-in context will be cancelled in the event of a timeout or interrupt. +4. A function that takes arguments (and optionally returns multiple values) followed by a list of arguments to pass to the function. +5. A function that takes SpecContext and a list of arguments (and optionally returns multiple values) followed by a list of arguments to pass to the function. + +For example: - BeforeEach(func() { - DeferCleanup(os.SetEnv, "FOO", os.GetEnv("FOO")) - os.SetEnv("FOO", "BAR") - }) + BeforeEach(func() { + DeferCleanup(os.SetEnv, "FOO", os.GetEnv("FOO")) + os.SetEnv("FOO", "BAR") + }) + +will register a cleanup handler that will set the environment variable "FOO" to its current value (obtained by os.GetEnv("FOO")) after the spec runs and then sets the environment variable "FOO" to "BAR" for the current spec. + +Similarly: + + BeforeEach(func() { + DeferCleanup(func(ctx SpecContext, path) { + req, err := http.NewRequestWithContext(ctx, "POST", path, nil) + Expect(err).NotTo(HaveOccured()) + _, err := http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccured()) + }, "example.com/cleanup", NodeTimeout(time.Second*3)) + }) -will register a cleanup handler that will set the environment variable "FOO" to it's current value (obtained by os.GetEnv("FOO")) after the spec runs and then sets the environment variable "FOO" to "BAR" for the current spec. +will register a cleanup handler that will have three seconds to successfully complete a request to the specified path. Note that we do not specify a context in the list of arguments passed to DeferCleanup - only in the signature of the function we pass in. Ginkgo will detect the requested context and supply a SpecContext when it invokes the cleanup node. If you want to pass in your own context in addition to the Ginkgo-provided SpecContext you must specify the SpecContext as the first argument (e.g. func(ctx SpecContext, otherCtx context.Context)). When DeferCleanup is called in BeforeEach, JustBeforeEach, It, AfterEach, or JustAfterEach the registered callback will be invoked when the spec completes (i.e. it will behave like an AfterEach node) When DeferCleanup is called in BeforeAll or AfterAll the registered callback will be invoked when the ordered container completes (i.e. it will behave like an AfterAll node) @@ -698,5 +769,5 @@ func DeferCleanup(args ...interface{}) { fail := func(message string, cl types.CodeLocation) { global.Failer.Fail(message, cl) } - pushNode(internal.NewCleanupNode(fail, args...)) + pushNode(internal.NewCleanupNode(deprecationTracker, fail, args...)) } diff --git a/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go b/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go index a31a2ee6d..c65af4ce1 100644 --- a/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go @@ -13,13 +13,21 @@ You can learn more about decorators here: https://onsi.github.io/ginkgo/#decorat type Offset = internal.Offset /* -FlakeAttempts(uint N) is a decorator that allows you to mark individual specs or spec containers as flaky. Ginkgo will run them up to `N` times until they pass. +FlakeAttempts(uint N) is a decorator that allows you to mark individual specs or spec containers as flaky. Ginkgo will run them up to `N` times until they pass. -You can learn more here: https://onsi.github.io/ginkgo/#repeating-spec-runs-and-managing-flaky-specs +You can learn more here: https://onsi.github.io/ginkgo/#the-flakeattempts-decorator You can learn more about decorators here: https://onsi.github.io/ginkgo/#decorator-reference */ type FlakeAttempts = internal.FlakeAttempts +/* +MustPassRepeatedly(uint N) is a decorator that allows you to repeat the execution of individual specs or spec containers. Ginkgo will run them up to `N` times until they fail. + +You can learn more here: https://onsi.github.io/ginkgo/#the-mustpassrepeatedly-decorator +You can learn more about decorators here: https://onsi.github.io/ginkgo/#decorator-reference +*/ +type MustPassRepeatedly = internal.MustPassRepeatedly + /* Focus is a decorator that allows you to mark a spec or container as focused. Identical to FIt and FDescribe. @@ -38,7 +46,7 @@ const Pending = internal.Pending /* Serial is a decorator that allows you to mark a spec or container as serial. These specs will never run in parallel with other specs. -Tests in ordered containers cannot be marked as serial - mark the ordered container instead. +Specs in ordered containers cannot be marked as serial - mark the ordered container instead. You can learn more here: https://onsi.github.io/ginkgo/#serial-specs You can learn more about decorators here: https://onsi.github.io/ginkgo/#decorator-reference @@ -46,7 +54,7 @@ You can learn more about decorators here: https://onsi.github.io/ginkgo/#decorat const Serial = internal.Serial /* -Ordered is a decorator that allows you to mark a container as ordered. Tests in the container will always run in the order they appear. +Ordered is a decorator that allows you to mark a container as ordered. Specs in the container will always run in the order they appear. They will never be randomized and they will never run in parallel with one another, though they may run in parallel with other specs. You can learn more here: https://onsi.github.io/ginkgo/#ordered-containers @@ -54,6 +62,16 @@ You can learn more about decorators here: https://onsi.github.io/ginkgo/#decorat */ const Ordered = internal.Ordered +/* +ContinueOnFailure is a decorator that allows you to mark an Ordered container to continue running specs even if failures occur. Ordinarily an ordered container will stop running specs after the first failure occurs. Note that if a BeforeAll or a BeforeEach/JustBeforeEach annotated with OncePerOrdered fails then no specs will run as the precondition for the Ordered container will consider to be failed. + +ContinueOnFailure only applies to the outermost Ordered container. Attempting to place ContinueOnFailure in a nested container will result in an error. + +You can learn more here: https://onsi.github.io/ginkgo/#ordered-containers +You can learn more about decorators here: https://onsi.github.io/ginkgo/#decorator-reference +*/ +const ContinueOnFailure = internal.ContinueOnFailure + /* OncePerOrdered is a decorator that allows you to mark outer BeforeEach, AfterEach, JustBeforeEach, and JustAfterEach setup nodes to run once per ordered context. Normally these setup nodes run around each individual spec, with OncePerOrdered they will run once around the set of specs in an ordered container. @@ -95,6 +113,29 @@ Once a node has been running for longer than PollProgressAfter Ginkgo will emit */ type PollProgressInterval = internal.PollProgressInterval +/* +NodeTimeout allows you to specify a timeout for an indivdiual node. The node cannot be a container and must be interruptible (i.e. it must be passed a function that accepts a SpecContext or context.Context). + +If the node does not exit within the specified NodeTimeout its context will be cancelled. The node wil then have a period of time controlled by the GracePeriod decorator (or global --grace-period command-line argument) to exit. If the node does not exit within GracePeriod Ginkgo will leak the node and proceed to any clean-up nodes associated with the current spec. +*/ +type NodeTimeout = internal.NodeTimeout + +/* +SpecTimeout allows you to specify a timeout for an indivdiual spec. SpecTimeout can only decorate interruptible It nodes. + +All nodes associated with the It node will need to complete before the SpecTimeout has elapsed. Individual nodes (e.g. BeforeEach) may be decorated with different NodeTimeouts - but these can only serve to provide a more stringent deadline for the node in question; they cannot extend the deadline past the SpecTimeout. + +If the spec does not complete within the specified SpecTimeout the currently running node will have its context cancelled. The node wil then have a period of time controlled by that node's GracePeriod decorator (or global --grace-period command-line argument) to exit. If the node does not exit within GracePeriod Ginkgo will leak the node and proceed to any clean-up nodes associated with the current spec. +*/ +type SpecTimeout = internal.SpecTimeout + +/* +GracePeriod denotes the period of time Ginkgo will wait for an interruptible node to exit once an interruption (whether due to a timeout or a user-invoked signal) has occurred. If both the global --grace-period cli flag and a GracePeriod decorator are specified the value in the decorator will take precedence. + +Nodes that do not finish within a GracePeriod will be leaked and Ginkgo will proceed to run subsequent nodes. In the event of a timeout, such leaks will be reported to the user. +*/ +type GracePeriod = internal.GracePeriod + /* SuppressProgressReporting is a decorator that allows you to disable progress reporting of a particular node. This is useful if `ginkgo -v -progress` is generating too much noise; particularly if you have a `ReportAfterEach` node that is running for every skipped spec and is generating lots of progress reports. diff --git a/vendor/github.com/onsi/ginkgo/v2/formatter/formatter.go b/vendor/github.com/onsi/ginkgo/v2/formatter/formatter.go index 43b16211d..743555dde 100644 --- a/vendor/github.com/onsi/ginkgo/v2/formatter/formatter.go +++ b/vendor/github.com/onsi/ginkgo/v2/formatter/formatter.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "regexp" + "strconv" "strings" ) @@ -50,6 +51,37 @@ func NewWithNoColorBool(noColor bool) Formatter { } func New(colorMode ColorMode) Formatter { + colorAliases := map[string]int{ + "black": 0, + "red": 1, + "green": 2, + "yellow": 3, + "blue": 4, + "magenta": 5, + "cyan": 6, + "white": 7, + } + for colorAlias, n := range colorAliases { + colorAliases[fmt.Sprintf("bright-%s", colorAlias)] = n + 8 + } + + getColor := func(color, defaultEscapeCode string) string { + color = strings.ToUpper(strings.ReplaceAll(color, "-", "_")) + envVar := fmt.Sprintf("GINKGO_CLI_COLOR_%s", color) + envVarColor := os.Getenv(envVar) + if envVarColor == "" { + return defaultEscapeCode + } + if colorCode, ok := colorAliases[envVarColor]; ok { + return fmt.Sprintf("\x1b[38;5;%dm", colorCode) + } + colorCode, err := strconv.Atoi(envVarColor) + if err != nil || colorCode < 0 || colorCode > 255 { + return defaultEscapeCode + } + return fmt.Sprintf("\x1b[38;5;%dm", colorCode) + } + f := Formatter{ ColorMode: colorMode, colors: map[string]string{ @@ -57,18 +89,18 @@ func New(colorMode ColorMode) Formatter { "bold": "\x1b[1m", "underline": "\x1b[4m", - "red": "\x1b[38;5;9m", - "orange": "\x1b[38;5;214m", - "coral": "\x1b[38;5;204m", - "magenta": "\x1b[38;5;13m", - "green": "\x1b[38;5;10m", - "dark-green": "\x1b[38;5;28m", - "yellow": "\x1b[38;5;11m", - "light-yellow": "\x1b[38;5;228m", - "cyan": "\x1b[38;5;14m", - "gray": "\x1b[38;5;243m", - "light-gray": "\x1b[38;5;246m", - "blue": "\x1b[38;5;12m", + "red": getColor("red", "\x1b[38;5;9m"), + "orange": getColor("orange", "\x1b[38;5;214m"), + "coral": getColor("coral", "\x1b[38;5;204m"), + "magenta": getColor("magenta", "\x1b[38;5;13m"), + "green": getColor("green", "\x1b[38;5;10m"), + "dark-green": getColor("dark-green", "\x1b[38;5;28m"), + "yellow": getColor("yellow", "\x1b[38;5;11m"), + "light-yellow": getColor("light-yellow", "\x1b[38;5;228m"), + "cyan": getColor("cyan", "\x1b[38;5;14m"), + "gray": getColor("gray", "\x1b[38;5;243m"), + "light-gray": getColor("light-gray", "\x1b[38;5;246m"), + "blue": getColor("blue", "\x1b[38;5;12m"), }, } colors := []string{} @@ -88,7 +120,10 @@ func (f Formatter) Fi(indentation uint, format string, args ...interface{}) stri } func (f Formatter) Fiw(indentation uint, maxWidth uint, format string, args ...interface{}) string { - out := fmt.Sprintf(f.style(format), args...) + out := f.style(format) + if len(args) > 0 { + out = fmt.Sprintf(out, args...) + } if indentation == 0 && maxWidth == 0 { return out diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/build/build_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/build/build_command.go new file mode 100644 index 000000000..5db5d1a7b --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/build/build_command.go @@ -0,0 +1,63 @@ +package build + +import ( + "fmt" + + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/ginkgo/internal" + "github.com/onsi/ginkgo/v2/types" +) + +func BuildBuildCommand() command.Command { + var cliConfig = types.NewDefaultCLIConfig() + var goFlagsConfig = types.NewDefaultGoFlagsConfig() + + flags, err := types.BuildBuildCommandFlagSet(&cliConfig, &goFlagsConfig) + if err != nil { + panic(err) + } + + return command.Command{ + Name: "build", + Flags: flags, + Usage: "ginkgo build ", + ShortDoc: "Build the passed in (or the package in the current directory if left blank).", + DocLink: "precompiling-suites", + Command: func(args []string, _ []string) { + var errors []error + cliConfig, goFlagsConfig, errors = types.VetAndInitializeCLIAndGoConfig(cliConfig, goFlagsConfig) + command.AbortIfErrors("Ginkgo detected configuration issues:", errors) + + buildSpecs(args, cliConfig, goFlagsConfig) + }, + } +} + +func buildSpecs(args []string, cliConfig types.CLIConfig, goFlagsConfig types.GoFlagsConfig) { + suites := internal.FindSuites(args, cliConfig, false).WithoutState(internal.TestSuiteStateSkippedByFilter) + if len(suites) == 0 { + command.AbortWith("Found no test suites") + } + + internal.VerifyCLIAndFrameworkVersion(suites) + + opc := internal.NewOrderedParallelCompiler(cliConfig.ComputedNumCompilers()) + opc.StartCompiling(suites, goFlagsConfig) + + for { + suiteIdx, suite := opc.Next() + if suiteIdx >= len(suites) { + break + } + suites[suiteIdx] = suite + if suite.State.Is(internal.TestSuiteStateFailedToCompile) { + fmt.Println(suite.CompilationError.Error()) + } else { + fmt.Printf("Compiled %s.test\n", suite.PackageName) + } + } + + if suites.CountWithState(internal.TestSuiteStateFailedToCompile) > 0 { + command.AbortWith("Failed to compile all tests") + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/abort.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/abort.go new file mode 100644 index 000000000..2efd28608 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/abort.go @@ -0,0 +1,61 @@ +package command + +import "fmt" + +type AbortDetails struct { + ExitCode int + Error error + EmitUsage bool +} + +func Abort(details AbortDetails) { + panic(details) +} + +func AbortGracefullyWith(format string, args ...interface{}) { + Abort(AbortDetails{ + ExitCode: 0, + Error: fmt.Errorf(format, args...), + EmitUsage: false, + }) +} + +func AbortWith(format string, args ...interface{}) { + Abort(AbortDetails{ + ExitCode: 1, + Error: fmt.Errorf(format, args...), + EmitUsage: false, + }) +} + +func AbortWithUsage(format string, args ...interface{}) { + Abort(AbortDetails{ + ExitCode: 1, + Error: fmt.Errorf(format, args...), + EmitUsage: true, + }) +} + +func AbortIfError(preamble string, err error) { + if err != nil { + Abort(AbortDetails{ + ExitCode: 1, + Error: fmt.Errorf("%s\n%s", preamble, err.Error()), + EmitUsage: false, + }) + } +} + +func AbortIfErrors(preamble string, errors []error) { + if len(errors) > 0 { + out := "" + for _, err := range errors { + out += err.Error() + } + Abort(AbortDetails{ + ExitCode: 1, + Error: fmt.Errorf("%s\n%s", preamble, out), + EmitUsage: false, + }) + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/command.go new file mode 100644 index 000000000..12e0e5659 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/command.go @@ -0,0 +1,50 @@ +package command + +import ( + "fmt" + "io" + "strings" + + "github.com/onsi/ginkgo/v2/formatter" + "github.com/onsi/ginkgo/v2/types" +) + +type Command struct { + Name string + Flags types.GinkgoFlagSet + Usage string + ShortDoc string + Documentation string + DocLink string + Command func(args []string, additionalArgs []string) +} + +func (c Command) Run(args []string, additionalArgs []string) { + args, err := c.Flags.Parse(args) + if err != nil { + AbortWithUsage(err.Error()) + } + + c.Command(args, additionalArgs) +} + +func (c Command) EmitUsage(writer io.Writer) { + fmt.Fprintln(writer, formatter.F("{{bold}}"+c.Usage+"{{/}}")) + fmt.Fprintln(writer, formatter.F("{{gray}}%s{{/}}", strings.Repeat("-", len(c.Usage)))) + if c.ShortDoc != "" { + fmt.Fprintln(writer, formatter.Fiw(0, formatter.COLS, c.ShortDoc)) + fmt.Fprintln(writer, "") + } + if c.Documentation != "" { + fmt.Fprintln(writer, formatter.Fiw(0, formatter.COLS, c.Documentation)) + fmt.Fprintln(writer, "") + } + if c.DocLink != "" { + fmt.Fprintln(writer, formatter.Fi(0, "{{bold}}Learn more at:{{/}} {{cyan}}{{underline}}http://onsi.github.io/ginkgo/#%s{{/}}", c.DocLink)) + fmt.Fprintln(writer, "") + } + flagUsage := c.Flags.Usage() + if flagUsage != "" { + fmt.Fprintf(writer, formatter.F(flagUsage)) + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/program.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/program.go new file mode 100644 index 000000000..88dd8d6b0 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/command/program.go @@ -0,0 +1,182 @@ +package command + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/onsi/ginkgo/v2/formatter" + "github.com/onsi/ginkgo/v2/types" +) + +type Program struct { + Name string + Heading string + Commands []Command + DefaultCommand Command + DeprecatedCommands []DeprecatedCommand + + //For testing - leave as nil in production + OutWriter io.Writer + ErrWriter io.Writer + Exiter func(code int) +} + +type DeprecatedCommand struct { + Name string + Deprecation types.Deprecation +} + +func (p Program) RunAndExit(osArgs []string) { + var command Command + deprecationTracker := types.NewDeprecationTracker() + if p.Exiter == nil { + p.Exiter = os.Exit + } + if p.OutWriter == nil { + p.OutWriter = formatter.ColorableStdOut + } + if p.ErrWriter == nil { + p.ErrWriter = formatter.ColorableStdErr + } + + defer func() { + exitCode := 0 + + if r := recover(); r != nil { + details, ok := r.(AbortDetails) + if !ok { + panic(r) + } + + if details.Error != nil { + fmt.Fprintln(p.ErrWriter, formatter.F("{{red}}{{bold}}%s %s{{/}} {{red}}failed{{/}}", p.Name, command.Name)) + fmt.Fprintln(p.ErrWriter, formatter.Fi(1, details.Error.Error())) + } + if details.EmitUsage { + if details.Error != nil { + fmt.Fprintln(p.ErrWriter, "") + } + command.EmitUsage(p.ErrWriter) + } + exitCode = details.ExitCode + } + + command.Flags.ValidateDeprecations(deprecationTracker) + if deprecationTracker.DidTrackDeprecations() { + fmt.Fprintln(p.ErrWriter, deprecationTracker.DeprecationsReport()) + } + p.Exiter(exitCode) + return + }() + + args, additionalArgs := []string{}, []string{} + + foundDelimiter := false + for _, arg := range osArgs[1:] { + if !foundDelimiter { + if arg == "--" { + foundDelimiter = true + continue + } + } + + if foundDelimiter { + additionalArgs = append(additionalArgs, arg) + } else { + args = append(args, arg) + } + } + + command = p.DefaultCommand + if len(args) > 0 { + p.handleHelpRequestsAndExit(p.OutWriter, args) + if command.Name == args[0] { + args = args[1:] + } else { + for _, deprecatedCommand := range p.DeprecatedCommands { + if deprecatedCommand.Name == args[0] { + deprecationTracker.TrackDeprecation(deprecatedCommand.Deprecation) + return + } + } + for _, tryCommand := range p.Commands { + if tryCommand.Name == args[0] { + command, args = tryCommand, args[1:] + break + } + } + } + } + + command.Run(args, additionalArgs) +} + +func (p Program) handleHelpRequestsAndExit(writer io.Writer, args []string) { + if len(args) == 0 { + return + } + + matchesHelpFlag := func(args ...string) bool { + for _, arg := range args { + if arg == "--help" || arg == "-help" || arg == "-h" || arg == "--h" { + return true + } + } + return false + } + if len(args) == 1 { + if args[0] == "help" || matchesHelpFlag(args[0]) { + p.EmitUsage(writer) + Abort(AbortDetails{}) + } + } else { + var name string + if args[0] == "help" || matchesHelpFlag(args[0]) { + name = args[1] + } else if matchesHelpFlag(args[1:]...) { + name = args[0] + } else { + return + } + + if p.DefaultCommand.Name == name || p.Name == name { + p.DefaultCommand.EmitUsage(writer) + Abort(AbortDetails{}) + } + for _, command := range p.Commands { + if command.Name == name { + command.EmitUsage(writer) + Abort(AbortDetails{}) + } + } + + fmt.Fprintln(writer, formatter.F("{{red}}Unknown Command: {{bold}}%s{{/}}", name)) + fmt.Fprintln(writer, "") + p.EmitUsage(writer) + Abort(AbortDetails{ExitCode: 1}) + } + return +} + +func (p Program) EmitUsage(writer io.Writer) { + fmt.Fprintln(writer, formatter.F(p.Heading)) + fmt.Fprintln(writer, formatter.F("{{gray}}%s{{/}}", strings.Repeat("-", len(p.Heading)))) + fmt.Fprintln(writer, formatter.F("For usage information for a command, run {{bold}}%s help COMMAND{{/}}.", p.Name)) + fmt.Fprintln(writer, formatter.F("For usage information for the default command, run {{bold}}%s help %s{{/}} or {{bold}}%s help %s{{/}}.", p.Name, p.Name, p.Name, p.DefaultCommand.Name)) + fmt.Fprintln(writer, "") + fmt.Fprintln(writer, formatter.F("The following commands are available:")) + + fmt.Fprintln(writer, formatter.Fi(1, "{{bold}}%s{{/}} or %s {{bold}}%s{{/}} - {{gray}}%s{{/}}", p.Name, p.Name, p.DefaultCommand.Name, p.DefaultCommand.Usage)) + if p.DefaultCommand.ShortDoc != "" { + fmt.Fprintln(writer, formatter.Fi(2, p.DefaultCommand.ShortDoc)) + } + + for _, command := range p.Commands { + fmt.Fprintln(writer, formatter.Fi(1, "{{bold}}%s{{/}} - {{gray}}%s{{/}}", command.Name, command.Usage)) + if command.ShortDoc != "" { + fmt.Fprintln(writer, formatter.Fi(2, command.ShortDoc)) + } + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/boostrap_templates.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/boostrap_templates.go new file mode 100644 index 000000000..a367a1fc9 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/boostrap_templates.go @@ -0,0 +1,48 @@ +package generators + +var bootstrapText = `package {{.Package}} + +import ( + "testing" + + {{.GinkgoImport}} + {{.GomegaImport}} +) + +func Test{{.FormattedName}}(t *testing.T) { + {{.GomegaPackage}}RegisterFailHandler({{.GinkgoPackage}}Fail) + {{.GinkgoPackage}}RunSpecs(t, "{{.FormattedName}} Suite") +} +` + +var agoutiBootstrapText = `package {{.Package}} + +import ( + "testing" + + {{.GinkgoImport}} + {{.GomegaImport}} + "github.com/sclevine/agouti" +) + +func Test{{.FormattedName}}(t *testing.T) { + {{.GomegaPackage}}RegisterFailHandler({{.GinkgoPackage}}Fail) + {{.GinkgoPackage}}RunSpecs(t, "{{.FormattedName}} Suite") +} + +var agoutiDriver *agouti.WebDriver + +var _ = {{.GinkgoPackage}}BeforeSuite(func() { + // Choose a WebDriver: + + agoutiDriver = agouti.PhantomJS() + // agoutiDriver = agouti.Selenium() + // agoutiDriver = agouti.ChromeDriver() + + {{.GomegaPackage}}Expect(agoutiDriver.Start()).To({{.GomegaPackage}}Succeed()) +}) + +var _ = {{.GinkgoPackage}}AfterSuite(func() { + {{.GomegaPackage}}Expect(agoutiDriver.Stop()).To({{.GomegaPackage}}Succeed()) +}) +` diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/bootstrap_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/bootstrap_command.go new file mode 100644 index 000000000..73aff0b7a --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/bootstrap_command.go @@ -0,0 +1,133 @@ +package generators + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "text/template" + + sprig "github.com/go-task/slim-sprig" + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/ginkgo/internal" + "github.com/onsi/ginkgo/v2/types" +) + +func BuildBootstrapCommand() command.Command { + conf := GeneratorsConfig{} + flags, err := types.NewGinkgoFlagSet( + types.GinkgoFlags{ + {Name: "agouti", KeyPath: "Agouti", + Usage: "If set, bootstrap will generate a bootstrap file for writing Agouti tests"}, + {Name: "nodot", KeyPath: "NoDot", + Usage: "If set, bootstrap will generate a bootstrap test file that does not dot-import ginkgo and gomega"}, + {Name: "internal", KeyPath: "Internal", + Usage: "If set, bootstrap will generate a bootstrap test file that uses the regular package name (i.e. `package X`, not `package X_test`)"}, + {Name: "template", KeyPath: "CustomTemplate", + UsageArgument: "template-file", + Usage: "If specified, generate will use the contents of the file passed as the bootstrap template"}, + {Name: "template-data", KeyPath: "CustomTemplateData", + UsageArgument: "template-data-file", + Usage: "If specified, generate will use the contents of the file passed as data to be rendered in the bootstrap template"}, + }, + &conf, + types.GinkgoFlagSections{}, + ) + + if err != nil { + panic(err) + } + + return command.Command{ + Name: "bootstrap", + Usage: "ginkgo bootstrap", + ShortDoc: "Bootstrap a test suite for the current package", + Documentation: `Tests written in Ginkgo and Gomega require a small amount of boilerplate to hook into Go's testing infrastructure. + +{{bold}}ginkgo bootstrap{{/}} generates this boilerplate for you in a file named X_suite_test.go where X is the name of the package under test.`, + DocLink: "generators", + Flags: flags, + Command: func(_ []string, _ []string) { + generateBootstrap(conf) + }, + } +} + +type bootstrapData struct { + Package string + FormattedName string + + GinkgoImport string + GomegaImport string + GinkgoPackage string + GomegaPackage string + CustomData map[string]any +} + +func generateBootstrap(conf GeneratorsConfig) { + packageName, bootstrapFilePrefix, formattedName := getPackageAndFormattedName() + + data := bootstrapData{ + Package: determinePackageName(packageName, conf.Internal), + FormattedName: formattedName, + + GinkgoImport: `. "github.com/onsi/ginkgo/v2"`, + GomegaImport: `. "github.com/onsi/gomega"`, + GinkgoPackage: "", + GomegaPackage: "", + } + + if conf.NoDot { + data.GinkgoImport = `"github.com/onsi/ginkgo/v2"` + data.GomegaImport = `"github.com/onsi/gomega"` + data.GinkgoPackage = `ginkgo.` + data.GomegaPackage = `gomega.` + } + + targetFile := fmt.Sprintf("%s_suite_test.go", bootstrapFilePrefix) + if internal.FileExists(targetFile) { + command.AbortWith("{{bold}}%s{{/}} already exists", targetFile) + } else { + fmt.Printf("Generating ginkgo test suite bootstrap for %s in:\n\t%s\n", packageName, targetFile) + } + + f, err := os.Create(targetFile) + command.AbortIfError("Failed to create file:", err) + defer f.Close() + + var templateText string + if conf.CustomTemplate != "" { + tpl, err := os.ReadFile(conf.CustomTemplate) + command.AbortIfError("Failed to read custom bootstrap file:", err) + templateText = string(tpl) + if conf.CustomTemplateData != "" { + var tplCustomDataMap map[string]any + tplCustomData, err := os.ReadFile(conf.CustomTemplateData) + command.AbortIfError("Failed to read custom boostrap data file:", err) + if !json.Valid([]byte(tplCustomData)) { + command.AbortWith("Invalid JSON object in custom data file.") + } + //create map from the custom template data + json.Unmarshal(tplCustomData, &tplCustomDataMap) + data.CustomData = tplCustomDataMap + } + } else if conf.Agouti { + templateText = agoutiBootstrapText + } else { + templateText = bootstrapText + } + + //Setting the option to explicitly fail if template is rendered trying to access missing key + bootstrapTemplate, err := template.New("bootstrap").Funcs(sprig.TxtFuncMap()).Option("missingkey=error").Parse(templateText) + command.AbortIfError("Failed to parse bootstrap template:", err) + + buf := &bytes.Buffer{} + //Being explicit about failing sooner during template rendering + //when accessing custom data rather than during the go fmt command + err = bootstrapTemplate.Execute(buf, data) + command.AbortIfError("Failed to render bootstrap template:", err) + + buf.WriteTo(f) + + internal.GoFmt(targetFile) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_command.go new file mode 100644 index 000000000..48d23f919 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_command.go @@ -0,0 +1,259 @@ +package generators + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "text/template" + + sprig "github.com/go-task/slim-sprig" + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/ginkgo/internal" + "github.com/onsi/ginkgo/v2/types" +) + +func BuildGenerateCommand() command.Command { + conf := GeneratorsConfig{} + flags, err := types.NewGinkgoFlagSet( + types.GinkgoFlags{ + {Name: "agouti", KeyPath: "Agouti", + Usage: "If set, generate will create a test file for writing Agouti tests"}, + {Name: "nodot", KeyPath: "NoDot", + Usage: "If set, generate will create a test file that does not dot-import ginkgo and gomega"}, + {Name: "internal", KeyPath: "Internal", + Usage: "If set, generate will create a test file that uses the regular package name (i.e. `package X`, not `package X_test`)"}, + {Name: "template", KeyPath: "CustomTemplate", + UsageArgument: "template-file", + Usage: "If specified, generate will use the contents of the file passed as the test file template"}, + {Name: "template-data", KeyPath: "CustomTemplateData", + UsageArgument: "template-data-file", + Usage: "If specified, generate will use the contents of the file passed as data to be rendered in the test file template"}, + }, + &conf, + types.GinkgoFlagSections{}, + ) + + if err != nil { + panic(err) + } + + return command.Command{ + Name: "generate", + Usage: "ginkgo generate ", + ShortDoc: "Generate a test file named _test.go", + Documentation: `If the optional argument is omitted, a file named after the package in the current directory will be created. + +You can pass multiple to generate multiple files simultaneously. The resulting files are named _test.go. + +You can also pass a of the form "file.go" and generate will emit "file_test.go".`, + DocLink: "generators", + Flags: flags, + Command: func(args []string, _ []string) { + generateTestFiles(conf, args) + }, + } +} + +type specData struct { + Package string + Subject string + PackageImportPath string + ImportPackage bool + + GinkgoImport string + GomegaImport string + GinkgoPackage string + GomegaPackage string + CustomData map[string]any +} + +func generateTestFiles(conf GeneratorsConfig, args []string) { + subjects := args + if len(subjects) == 0 { + subjects = []string{""} + } + for _, subject := range subjects { + generateTestFileForSubject(subject, conf) + } +} + +func generateTestFileForSubject(subject string, conf GeneratorsConfig) { + packageName, specFilePrefix, formattedName := getPackageAndFormattedName() + if subject != "" { + specFilePrefix = formatSubject(subject) + formattedName = prettifyName(specFilePrefix) + } + + if conf.Internal { + specFilePrefix = specFilePrefix + "_internal" + } + + data := specData{ + Package: determinePackageName(packageName, conf.Internal), + Subject: formattedName, + PackageImportPath: getPackageImportPath(), + ImportPackage: !conf.Internal, + + GinkgoImport: `. "github.com/onsi/ginkgo/v2"`, + GomegaImport: `. "github.com/onsi/gomega"`, + GinkgoPackage: "", + GomegaPackage: "", + } + + if conf.NoDot { + data.GinkgoImport = `"github.com/onsi/ginkgo/v2"` + data.GomegaImport = `"github.com/onsi/gomega"` + data.GinkgoPackage = `ginkgo.` + data.GomegaPackage = `gomega.` + } + + targetFile := fmt.Sprintf("%s_test.go", specFilePrefix) + if internal.FileExists(targetFile) { + command.AbortWith("{{bold}}%s{{/}} already exists", targetFile) + } else { + fmt.Printf("Generating ginkgo test for %s in:\n %s\n", data.Subject, targetFile) + } + + f, err := os.Create(targetFile) + command.AbortIfError("Failed to create test file:", err) + defer f.Close() + + var templateText string + if conf.CustomTemplate != "" { + tpl, err := os.ReadFile(conf.CustomTemplate) + command.AbortIfError("Failed to read custom template file:", err) + templateText = string(tpl) + if conf.CustomTemplateData != "" { + var tplCustomDataMap map[string]any + tplCustomData, err := os.ReadFile(conf.CustomTemplateData) + command.AbortIfError("Failed to read custom template data file:", err) + if !json.Valid([]byte(tplCustomData)) { + command.AbortWith("Invalid JSON object in custom data file.") + } + //create map from the custom template data + json.Unmarshal(tplCustomData, &tplCustomDataMap) + data.CustomData = tplCustomDataMap + } + } else if conf.Agouti { + templateText = agoutiSpecText + } else { + templateText = specText + } + + //Setting the option to explicitly fail if template is rendered trying to access missing key + specTemplate, err := template.New("spec").Funcs(sprig.TxtFuncMap()).Option("missingkey=error").Parse(templateText) + command.AbortIfError("Failed to read parse test template:", err) + + //Being explicit about failing sooner during template rendering + //when accessing custom data rather than during the go fmt command + err = specTemplate.Execute(f, data) + command.AbortIfError("Failed to render bootstrap template:", err) + internal.GoFmt(targetFile) +} + +func formatSubject(name string) string { + name = strings.ReplaceAll(name, "-", "_") + name = strings.ReplaceAll(name, " ", "_") + name = strings.Split(name, ".go")[0] + name = strings.Split(name, "_test")[0] + return name +} + +// moduleName returns module name from go.mod from given module root directory +func moduleName(modRoot string) string { + modFile, err := os.Open(filepath.Join(modRoot, "go.mod")) + if err != nil { + return "" + } + + mod := make([]byte, 128) + _, err = modFile.Read(mod) + if err != nil { + return "" + } + + slashSlash := []byte("//") + moduleStr := []byte("module") + + for len(mod) > 0 { + line := mod + mod = nil + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, mod = line[:i], line[i+1:] + } + if i := bytes.Index(line, slashSlash); i >= 0 { + line = line[:i] + } + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, moduleStr) { + continue + } + line = line[len(moduleStr):] + n := len(line) + line = bytes.TrimSpace(line) + if len(line) == n || len(line) == 0 { + continue + } + + if line[0] == '"' || line[0] == '`' { + p, err := strconv.Unquote(string(line)) + if err != nil { + return "" // malformed quoted string or multiline module path + } + return p + } + + return string(line) + } + + return "" // missing module path +} + +func findModuleRoot(dir string) (root string) { + dir = filepath.Clean(dir) + + // Look for enclosing go.mod. + for { + if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { + return dir + } + d := filepath.Dir(dir) + if d == dir { + break + } + dir = d + } + return "" +} + +func getPackageImportPath() string { + workingDir, err := os.Getwd() + if err != nil { + panic(err.Error()) + } + + sep := string(filepath.Separator) + + // Try go.mod file first + modRoot := findModuleRoot(workingDir) + if modRoot != "" { + modName := moduleName(modRoot) + if modName != "" { + cd := strings.ReplaceAll(workingDir, modRoot, "") + cd = strings.ReplaceAll(cd, sep, "/") + return modName + cd + } + } + + // Fallback to GOPATH structure + paths := strings.Split(workingDir, sep+"src"+sep) + if len(paths) == 1 { + fmt.Printf("\nCouldn't identify package import path.\n\n\tginkgo generate\n\nMust be run within a package directory under $GOPATH/src/...\nYou're going to have to change UNKNOWN_PACKAGE_PATH in the generated file...\n\n") + return "UNKNOWN_PACKAGE_PATH" + } + return filepath.ToSlash(paths[len(paths)-1]) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_templates.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_templates.go new file mode 100644 index 000000000..c3470adbf --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generate_templates.go @@ -0,0 +1,41 @@ +package generators + +var specText = `package {{.Package}} + +import ( + {{.GinkgoImport}} + {{.GomegaImport}} + + {{if .ImportPackage}}"{{.PackageImportPath}}"{{end}} +) + +var _ = {{.GinkgoPackage}}Describe("{{.Subject}}", func() { + +}) +` + +var agoutiSpecText = `package {{.Package}} + +import ( + {{.GinkgoImport}} + {{.GomegaImport}} + "github.com/sclevine/agouti" + . "github.com/sclevine/agouti/matchers" + + {{if .ImportPackage}}"{{.PackageImportPath}}"{{end}} +) + +var _ = {{.GinkgoPackage}}Describe("{{.Subject}}", func() { + var page *agouti.Page + + {{.GinkgoPackage}}BeforeEach(func() { + var err error + page, err = agoutiDriver.NewPage() + {{.GomegaPackage}}Expect(err).NotTo({{.GomegaPackage}}HaveOccurred()) + }) + + {{.GinkgoPackage}}AfterEach(func() { + {{.GomegaPackage}}Expect(page.Destroy()).To({{.GomegaPackage}}Succeed()) + }) +}) +` diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generators_common.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generators_common.go new file mode 100644 index 000000000..3046a4487 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/generators/generators_common.go @@ -0,0 +1,64 @@ +package generators + +import ( + "go/build" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/onsi/ginkgo/v2/ginkgo/command" +) + +type GeneratorsConfig struct { + Agouti, NoDot, Internal bool + CustomTemplate string + CustomTemplateData string +} + +func getPackageAndFormattedName() (string, string, string) { + path, err := os.Getwd() + command.AbortIfError("Could not get current working directory:", err) + + dirName := strings.ReplaceAll(filepath.Base(path), "-", "_") + dirName = strings.ReplaceAll(dirName, " ", "_") + + pkg, err := build.ImportDir(path, 0) + packageName := pkg.Name + if err != nil { + packageName = ensureLegalPackageName(dirName) + } + + formattedName := prettifyName(filepath.Base(path)) + return packageName, dirName, formattedName +} + +func ensureLegalPackageName(name string) string { + if name == "_" { + return "underscore" + } + if len(name) == 0 { + return "empty" + } + n, isDigitErr := strconv.Atoi(string(name[0])) + if isDigitErr == nil { + return []string{"zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"}[n] + name[1:] + } + return name +} + +func prettifyName(name string) string { + name = strings.ReplaceAll(name, "-", " ") + name = strings.ReplaceAll(name, "_", " ") + name = strings.Title(name) + name = strings.ReplaceAll(name, " ", "") + return name +} + +func determinePackageName(name string, internal bool) string { + if internal { + return name + } + + return name + "_test" +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/compile.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/compile.go new file mode 100644 index 000000000..496ec4a28 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/compile.go @@ -0,0 +1,152 @@ +package internal + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "github.com/onsi/ginkgo/v2/types" +) + +func CompileSuite(suite TestSuite, goFlagsConfig types.GoFlagsConfig) TestSuite { + if suite.PathToCompiledTest != "" { + return suite + } + + suite.CompilationError = nil + + path, err := filepath.Abs(filepath.Join(suite.Path, suite.PackageName+".test")) + if err != nil { + suite.State = TestSuiteStateFailedToCompile + suite.CompilationError = fmt.Errorf("Failed to compute compilation target path:\n%s", err.Error()) + return suite + } + + args, err := types.GenerateGoTestCompileArgs(goFlagsConfig, path, "./") + if err != nil { + suite.State = TestSuiteStateFailedToCompile + suite.CompilationError = fmt.Errorf("Failed to generate go test compile flags:\n%s", err.Error()) + return suite + } + + cmd := exec.Command("go", args...) + cmd.Dir = suite.Path + output, err := cmd.CombinedOutput() + if err != nil { + if len(output) > 0 { + suite.State = TestSuiteStateFailedToCompile + suite.CompilationError = fmt.Errorf("Failed to compile %s:\n\n%s", suite.PackageName, output) + } else { + suite.State = TestSuiteStateFailedToCompile + suite.CompilationError = fmt.Errorf("Failed to compile %s\n%s", suite.PackageName, err.Error()) + } + return suite + } + + if strings.Contains(string(output), "[no test files]") { + suite.State = TestSuiteStateSkippedDueToEmptyCompilation + return suite + } + + if len(output) > 0 { + fmt.Println(string(output)) + } + + if !FileExists(path) { + suite.State = TestSuiteStateFailedToCompile + suite.CompilationError = fmt.Errorf("Failed to compile %s:\nOutput file %s could not be found", suite.PackageName, path) + return suite + } + + suite.State = TestSuiteStateCompiled + suite.PathToCompiledTest = path + return suite +} + +func Cleanup(goFlagsConfig types.GoFlagsConfig, suites ...TestSuite) { + if goFlagsConfig.BinaryMustBePreserved() { + return + } + for _, suite := range suites { + if !suite.Precompiled { + os.Remove(suite.PathToCompiledTest) + } + } +} + +type parallelSuiteBundle struct { + suite TestSuite + compiled chan TestSuite +} + +type OrderedParallelCompiler struct { + mutex *sync.Mutex + stopped bool + numCompilers int + + idx int + numSuites int + completionChannels []chan TestSuite +} + +func NewOrderedParallelCompiler(numCompilers int) *OrderedParallelCompiler { + return &OrderedParallelCompiler{ + mutex: &sync.Mutex{}, + numCompilers: numCompilers, + } +} + +func (opc *OrderedParallelCompiler) StartCompiling(suites TestSuites, goFlagsConfig types.GoFlagsConfig) { + opc.stopped = false + opc.idx = 0 + opc.numSuites = len(suites) + opc.completionChannels = make([]chan TestSuite, opc.numSuites) + + toCompile := make(chan parallelSuiteBundle, opc.numCompilers) + for compiler := 0; compiler < opc.numCompilers; compiler++ { + go func() { + for bundle := range toCompile { + c, suite := bundle.compiled, bundle.suite + opc.mutex.Lock() + stopped := opc.stopped + opc.mutex.Unlock() + if !stopped { + suite = CompileSuite(suite, goFlagsConfig) + } + c <- suite + } + }() + } + + for idx, suite := range suites { + opc.completionChannels[idx] = make(chan TestSuite, 1) + toCompile <- parallelSuiteBundle{suite, opc.completionChannels[idx]} + if idx == 0 { //compile first suite serially + suite = <-opc.completionChannels[0] + opc.completionChannels[0] <- suite + } + } + + close(toCompile) +} + +func (opc *OrderedParallelCompiler) Next() (int, TestSuite) { + if opc.idx >= opc.numSuites { + return opc.numSuites, TestSuite{} + } + + idx := opc.idx + suite := <-opc.completionChannels[idx] + opc.idx = opc.idx + 1 + + return idx, suite +} + +func (opc *OrderedParallelCompiler) StopAndDrain() { + opc.mutex.Lock() + opc.stopped = true + opc.mutex.Unlock() +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/profiles_and_reports.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/profiles_and_reports.go new file mode 100644 index 000000000..bd3c6d028 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/profiles_and_reports.go @@ -0,0 +1,237 @@ +package internal + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + + "github.com/google/pprof/profile" + "github.com/onsi/ginkgo/v2/reporters" + "github.com/onsi/ginkgo/v2/types" +) + +func AbsPathForGeneratedAsset(assetName string, suite TestSuite, cliConfig types.CLIConfig, process int) string { + suffix := "" + if process != 0 { + suffix = fmt.Sprintf(".%d", process) + } + if cliConfig.OutputDir == "" { + return filepath.Join(suite.AbsPath(), assetName+suffix) + } + outputDir, _ := filepath.Abs(cliConfig.OutputDir) + return filepath.Join(outputDir, suite.NamespacedName()+"_"+assetName+suffix) +} + +func FinalizeProfilesAndReportsForSuites(suites TestSuites, cliConfig types.CLIConfig, suiteConfig types.SuiteConfig, reporterConfig types.ReporterConfig, goFlagsConfig types.GoFlagsConfig) ([]string, error) { + messages := []string{} + suitesWithProfiles := suites.WithState(TestSuiteStatePassed, TestSuiteStateFailed) //anything else won't have actually run and generated a profile + + // merge cover profiles if need be + if goFlagsConfig.Cover && !cliConfig.KeepSeparateCoverprofiles { + coverProfiles := []string{} + for _, suite := range suitesWithProfiles { + if !suite.HasProgrammaticFocus { + coverProfiles = append(coverProfiles, AbsPathForGeneratedAsset(goFlagsConfig.CoverProfile, suite, cliConfig, 0)) + } + } + + if len(coverProfiles) > 0 { + dst := goFlagsConfig.CoverProfile + if cliConfig.OutputDir != "" { + dst = filepath.Join(cliConfig.OutputDir, goFlagsConfig.CoverProfile) + } + err := MergeAndCleanupCoverProfiles(coverProfiles, dst) + if err != nil { + return messages, err + } + coverage, err := GetCoverageFromCoverProfile(dst) + if err != nil { + return messages, err + } + if coverage == 0 { + messages = append(messages, "composite coverage: [no statements]") + } else if suitesWithProfiles.AnyHaveProgrammaticFocus() { + messages = append(messages, fmt.Sprintf("composite coverage: %.1f%% of statements however some suites did not contribute because they included programatically focused specs", coverage)) + } else { + messages = append(messages, fmt.Sprintf("composite coverage: %.1f%% of statements", coverage)) + } + } else { + messages = append(messages, "no composite coverage computed: all suites included programatically focused specs") + } + } + + // copy binaries if need be + for _, suite := range suitesWithProfiles { + if goFlagsConfig.BinaryMustBePreserved() && cliConfig.OutputDir != "" { + src := suite.PathToCompiledTest + dst := filepath.Join(cliConfig.OutputDir, suite.NamespacedName()+".test") + if suite.Precompiled { + if err := CopyFile(src, dst); err != nil { + return messages, err + } + } else { + if err := os.Rename(src, dst); err != nil { + return messages, err + } + } + } + } + + type reportFormat struct { + ReportName string + GenerateFunc func(types.Report, string) error + MergeFunc func([]string, string) ([]string, error) + } + reportFormats := []reportFormat{} + if reporterConfig.JSONReport != "" { + reportFormats = append(reportFormats, reportFormat{ReportName: reporterConfig.JSONReport, GenerateFunc: reporters.GenerateJSONReport, MergeFunc: reporters.MergeAndCleanupJSONReports}) + } + if reporterConfig.JUnitReport != "" { + reportFormats = append(reportFormats, reportFormat{ReportName: reporterConfig.JUnitReport, GenerateFunc: reporters.GenerateJUnitReport, MergeFunc: reporters.MergeAndCleanupJUnitReports}) + } + if reporterConfig.TeamcityReport != "" { + reportFormats = append(reportFormats, reportFormat{ReportName: reporterConfig.TeamcityReport, GenerateFunc: reporters.GenerateTeamcityReport, MergeFunc: reporters.MergeAndCleanupTeamcityReports}) + } + + // Generate reports for suites that failed to run + reportableSuites := suites.ThatAreGinkgoSuites() + for _, suite := range reportableSuites.WithState(TestSuiteStateFailedToCompile, TestSuiteStateFailedDueToTimeout, TestSuiteStateSkippedDueToPriorFailures, TestSuiteStateSkippedDueToEmptyCompilation) { + report := types.Report{ + SuitePath: suite.AbsPath(), + SuiteConfig: suiteConfig, + SuiteSucceeded: false, + } + switch suite.State { + case TestSuiteStateFailedToCompile: + report.SpecialSuiteFailureReasons = append(report.SpecialSuiteFailureReasons, suite.CompilationError.Error()) + case TestSuiteStateFailedDueToTimeout: + report.SpecialSuiteFailureReasons = append(report.SpecialSuiteFailureReasons, TIMEOUT_ELAPSED_FAILURE_REASON) + case TestSuiteStateSkippedDueToPriorFailures: + report.SpecialSuiteFailureReasons = append(report.SpecialSuiteFailureReasons, PRIOR_FAILURES_FAILURE_REASON) + case TestSuiteStateSkippedDueToEmptyCompilation: + report.SpecialSuiteFailureReasons = append(report.SpecialSuiteFailureReasons, EMPTY_SKIP_FAILURE_REASON) + report.SuiteSucceeded = true + } + + for _, format := range reportFormats { + format.GenerateFunc(report, AbsPathForGeneratedAsset(format.ReportName, suite, cliConfig, 0)) + } + } + + // Merge reports unless we've been asked to keep them separate + if !cliConfig.KeepSeparateReports { + for _, format := range reportFormats { + reports := []string{} + for _, suite := range reportableSuites { + reports = append(reports, AbsPathForGeneratedAsset(format.ReportName, suite, cliConfig, 0)) + } + dst := format.ReportName + if cliConfig.OutputDir != "" { + dst = filepath.Join(cliConfig.OutputDir, format.ReportName) + } + mergeMessages, err := format.MergeFunc(reports, dst) + messages = append(messages, mergeMessages...) + if err != nil { + return messages, err + } + } + } + + return messages, nil +} + +//loads each profile, combines them, deletes them, stores them in destination +func MergeAndCleanupCoverProfiles(profiles []string, destination string) error { + combined := &bytes.Buffer{} + modeRegex := regexp.MustCompile(`^mode: .*\n`) + for i, profile := range profiles { + contents, err := os.ReadFile(profile) + if err != nil { + return fmt.Errorf("Unable to read coverage file %s:\n%s", profile, err.Error()) + } + os.Remove(profile) + + // remove the cover mode line from every file + // except the first one + if i > 0 { + contents = modeRegex.ReplaceAll(contents, []byte{}) + } + + _, err = combined.Write(contents) + + // Add a newline to the end of every file if missing. + if err == nil && len(contents) > 0 && contents[len(contents)-1] != '\n' { + _, err = combined.Write([]byte("\n")) + } + + if err != nil { + return fmt.Errorf("Unable to append to coverprofile:\n%s", err.Error()) + } + } + + err := os.WriteFile(destination, combined.Bytes(), 0666) + if err != nil { + return fmt.Errorf("Unable to create combined cover profile:\n%s", err.Error()) + } + return nil +} + +func GetCoverageFromCoverProfile(profile string) (float64, error) { + cmd := exec.Command("go", "tool", "cover", "-func", profile) + output, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("Could not process Coverprofile %s: %s", profile, err.Error()) + } + re := regexp.MustCompile(`total:\s*\(statements\)\s*(\d*\.\d*)\%`) + matches := re.FindStringSubmatch(string(output)) + if matches == nil { + return 0, fmt.Errorf("Could not parse Coverprofile to compute coverage percentage") + } + coverageString := matches[1] + coverage, err := strconv.ParseFloat(coverageString, 64) + if err != nil { + return 0, fmt.Errorf("Could not parse Coverprofile to compute coverage percentage: %s", err.Error()) + } + + return coverage, nil +} + +func MergeProfiles(profilePaths []string, destination string) error { + profiles := []*profile.Profile{} + for _, profilePath := range profilePaths { + proFile, err := os.Open(profilePath) + if err != nil { + return fmt.Errorf("Could not open profile: %s\n%s", profilePath, err.Error()) + } + prof, err := profile.Parse(proFile) + if err != nil { + return fmt.Errorf("Could not parse profile: %s\n%s", profilePath, err.Error()) + } + profiles = append(profiles, prof) + os.Remove(profilePath) + } + + mergedProfile, err := profile.Merge(profiles) + if err != nil { + return fmt.Errorf("Could not merge profiles:\n%s", err.Error()) + } + + outFile, err := os.Create(destination) + if err != nil { + return fmt.Errorf("Could not create merged profile %s:\n%s", destination, err.Error()) + } + err = mergedProfile.Write(outFile) + if err != nil { + return fmt.Errorf("Could not write merged profile %s:\n%s", destination, err.Error()) + } + err = outFile.Close() + if err != nil { + return fmt.Errorf("Could not close merged profile %s:\n%s", destination, err.Error()) + } + + return nil +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/run.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/run.go new file mode 100644 index 000000000..41052ea19 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/run.go @@ -0,0 +1,355 @@ +package internal + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "syscall" + "time" + + "github.com/onsi/ginkgo/v2/formatter" + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/internal/parallel_support" + "github.com/onsi/ginkgo/v2/reporters" + "github.com/onsi/ginkgo/v2/types" +) + +func RunCompiledSuite(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig types.ReporterConfig, cliConfig types.CLIConfig, goFlagsConfig types.GoFlagsConfig, additionalArgs []string) TestSuite { + suite.State = TestSuiteStateFailed + suite.HasProgrammaticFocus = false + + if suite.PathToCompiledTest == "" { + return suite + } + + if suite.IsGinkgo && cliConfig.ComputedProcs() > 1 { + suite = runParallel(suite, ginkgoConfig, reporterConfig, cliConfig, goFlagsConfig, additionalArgs) + } else if suite.IsGinkgo { + suite = runSerial(suite, ginkgoConfig, reporterConfig, cliConfig, goFlagsConfig, additionalArgs) + } else { + suite = runGoTest(suite, cliConfig, goFlagsConfig) + } + runAfterRunHook(cliConfig.AfterRunHook, reporterConfig.NoColor, suite) + return suite +} + +func buildAndStartCommand(suite TestSuite, args []string, pipeToStdout bool) (*exec.Cmd, *bytes.Buffer) { + buf := &bytes.Buffer{} + cmd := exec.Command(suite.PathToCompiledTest, args...) + cmd.Dir = suite.Path + if pipeToStdout { + cmd.Stderr = io.MultiWriter(os.Stdout, buf) + cmd.Stdout = os.Stdout + } else { + cmd.Stderr = buf + cmd.Stdout = buf + } + err := cmd.Start() + command.AbortIfError("Failed to start test suite", err) + + return cmd, buf +} + +func checkForNoTestsWarning(buf *bytes.Buffer) bool { + if strings.Contains(buf.String(), "warning: no tests to run") { + fmt.Fprintf(os.Stderr, `Found no test suites, did you forget to run "ginkgo bootstrap"?`) + return true + } + return false +} + +func runGoTest(suite TestSuite, cliConfig types.CLIConfig, goFlagsConfig types.GoFlagsConfig) TestSuite { + // As we run the go test from the suite directory, make sure the cover profile is absolute + // and placed into the expected output directory when one is configured. + if goFlagsConfig.Cover && !filepath.IsAbs(goFlagsConfig.CoverProfile) { + goFlagsConfig.CoverProfile = AbsPathForGeneratedAsset(goFlagsConfig.CoverProfile, suite, cliConfig, 0) + } + + args, err := types.GenerateGoTestRunArgs(goFlagsConfig) + command.AbortIfError("Failed to generate test run arguments", err) + cmd, buf := buildAndStartCommand(suite, args, true) + + cmd.Wait() + + exitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() + passed := (exitStatus == 0) || (exitStatus == types.GINKGO_FOCUS_EXIT_CODE) + passed = !(checkForNoTestsWarning(buf) && cliConfig.RequireSuite) && passed + if passed { + suite.State = TestSuiteStatePassed + } else { + suite.State = TestSuiteStateFailed + } + + return suite +} + +func runSerial(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig types.ReporterConfig, cliConfig types.CLIConfig, goFlagsConfig types.GoFlagsConfig, additionalArgs []string) TestSuite { + if goFlagsConfig.Cover { + goFlagsConfig.CoverProfile = AbsPathForGeneratedAsset(goFlagsConfig.CoverProfile, suite, cliConfig, 0) + } + if goFlagsConfig.BlockProfile != "" { + goFlagsConfig.BlockProfile = AbsPathForGeneratedAsset(goFlagsConfig.BlockProfile, suite, cliConfig, 0) + } + if goFlagsConfig.CPUProfile != "" { + goFlagsConfig.CPUProfile = AbsPathForGeneratedAsset(goFlagsConfig.CPUProfile, suite, cliConfig, 0) + } + if goFlagsConfig.MemProfile != "" { + goFlagsConfig.MemProfile = AbsPathForGeneratedAsset(goFlagsConfig.MemProfile, suite, cliConfig, 0) + } + if goFlagsConfig.MutexProfile != "" { + goFlagsConfig.MutexProfile = AbsPathForGeneratedAsset(goFlagsConfig.MutexProfile, suite, cliConfig, 0) + } + if reporterConfig.JSONReport != "" { + reporterConfig.JSONReport = AbsPathForGeneratedAsset(reporterConfig.JSONReport, suite, cliConfig, 0) + } + if reporterConfig.JUnitReport != "" { + reporterConfig.JUnitReport = AbsPathForGeneratedAsset(reporterConfig.JUnitReport, suite, cliConfig, 0) + } + if reporterConfig.TeamcityReport != "" { + reporterConfig.TeamcityReport = AbsPathForGeneratedAsset(reporterConfig.TeamcityReport, suite, cliConfig, 0) + } + + args, err := types.GenerateGinkgoTestRunArgs(ginkgoConfig, reporterConfig, goFlagsConfig) + command.AbortIfError("Failed to generate test run arguments", err) + args = append([]string{"--test.timeout=0"}, args...) + args = append(args, additionalArgs...) + + cmd, buf := buildAndStartCommand(suite, args, true) + + cmd.Wait() + + exitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() + suite.HasProgrammaticFocus = (exitStatus == types.GINKGO_FOCUS_EXIT_CODE) + passed := (exitStatus == 0) || (exitStatus == types.GINKGO_FOCUS_EXIT_CODE) + passed = !(checkForNoTestsWarning(buf) && cliConfig.RequireSuite) && passed + if passed { + suite.State = TestSuiteStatePassed + } else { + suite.State = TestSuiteStateFailed + } + + if suite.HasProgrammaticFocus { + if goFlagsConfig.Cover { + fmt.Fprintln(os.Stdout, "coverage: no coverfile was generated because specs are programmatically focused") + } + if goFlagsConfig.BlockProfile != "" { + fmt.Fprintln(os.Stdout, "no block profile was generated because specs are programmatically focused") + } + if goFlagsConfig.CPUProfile != "" { + fmt.Fprintln(os.Stdout, "no cpu profile was generated because specs are programmatically focused") + } + if goFlagsConfig.MemProfile != "" { + fmt.Fprintln(os.Stdout, "no mem profile was generated because specs are programmatically focused") + } + if goFlagsConfig.MutexProfile != "" { + fmt.Fprintln(os.Stdout, "no mutex profile was generated because specs are programmatically focused") + } + } + + return suite +} + +func runParallel(suite TestSuite, ginkgoConfig types.SuiteConfig, reporterConfig types.ReporterConfig, cliConfig types.CLIConfig, goFlagsConfig types.GoFlagsConfig, additionalArgs []string) TestSuite { + type procResult struct { + passed bool + hasProgrammaticFocus bool + } + + numProcs := cliConfig.ComputedProcs() + procOutput := make([]*bytes.Buffer, numProcs) + coverProfiles := []string{} + + blockProfiles := []string{} + cpuProfiles := []string{} + memProfiles := []string{} + mutexProfiles := []string{} + + procResults := make(chan procResult) + + server, err := parallel_support.NewServer(numProcs, reporters.NewDefaultReporter(reporterConfig, formatter.ColorableStdOut)) + command.AbortIfError("Failed to start parallel spec server", err) + server.Start() + defer server.Close() + + if reporterConfig.JSONReport != "" { + reporterConfig.JSONReport = AbsPathForGeneratedAsset(reporterConfig.JSONReport, suite, cliConfig, 0) + } + if reporterConfig.JUnitReport != "" { + reporterConfig.JUnitReport = AbsPathForGeneratedAsset(reporterConfig.JUnitReport, suite, cliConfig, 0) + } + if reporterConfig.TeamcityReport != "" { + reporterConfig.TeamcityReport = AbsPathForGeneratedAsset(reporterConfig.TeamcityReport, suite, cliConfig, 0) + } + + for proc := 1; proc <= numProcs; proc++ { + procGinkgoConfig := ginkgoConfig + procGinkgoConfig.ParallelProcess, procGinkgoConfig.ParallelTotal, procGinkgoConfig.ParallelHost = proc, numProcs, server.Address() + + procGoFlagsConfig := goFlagsConfig + if goFlagsConfig.Cover { + procGoFlagsConfig.CoverProfile = AbsPathForGeneratedAsset(goFlagsConfig.CoverProfile, suite, cliConfig, proc) + coverProfiles = append(coverProfiles, procGoFlagsConfig.CoverProfile) + } + if goFlagsConfig.BlockProfile != "" { + procGoFlagsConfig.BlockProfile = AbsPathForGeneratedAsset(goFlagsConfig.BlockProfile, suite, cliConfig, proc) + blockProfiles = append(blockProfiles, procGoFlagsConfig.BlockProfile) + } + if goFlagsConfig.CPUProfile != "" { + procGoFlagsConfig.CPUProfile = AbsPathForGeneratedAsset(goFlagsConfig.CPUProfile, suite, cliConfig, proc) + cpuProfiles = append(cpuProfiles, procGoFlagsConfig.CPUProfile) + } + if goFlagsConfig.MemProfile != "" { + procGoFlagsConfig.MemProfile = AbsPathForGeneratedAsset(goFlagsConfig.MemProfile, suite, cliConfig, proc) + memProfiles = append(memProfiles, procGoFlagsConfig.MemProfile) + } + if goFlagsConfig.MutexProfile != "" { + procGoFlagsConfig.MutexProfile = AbsPathForGeneratedAsset(goFlagsConfig.MutexProfile, suite, cliConfig, proc) + mutexProfiles = append(mutexProfiles, procGoFlagsConfig.MutexProfile) + } + + args, err := types.GenerateGinkgoTestRunArgs(procGinkgoConfig, reporterConfig, procGoFlagsConfig) + command.AbortIfError("Failed to generate test run arguments", err) + args = append([]string{"--test.timeout=0"}, args...) + args = append(args, additionalArgs...) + + cmd, buf := buildAndStartCommand(suite, args, false) + procOutput[proc-1] = buf + server.RegisterAlive(proc, func() bool { return cmd.ProcessState == nil || !cmd.ProcessState.Exited() }) + + go func() { + cmd.Wait() + exitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() + procResults <- procResult{ + passed: (exitStatus == 0) || (exitStatus == types.GINKGO_FOCUS_EXIT_CODE), + hasProgrammaticFocus: exitStatus == types.GINKGO_FOCUS_EXIT_CODE, + } + }() + } + + passed := true + for proc := 1; proc <= cliConfig.ComputedProcs(); proc++ { + result := <-procResults + passed = passed && result.passed + suite.HasProgrammaticFocus = suite.HasProgrammaticFocus || result.hasProgrammaticFocus + } + if passed { + suite.State = TestSuiteStatePassed + } else { + suite.State = TestSuiteStateFailed + } + + select { + case <-server.GetSuiteDone(): + fmt.Println("") + case <-time.After(time.Second): + //one of the nodes never finished reporting to the server. Something must have gone wrong. + fmt.Fprint(formatter.ColorableStdErr, formatter.F("\n{{bold}}{{red}}Ginkgo timed out waiting for all parallel procs to report back{{/}}\n")) + fmt.Fprint(formatter.ColorableStdErr, formatter.F("{{gray}}Test suite:{{/}} %s (%s)\n\n", suite.PackageName, suite.Path)) + fmt.Fprint(formatter.ColorableStdErr, formatter.Fiw(0, formatter.COLS, "This occurs if a parallel process exits before it reports its results to the Ginkgo CLI. The CLI will now print out all the stdout/stderr output it's collected from the running processes. However you may not see anything useful in these logs because the individual test processes usually intercept output to stdout/stderr in order to capture it in the spec reports.\n\nYou may want to try rerunning your test suite with {{light-gray}}--output-interceptor-mode=none{{/}} to see additional output here and debug your suite.\n")) + fmt.Fprintln(formatter.ColorableStdErr, " ") + for proc := 1; proc <= cliConfig.ComputedProcs(); proc++ { + fmt.Fprintf(formatter.ColorableStdErr, formatter.F("{{bold}}Output from proc %d:{{/}}\n", proc)) + fmt.Fprintln(os.Stderr, formatter.Fi(1, "%s", procOutput[proc-1].String())) + } + fmt.Fprintf(os.Stderr, "** End **") + } + + for proc := 1; proc <= cliConfig.ComputedProcs(); proc++ { + output := procOutput[proc-1].String() + if proc == 1 && checkForNoTestsWarning(procOutput[0]) && cliConfig.RequireSuite { + suite.State = TestSuiteStateFailed + } + if strings.Contains(output, "deprecated Ginkgo functionality") { + fmt.Fprintln(os.Stderr, output) + } + } + + if len(coverProfiles) > 0 { + if suite.HasProgrammaticFocus { + fmt.Fprintln(os.Stdout, "coverage: no coverfile was generated because specs are programmatically focused") + } else { + coverProfile := AbsPathForGeneratedAsset(goFlagsConfig.CoverProfile, suite, cliConfig, 0) + err := MergeAndCleanupCoverProfiles(coverProfiles, coverProfile) + command.AbortIfError("Failed to combine cover profiles", err) + + coverage, err := GetCoverageFromCoverProfile(coverProfile) + command.AbortIfError("Failed to compute coverage", err) + if coverage == 0 { + fmt.Fprintln(os.Stdout, "coverage: [no statements]") + } else { + fmt.Fprintf(os.Stdout, "coverage: %.1f%% of statements\n", coverage) + } + } + } + if len(blockProfiles) > 0 { + if suite.HasProgrammaticFocus { + fmt.Fprintln(os.Stdout, "no block profile was generated because specs are programmatically focused") + } else { + blockProfile := AbsPathForGeneratedAsset(goFlagsConfig.BlockProfile, suite, cliConfig, 0) + err := MergeProfiles(blockProfiles, blockProfile) + command.AbortIfError("Failed to combine blockprofiles", err) + } + } + if len(cpuProfiles) > 0 { + if suite.HasProgrammaticFocus { + fmt.Fprintln(os.Stdout, "no cpu profile was generated because specs are programmatically focused") + } else { + cpuProfile := AbsPathForGeneratedAsset(goFlagsConfig.CPUProfile, suite, cliConfig, 0) + err := MergeProfiles(cpuProfiles, cpuProfile) + command.AbortIfError("Failed to combine cpuprofiles", err) + } + } + if len(memProfiles) > 0 { + if suite.HasProgrammaticFocus { + fmt.Fprintln(os.Stdout, "no mem profile was generated because specs are programmatically focused") + } else { + memProfile := AbsPathForGeneratedAsset(goFlagsConfig.MemProfile, suite, cliConfig, 0) + err := MergeProfiles(memProfiles, memProfile) + command.AbortIfError("Failed to combine memprofiles", err) + } + } + if len(mutexProfiles) > 0 { + if suite.HasProgrammaticFocus { + fmt.Fprintln(os.Stdout, "no mutex profile was generated because specs are programmatically focused") + } else { + mutexProfile := AbsPathForGeneratedAsset(goFlagsConfig.MutexProfile, suite, cliConfig, 0) + err := MergeProfiles(mutexProfiles, mutexProfile) + command.AbortIfError("Failed to combine mutexprofiles", err) + } + } + + return suite +} + +func runAfterRunHook(command string, noColor bool, suite TestSuite) { + if command == "" { + return + } + f := formatter.NewWithNoColorBool(noColor) + + // Allow for string replacement to pass input to the command + passed := "[FAIL]" + if suite.State.Is(TestSuiteStatePassed) { + passed = "[PASS]" + } + command = strings.ReplaceAll(command, "(ginkgo-suite-passed)", passed) + command = strings.ReplaceAll(command, "(ginkgo-suite-name)", suite.PackageName) + + // Must break command into parts + splitArgs := regexp.MustCompile(`'.+'|".+"|\S+`) + parts := splitArgs.FindAllString(command, -1) + + output, err := exec.Command(parts[0], parts[1:]...).CombinedOutput() + if err != nil { + fmt.Fprintln(formatter.ColorableStdOut, f.Fi(0, "{{red}}{{bold}}After-run-hook failed:{{/}}")) + fmt.Fprintln(formatter.ColorableStdOut, f.Fi(1, "{{red}}%s{{/}}", output)) + } else { + fmt.Fprintln(formatter.ColorableStdOut, f.Fi(0, "{{green}}{{bold}}After-run-hook succeeded:{{/}}")) + fmt.Fprintln(formatter.ColorableStdOut, f.Fi(1, "{{green}}%s{{/}}", output)) + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/test_suite.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/test_suite.go new file mode 100644 index 000000000..64dcb1b78 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/test_suite.go @@ -0,0 +1,283 @@ +package internal + +import ( + "errors" + "math/rand" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/onsi/ginkgo/v2/types" +) + +const TIMEOUT_ELAPSED_FAILURE_REASON = "Suite did not run because the timeout elapsed" +const PRIOR_FAILURES_FAILURE_REASON = "Suite did not run because prior suites failed and --keep-going is not set" +const EMPTY_SKIP_FAILURE_REASON = "Suite did not run go test reported that no test files were found" + +type TestSuiteState uint + +const ( + TestSuiteStateInvalid TestSuiteState = iota + + TestSuiteStateUncompiled + TestSuiteStateCompiled + + TestSuiteStatePassed + + TestSuiteStateSkippedDueToEmptyCompilation + TestSuiteStateSkippedByFilter + TestSuiteStateSkippedDueToPriorFailures + + TestSuiteStateFailed + TestSuiteStateFailedDueToTimeout + TestSuiteStateFailedToCompile +) + +var TestSuiteStateFailureStates = []TestSuiteState{TestSuiteStateFailed, TestSuiteStateFailedDueToTimeout, TestSuiteStateFailedToCompile} + +func (state TestSuiteState) Is(states ...TestSuiteState) bool { + for _, suiteState := range states { + if suiteState == state { + return true + } + } + + return false +} + +type TestSuite struct { + Path string + PackageName string + IsGinkgo bool + + Precompiled bool + PathToCompiledTest string + CompilationError error + + HasProgrammaticFocus bool + State TestSuiteState +} + +func (ts TestSuite) AbsPath() string { + path, _ := filepath.Abs(ts.Path) + return path +} + +func (ts TestSuite) NamespacedName() string { + name := relPath(ts.Path) + name = strings.TrimLeft(name, "."+string(filepath.Separator)) + name = strings.ReplaceAll(name, string(filepath.Separator), "_") + name = strings.ReplaceAll(name, " ", "_") + if name == "" { + return ts.PackageName + } + return name +} + +type TestSuites []TestSuite + +func (ts TestSuites) AnyHaveProgrammaticFocus() bool { + for _, suite := range ts { + if suite.HasProgrammaticFocus { + return true + } + } + + return false +} + +func (ts TestSuites) ThatAreGinkgoSuites() TestSuites { + out := TestSuites{} + for _, suite := range ts { + if suite.IsGinkgo { + out = append(out, suite) + } + } + return out +} + +func (ts TestSuites) CountWithState(states ...TestSuiteState) int { + n := 0 + for _, suite := range ts { + if suite.State.Is(states...) { + n += 1 + } + } + + return n +} + +func (ts TestSuites) WithState(states ...TestSuiteState) TestSuites { + out := TestSuites{} + for _, suite := range ts { + if suite.State.Is(states...) { + out = append(out, suite) + } + } + + return out +} + +func (ts TestSuites) WithoutState(states ...TestSuiteState) TestSuites { + out := TestSuites{} + for _, suite := range ts { + if !suite.State.Is(states...) { + out = append(out, suite) + } + } + + return out +} + +func (ts TestSuites) ShuffledCopy(seed int64) TestSuites { + out := make(TestSuites, len(ts)) + permutation := rand.New(rand.NewSource(seed)).Perm(len(ts)) + for i, j := range permutation { + out[i] = ts[j] + } + return out +} + +func FindSuites(args []string, cliConfig types.CLIConfig, allowPrecompiled bool) TestSuites { + suites := TestSuites{} + + if len(args) > 0 { + for _, arg := range args { + if allowPrecompiled { + suite, err := precompiledTestSuite(arg) + if err == nil { + suites = append(suites, suite) + continue + } + } + recurseForSuite := cliConfig.Recurse + if strings.HasSuffix(arg, "/...") && arg != "/..." { + arg = arg[:len(arg)-4] + recurseForSuite = true + } + suites = append(suites, suitesInDir(arg, recurseForSuite)...) + } + } else { + suites = suitesInDir(".", cliConfig.Recurse) + } + + if cliConfig.SkipPackage != "" { + skipFilters := strings.Split(cliConfig.SkipPackage, ",") + for idx := range suites { + for _, skipFilter := range skipFilters { + if strings.Contains(suites[idx].Path, skipFilter) { + suites[idx].State = TestSuiteStateSkippedByFilter + break + } + } + } + } + + return suites +} + +func precompiledTestSuite(path string) (TestSuite, error) { + info, err := os.Stat(path) + if err != nil { + return TestSuite{}, err + } + + if info.IsDir() { + return TestSuite{}, errors.New("this is a directory, not a file") + } + + if filepath.Ext(path) != ".test" && filepath.Ext(path) != ".exe" { + return TestSuite{}, errors.New("this is not a .test binary") + } + + if filepath.Ext(path) == ".test" && info.Mode()&0111 == 0 { + return TestSuite{}, errors.New("this is not executable") + } + + dir := relPath(filepath.Dir(path)) + packageName := strings.TrimSuffix(filepath.Base(path), ".exe") + packageName = strings.TrimSuffix(packageName, ".test") + + path, err = filepath.Abs(path) + if err != nil { + return TestSuite{}, err + } + + return TestSuite{ + Path: dir, + PackageName: packageName, + IsGinkgo: true, + Precompiled: true, + PathToCompiledTest: path, + State: TestSuiteStateCompiled, + }, nil +} + +func suitesInDir(dir string, recurse bool) TestSuites { + suites := TestSuites{} + + if path.Base(dir) == "vendor" { + return suites + } + + files, _ := os.ReadDir(dir) + re := regexp.MustCompile(`^[^._].*_test\.go$`) + for _, file := range files { + if !file.IsDir() && re.Match([]byte(file.Name())) { + suite := TestSuite{ + Path: relPath(dir), + PackageName: packageNameForSuite(dir), + IsGinkgo: filesHaveGinkgoSuite(dir, files), + State: TestSuiteStateUncompiled, + } + suites = append(suites, suite) + break + } + } + + if recurse { + re = regexp.MustCompile(`^[._]`) + for _, file := range files { + if file.IsDir() && !re.Match([]byte(file.Name())) { + suites = append(suites, suitesInDir(dir+"/"+file.Name(), recurse)...) + } + } + } + + return suites +} + +func relPath(dir string) string { + dir, _ = filepath.Abs(dir) + cwd, _ := os.Getwd() + dir, _ = filepath.Rel(cwd, filepath.Clean(dir)) + + if string(dir[0]) != "." { + dir = "." + string(filepath.Separator) + dir + } + + return dir +} + +func packageNameForSuite(dir string) string { + path, _ := filepath.Abs(dir) + return filepath.Base(path) +} + +func filesHaveGinkgoSuite(dir string, files []os.DirEntry) bool { + reTestFile := regexp.MustCompile(`_test\.go$`) + reGinkgo := regexp.MustCompile(`package ginkgo|\/ginkgo"|\/ginkgo\/v2"|\/ginkgo\/v2/dsl/`) + + for _, file := range files { + if !file.IsDir() && reTestFile.Match([]byte(file.Name())) { + contents, _ := os.ReadFile(dir + "/" + file.Name()) + if reGinkgo.Match(contents) { + return true + } + } + } + + return false +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/utils.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/utils.go new file mode 100644 index 000000000..bd9ca7d51 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/utils.go @@ -0,0 +1,86 @@ +package internal + +import ( + "fmt" + "io" + "os" + "os/exec" + + "github.com/onsi/ginkgo/v2/formatter" + "github.com/onsi/ginkgo/v2/ginkgo/command" +) + +func FileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func CopyFile(src string, dest string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + + srcStat, err := srcFile.Stat() + if err != nil { + return err + } + + if _, err := os.Stat(dest); err == nil { + os.Remove(dest) + } + + destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, srcStat.Mode()) + if err != nil { + return err + } + + _, err = io.Copy(destFile, srcFile) + if err != nil { + return err + } + + if err := srcFile.Close(); err != nil { + return err + } + return destFile.Close() +} + +func GoFmt(path string) { + out, err := exec.Command("go", "fmt", path).CombinedOutput() + if err != nil { + command.AbortIfError(fmt.Sprintf("Could not fmt:\n%s\n", string(out)), err) + } +} + +func PluralizedWord(singular, plural string, count int) string { + if count == 1 { + return singular + } + return plural +} + +func FailedSuitesReport(suites TestSuites, f formatter.Formatter) string { + out := "" + out += "There were failures detected in the following suites:\n" + + maxPackageNameLength := 0 + for _, suite := range suites.WithState(TestSuiteStateFailureStates...) { + if len(suite.PackageName) > maxPackageNameLength { + maxPackageNameLength = len(suite.PackageName) + } + } + + packageNameFormatter := fmt.Sprintf("%%%ds", maxPackageNameLength) + for _, suite := range suites { + switch suite.State { + case TestSuiteStateFailed: + out += f.Fi(1, "{{red}}"+packageNameFormatter+" {{gray}}%s{{/}}\n", suite.PackageName, suite.Path) + case TestSuiteStateFailedToCompile: + out += f.Fi(1, "{{red}}"+packageNameFormatter+" {{gray}}%s {{magenta}}[Compilation failure]{{/}}\n", suite.PackageName, suite.Path) + case TestSuiteStateFailedDueToTimeout: + out += f.Fi(1, "{{red}}"+packageNameFormatter+" {{gray}}%s {{orange}}[%s]{{/}}\n", suite.PackageName, suite.Path, TIMEOUT_ELAPSED_FAILURE_REASON) + } + } + return out +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/verify_version.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/verify_version.go new file mode 100644 index 000000000..9da1bab3d --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/internal/verify_version.go @@ -0,0 +1,54 @@ +package internal + +import ( + "fmt" + "os/exec" + "regexp" + "strings" + + "github.com/onsi/ginkgo/v2/formatter" + "github.com/onsi/ginkgo/v2/types" +) + +var versiorRe = regexp.MustCompile(`v(\d+\.\d+\.\d+)`) + +func VerifyCLIAndFrameworkVersion(suites TestSuites) { + cliVersion := types.VERSION + mismatches := map[string][]string{} + + for _, suite := range suites { + cmd := exec.Command("go", "list", "-m", "github.com/onsi/ginkgo/v2") + cmd.Dir = suite.Path + output, err := cmd.CombinedOutput() + if err != nil { + continue + } + components := strings.Split(string(output), " ") + if len(components) != 2 { + continue + } + matches := versiorRe.FindStringSubmatch(components[1]) + if matches == nil || len(matches) != 2 { + continue + } + libraryVersion := matches[1] + if cliVersion != libraryVersion { + mismatches[libraryVersion] = append(mismatches[libraryVersion], suite.PackageName) + } + } + + if len(mismatches) == 0 { + return + } + + fmt.Println(formatter.F("{{red}}{{bold}}Ginkgo detected a version mismatch between the Ginkgo CLI and the version of Ginkgo imported by your packages:{{/}}")) + + fmt.Println(formatter.Fi(1, "Ginkgo CLI Version:")) + fmt.Println(formatter.Fi(2, "{{bold}}%s{{/}}", cliVersion)) + fmt.Println(formatter.Fi(1, "Mismatched package versions found:")) + for version, packages := range mismatches { + fmt.Println(formatter.Fi(2, "{{bold}}%s{{/}} used by %s", version, strings.Join(packages, ", "))) + } + fmt.Println("") + fmt.Println(formatter.Fiw(1, formatter.COLS, "{{gray}}Ginkgo will continue to attempt to run but you may see errors (including flag parsing errors) and should either update your go.mod or your version of the Ginkgo CLI to match.\n\nTo install the matching version of the CLI run\n {{bold}}go install github.com/onsi/ginkgo/v2/ginkgo{{/}}{{gray}}\nfrom a path that contains a go.mod file. Alternatively you can use\n {{bold}}go run github.com/onsi/ginkgo/v2/ginkgo{{/}}{{gray}}\nfrom a path that contains a go.mod file to invoke the matching version of the Ginkgo CLI.\n\nIf you are attempting to test multiple packages that each have a different version of the Ginkgo library with a single Ginkgo CLI that is currently unsupported.\n{{/}}")) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/labels/labels_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/labels/labels_command.go new file mode 100644 index 000000000..6c61f09d1 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/labels/labels_command.go @@ -0,0 +1,123 @@ +package labels + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "sort" + "strconv" + "strings" + + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/ginkgo/internal" + "github.com/onsi/ginkgo/v2/types" + "golang.org/x/tools/go/ast/inspector" +) + +func BuildLabelsCommand() command.Command { + var cliConfig = types.NewDefaultCLIConfig() + + flags, err := types.BuildLabelsCommandFlagSet(&cliConfig) + if err != nil { + panic(err) + } + + return command.Command{ + Name: "labels", + Usage: "ginkgo labels ", + Flags: flags, + ShortDoc: "List labels detected in the passed-in packages (or the package in the current directory if left blank).", + DocLink: "spec-labels", + Command: func(args []string, _ []string) { + ListLabels(args, cliConfig) + }, + } +} + +func ListLabels(args []string, cliConfig types.CLIConfig) { + suites := internal.FindSuites(args, cliConfig, false).WithoutState(internal.TestSuiteStateSkippedByFilter) + if len(suites) == 0 { + command.AbortWith("Found no test suites") + } + for _, suite := range suites { + labels := fetchLabelsFromPackage(suite.Path) + if len(labels) == 0 { + fmt.Printf("%s: No labels found\n", suite.PackageName) + } else { + fmt.Printf("%s: [%s]\n", suite.PackageName, strings.Join(labels, ", ")) + } + } +} + +func fetchLabelsFromPackage(packagePath string) []string { + fset := token.NewFileSet() + parsedPackages, err := parser.ParseDir(fset, packagePath, nil, 0) + command.AbortIfError("Failed to parse package source:", err) + + files := []*ast.File{} + hasTestPackage := false + for key, pkg := range parsedPackages { + if strings.HasSuffix(key, "_test") { + hasTestPackage = true + for _, file := range pkg.Files { + files = append(files, file) + } + } + } + if !hasTestPackage { + for _, pkg := range parsedPackages { + for _, file := range pkg.Files { + files = append(files, file) + } + } + } + + seen := map[string]bool{} + labels := []string{} + ispr := inspector.New(files) + ispr.Preorder([]ast.Node{&ast.CallExpr{}}, func(n ast.Node) { + potentialLabels := fetchLabels(n.(*ast.CallExpr)) + for _, label := range potentialLabels { + if !seen[label] { + seen[label] = true + labels = append(labels, strconv.Quote(label)) + } + } + }) + + sort.Strings(labels) + return labels +} + +func fetchLabels(callExpr *ast.CallExpr) []string { + out := []string{} + switch expr := callExpr.Fun.(type) { + case *ast.Ident: + if expr.Name != "Label" { + return out + } + case *ast.SelectorExpr: + if expr.Sel.Name != "Label" { + return out + } + default: + return out + } + for _, arg := range callExpr.Args { + switch expr := arg.(type) { + case *ast.BasicLit: + if expr.Kind == token.STRING { + unquoted, err := strconv.Unquote(expr.Value) + if err != nil { + unquoted = expr.Value + } + validated, err := types.ValidateAndCleanupLabel(unquoted, types.CodeLocation{}) + if err == nil { + out = append(out, validated) + } + } + } + } + return out +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/main.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/main.go new file mode 100644 index 000000000..e9abb27d8 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + "os" + + "github.com/onsi/ginkgo/v2/ginkgo/build" + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/ginkgo/generators" + "github.com/onsi/ginkgo/v2/ginkgo/labels" + "github.com/onsi/ginkgo/v2/ginkgo/outline" + "github.com/onsi/ginkgo/v2/ginkgo/run" + "github.com/onsi/ginkgo/v2/ginkgo/unfocus" + "github.com/onsi/ginkgo/v2/ginkgo/watch" + "github.com/onsi/ginkgo/v2/types" +) + +var program command.Program + +func GenerateCommands() []command.Command { + return []command.Command{ + watch.BuildWatchCommand(), + build.BuildBuildCommand(), + generators.BuildBootstrapCommand(), + generators.BuildGenerateCommand(), + labels.BuildLabelsCommand(), + outline.BuildOutlineCommand(), + unfocus.BuildUnfocusCommand(), + BuildVersionCommand(), + } +} + +func main() { + program = command.Program{ + Name: "ginkgo", + Heading: fmt.Sprintf("Ginkgo Version %s", types.VERSION), + Commands: GenerateCommands(), + DefaultCommand: run.BuildRunCommand(), + DeprecatedCommands: []command.DeprecatedCommand{ + {Name: "convert", Deprecation: types.Deprecations.Convert()}, + {Name: "blur", Deprecation: types.Deprecations.Blur()}, + {Name: "nodot", Deprecation: types.Deprecations.Nodot()}, + }, + } + + program.RunAndExit(os.Args) +} + +func BuildVersionCommand() command.Command { + return command.Command{ + Name: "version", + Usage: "ginkgo version", + ShortDoc: "Print Ginkgo's version", + Command: func(_ []string, _ []string) { + fmt.Printf("Ginkgo Version %s\n", types.VERSION) + }, + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/ginkgo.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/ginkgo.go new file mode 100644 index 000000000..0b9b19fe7 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/ginkgo.go @@ -0,0 +1,302 @@ +package outline + +import ( + "github.com/onsi/ginkgo/v2/types" + "go/ast" + "go/token" + "strconv" +) + +const ( + // undefinedTextAlt is used if the spec/container text cannot be derived + undefinedTextAlt = "undefined" +) + +// ginkgoMetadata holds useful bits of information for every entry in the outline +type ginkgoMetadata struct { + // Name is the spec or container function name, e.g. `Describe` or `It` + Name string `json:"name"` + + // Text is the `text` argument passed to specs, and some containers + Text string `json:"text"` + + // Start is the position of first character of the spec or container block + Start int `json:"start"` + + // End is the position of first character immediately after the spec or container block + End int `json:"end"` + + Spec bool `json:"spec"` + Focused bool `json:"focused"` + Pending bool `json:"pending"` + Labels []string `json:"labels"` +} + +// ginkgoNode is used to construct the outline as a tree +type ginkgoNode struct { + ginkgoMetadata + Nodes []*ginkgoNode `json:"nodes"` +} + +type walkFunc func(n *ginkgoNode) + +func (n *ginkgoNode) PreOrder(f walkFunc) { + f(n) + for _, m := range n.Nodes { + m.PreOrder(f) + } +} + +func (n *ginkgoNode) PostOrder(f walkFunc) { + for _, m := range n.Nodes { + m.PostOrder(f) + } + f(n) +} + +func (n *ginkgoNode) Walk(pre, post walkFunc) { + pre(n) + for _, m := range n.Nodes { + m.Walk(pre, post) + } + post(n) +} + +// PropagateInheritedProperties propagates the Pending and Focused properties +// through the subtree rooted at n. +func (n *ginkgoNode) PropagateInheritedProperties() { + n.PreOrder(func(thisNode *ginkgoNode) { + for _, descendantNode := range thisNode.Nodes { + if thisNode.Pending { + descendantNode.Pending = true + descendantNode.Focused = false + } + if thisNode.Focused && !descendantNode.Pending { + descendantNode.Focused = true + } + } + }) +} + +// BackpropagateUnfocus propagates the Focused property through the subtree +// rooted at n. It applies the rule described in the Ginkgo docs: +// > Nested programmatically focused specs follow a simple rule: if a +// > leaf-node is marked focused, any of its ancestor nodes that are marked +// > focus will be unfocused. +func (n *ginkgoNode) BackpropagateUnfocus() { + focusedSpecInSubtreeStack := []bool{} + n.PostOrder(func(thisNode *ginkgoNode) { + if thisNode.Spec { + focusedSpecInSubtreeStack = append(focusedSpecInSubtreeStack, thisNode.Focused) + return + } + focusedSpecInSubtree := false + for range thisNode.Nodes { + focusedSpecInSubtree = focusedSpecInSubtree || focusedSpecInSubtreeStack[len(focusedSpecInSubtreeStack)-1] + focusedSpecInSubtreeStack = focusedSpecInSubtreeStack[0 : len(focusedSpecInSubtreeStack)-1] + } + focusedSpecInSubtreeStack = append(focusedSpecInSubtreeStack, focusedSpecInSubtree) + if focusedSpecInSubtree { + thisNode.Focused = false + } + }) + +} + +func packageAndIdentNamesFromCallExpr(ce *ast.CallExpr) (string, string, bool) { + switch ex := ce.Fun.(type) { + case *ast.Ident: + return "", ex.Name, true + case *ast.SelectorExpr: + pkgID, ok := ex.X.(*ast.Ident) + if !ok { + return "", "", false + } + // A package identifier is top-level, so Obj must be nil + if pkgID.Obj != nil { + return "", "", false + } + if ex.Sel == nil { + return "", "", false + } + return pkgID.Name, ex.Sel.Name, true + default: + return "", "", false + } +} + +// absoluteOffsetsForNode derives the absolute character offsets of the node start and +// end positions. +func absoluteOffsetsForNode(fset *token.FileSet, n ast.Node) (start, end int) { + return fset.PositionFor(n.Pos(), false).Offset, fset.PositionFor(n.End(), false).Offset +} + +// ginkgoNodeFromCallExpr derives an outline entry from a go AST subtree +// corresponding to a Ginkgo container or spec. +func ginkgoNodeFromCallExpr(fset *token.FileSet, ce *ast.CallExpr, ginkgoPackageName *string) (*ginkgoNode, bool) { + packageName, identName, ok := packageAndIdentNamesFromCallExpr(ce) + if !ok { + return nil, false + } + + n := ginkgoNode{} + n.Name = identName + n.Start, n.End = absoluteOffsetsForNode(fset, ce) + n.Nodes = make([]*ginkgoNode, 0) + switch identName { + case "It", "Specify", "Entry": + n.Spec = true + n.Text = textOrAltFromCallExpr(ce, undefinedTextAlt) + n.Labels = labelFromCallExpr(ce) + n.Pending = pendingFromCallExpr(ce) + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "FIt", "FSpecify", "FEntry": + n.Spec = true + n.Focused = true + n.Text = textOrAltFromCallExpr(ce, undefinedTextAlt) + n.Labels = labelFromCallExpr(ce) + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "PIt", "PSpecify", "XIt", "XSpecify", "PEntry", "XEntry": + n.Spec = true + n.Pending = true + n.Text = textOrAltFromCallExpr(ce, undefinedTextAlt) + n.Labels = labelFromCallExpr(ce) + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "Context", "Describe", "When", "DescribeTable": + n.Text = textOrAltFromCallExpr(ce, undefinedTextAlt) + n.Labels = labelFromCallExpr(ce) + n.Pending = pendingFromCallExpr(ce) + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "FContext", "FDescribe", "FWhen", "FDescribeTable": + n.Focused = true + n.Text = textOrAltFromCallExpr(ce, undefinedTextAlt) + n.Labels = labelFromCallExpr(ce) + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "PContext", "PDescribe", "PWhen", "XContext", "XDescribe", "XWhen", "PDescribeTable", "XDescribeTable": + n.Pending = true + n.Text = textOrAltFromCallExpr(ce, undefinedTextAlt) + n.Labels = labelFromCallExpr(ce) + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "By": + n.Text = textOrAltFromCallExpr(ce, undefinedTextAlt) + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "AfterEach", "BeforeEach": + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "JustAfterEach", "JustBeforeEach": + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "AfterSuite", "BeforeSuite": + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + case "SynchronizedAfterSuite", "SynchronizedBeforeSuite": + return &n, ginkgoPackageName != nil && *ginkgoPackageName == packageName + default: + return nil, false + } +} + +// textOrAltFromCallExpr tries to derive the "text" of a Ginkgo spec or +// container. If it cannot derive it, it returns the alt text. +func textOrAltFromCallExpr(ce *ast.CallExpr, alt string) string { + text, defined := textFromCallExpr(ce) + if !defined { + return alt + } + return text +} + +// textFromCallExpr tries to derive the "text" of a Ginkgo spec or container. If +// it cannot derive it, it returns false. +func textFromCallExpr(ce *ast.CallExpr) (string, bool) { + if len(ce.Args) < 1 { + return "", false + } + text, ok := ce.Args[0].(*ast.BasicLit) + if !ok { + return "", false + } + switch text.Kind { + case token.CHAR, token.STRING: + // For token.CHAR and token.STRING, Value is quoted + unquoted, err := strconv.Unquote(text.Value) + if err != nil { + // If unquoting fails, just use the raw Value + return text.Value, true + } + return unquoted, true + default: + return text.Value, true + } +} + +func labelFromCallExpr(ce *ast.CallExpr) []string { + + labels := []string{} + if len(ce.Args) < 2 { + return labels + } + + for _, arg := range ce.Args[1:] { + switch expr := arg.(type) { + case *ast.CallExpr: + id, ok := expr.Fun.(*ast.Ident) + if !ok { + // to skip over cases where the expr.Fun. is actually *ast.SelectorExpr + continue + } + if id.Name == "Label" { + ls := extractLabels(expr) + for _, label := range ls { + labels = append(labels, label) + } + } + } + } + return labels +} + +func extractLabels(expr *ast.CallExpr) []string { + out := []string{} + for _, arg := range expr.Args { + switch expr := arg.(type) { + case *ast.BasicLit: + if expr.Kind == token.STRING { + unquoted, err := strconv.Unquote(expr.Value) + if err != nil { + unquoted = expr.Value + } + validated, err := types.ValidateAndCleanupLabel(unquoted, types.CodeLocation{}) + if err == nil { + out = append(out, validated) + } + } + } + } + + return out +} + +func pendingFromCallExpr(ce *ast.CallExpr) bool { + + pending := false + if len(ce.Args) < 2 { + return pending + } + + for _, arg := range ce.Args[1:] { + switch expr := arg.(type) { + case *ast.CallExpr: + id, ok := expr.Fun.(*ast.Ident) + if !ok { + // to skip over cases where the expr.Fun. is actually *ast.SelectorExpr + continue + } + if id.Name == "Pending" { + pending = true + } + case *ast.Ident: + if expr.Name == "Pending" { + pending = true + } + } + } + return pending +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/import.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/import.go new file mode 100644 index 000000000..67ec5ab75 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/import.go @@ -0,0 +1,65 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Most of the required functions were available in the +// "golang.org/x/tools/go/ast/astutil" package, but not exported. +// They were copied from https://github.com/golang/tools/blob/2b0845dc783e36ae26d683f4915a5840ef01ab0f/go/ast/astutil/imports.go + +package outline + +import ( + "go/ast" + "strconv" + "strings" +) + +// packageNameForImport returns the package name for the package. If the package +// is not imported, it returns nil. "Package name" refers to `pkgname` in the +// call expression `pkgname.ExportedIdentifier`. Examples: +// (import path not found) -> nil +// "import example.com/pkg/foo" -> "foo" +// "import fooalias example.com/pkg/foo" -> "fooalias" +// "import . example.com/pkg/foo" -> "" +func packageNameForImport(f *ast.File, path string) *string { + spec := importSpec(f, path) + if spec == nil { + return nil + } + name := spec.Name.String() + if name == "" { + // If the package name is not explicitly specified, + // make an educated guess. This is not guaranteed to be correct. + lastSlash := strings.LastIndex(path, "/") + if lastSlash == -1 { + name = path + } else { + name = path[lastSlash+1:] + } + } + if name == "." { + name = "" + } + return &name +} + +// importSpec returns the import spec if f imports path, +// or nil otherwise. +func importSpec(f *ast.File, path string) *ast.ImportSpec { + for _, s := range f.Imports { + if strings.HasPrefix(importPath(s), path) { + return s + } + } + return nil +} + +// importPath returns the unquoted import path of s, +// or "" if the path is not properly quoted. +func importPath(s *ast.ImportSpec) string { + t, err := strconv.Unquote(s.Path.Value) + if err != nil { + return "" + } + return t +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline.go new file mode 100644 index 000000000..c2327cda8 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline.go @@ -0,0 +1,110 @@ +package outline + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/token" + "strings" + + "golang.org/x/tools/go/ast/inspector" +) + +const ( + // ginkgoImportPath is the well-known ginkgo import path + ginkgoImportPath = "github.com/onsi/ginkgo/v2" +) + +// FromASTFile returns an outline for a Ginkgo test source file +func FromASTFile(fset *token.FileSet, src *ast.File) (*outline, error) { + ginkgoPackageName := packageNameForImport(src, ginkgoImportPath) + if ginkgoPackageName == nil { + return nil, fmt.Errorf("file does not import %q", ginkgoImportPath) + } + + root := ginkgoNode{} + stack := []*ginkgoNode{&root} + ispr := inspector.New([]*ast.File{src}) + ispr.Nodes([]ast.Node{(*ast.CallExpr)(nil)}, func(node ast.Node, push bool) bool { + if push { + // Pre-order traversal + ce, ok := node.(*ast.CallExpr) + if !ok { + // Because `Nodes` calls this function only when the node is an + // ast.CallExpr, this should never happen + panic(fmt.Errorf("node starting at %d, ending at %d is not an *ast.CallExpr", node.Pos(), node.End())) + } + gn, ok := ginkgoNodeFromCallExpr(fset, ce, ginkgoPackageName) + if !ok { + // Node is not a Ginkgo spec or container, continue + return true + } + parent := stack[len(stack)-1] + parent.Nodes = append(parent.Nodes, gn) + stack = append(stack, gn) + return true + } + // Post-order traversal + start, end := absoluteOffsetsForNode(fset, node) + lastVisitedGinkgoNode := stack[len(stack)-1] + if start != lastVisitedGinkgoNode.Start || end != lastVisitedGinkgoNode.End { + // Node is not a Ginkgo spec or container, so it was not pushed onto the stack, continue + return true + } + stack = stack[0 : len(stack)-1] + return true + }) + if len(root.Nodes) == 0 { + return &outline{[]*ginkgoNode{}}, nil + } + + // Derive the final focused property for all nodes. This must be done + // _before_ propagating the inherited focused property. + root.BackpropagateUnfocus() + // Now, propagate inherited properties, including focused and pending. + root.PropagateInheritedProperties() + + return &outline{root.Nodes}, nil +} + +type outline struct { + Nodes []*ginkgoNode `json:"nodes"` +} + +func (o *outline) MarshalJSON() ([]byte, error) { + return json.Marshal(o.Nodes) +} + +// String returns a CSV-formatted outline. Spec or container are output in +// depth-first order. +func (o *outline) String() string { + return o.StringIndent(0) +} + +// StringIndent returns a CSV-formated outline, but every line is indented by +// one 'width' of spaces for every level of nesting. +func (o *outline) StringIndent(width int) string { + var b strings.Builder + b.WriteString("Name,Text,Start,End,Spec,Focused,Pending,Labels\n") + + currentIndent := 0 + pre := func(n *ginkgoNode) { + b.WriteString(fmt.Sprintf("%*s", currentIndent, "")) + var labels string + if len(n.Labels) == 1 { + labels = n.Labels[0] + } else { + labels = strings.Join(n.Labels, ", ") + } + //enclosing labels in a double quoted comma separate listed so that when inmported into a CSV app the Labels column has comma separate strings + b.WriteString(fmt.Sprintf("%s,%s,%d,%d,%t,%t,%t,\"%s\"\n", n.Name, n.Text, n.Start, n.End, n.Spec, n.Focused, n.Pending, labels)) + currentIndent += width + } + post := func(n *ginkgoNode) { + currentIndent -= width + } + for _, n := range o.Nodes { + n.Walk(pre, post) + } + return b.String() +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline_command.go new file mode 100644 index 000000000..36698d46a --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/outline/outline_command.go @@ -0,0 +1,98 @@ +package outline + +import ( + "encoding/json" + "fmt" + "go/parser" + "go/token" + "os" + + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/types" +) + +const ( + // indentWidth is the width used by the 'indent' output + indentWidth = 4 + // stdinAlias is a portable alias for stdin. This convention is used in + // other CLIs, e.g., kubectl. + stdinAlias = "-" + usageCommand = "ginkgo outline " +) + +type outlineConfig struct { + Format string +} + +func BuildOutlineCommand() command.Command { + conf := outlineConfig{ + Format: "csv", + } + flags, err := types.NewGinkgoFlagSet( + types.GinkgoFlags{ + {Name: "format", KeyPath: "Format", + Usage: "Format of outline", + UsageArgument: "one of 'csv', 'indent', or 'json'", + UsageDefaultValue: conf.Format, + }, + }, + &conf, + types.GinkgoFlagSections{}, + ) + if err != nil { + panic(err) + } + + return command.Command{ + Name: "outline", + Usage: "ginkgo outline ", + ShortDoc: "Create an outline of Ginkgo symbols for a file", + Documentation: "To read from stdin, use: `ginkgo outline -`", + DocLink: "creating-an-outline-of-specs", + Flags: flags, + Command: func(args []string, _ []string) { + outlineFile(args, conf.Format) + }, + } +} + +func outlineFile(args []string, format string) { + if len(args) != 1 { + command.AbortWithUsage("outline expects exactly one argument") + } + + filename := args[0] + var src *os.File + if filename == stdinAlias { + src = os.Stdin + } else { + var err error + src, err = os.Open(filename) + command.AbortIfError("Failed to open file:", err) + } + + fset := token.NewFileSet() + + parsedSrc, err := parser.ParseFile(fset, filename, src, 0) + command.AbortIfError("Failed to parse source:", err) + + o, err := FromASTFile(fset, parsedSrc) + command.AbortIfError("Failed to create outline:", err) + + var oerr error + switch format { + case "csv": + _, oerr = fmt.Print(o) + case "indent": + _, oerr = fmt.Print(o.StringIndent(indentWidth)) + case "json": + b, err := json.Marshal(o) + if err != nil { + println(fmt.Sprintf("error marshalling to json: %s", err)) + } + _, oerr = fmt.Println(string(b)) + default: + command.AbortWith("Format %s not accepted", format) + } + command.AbortIfError("Failed to write outline:", oerr) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/run/run_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/run/run_command.go new file mode 100644 index 000000000..aaed4d570 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/run/run_command.go @@ -0,0 +1,232 @@ +package run + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/onsi/ginkgo/v2/formatter" + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/ginkgo/internal" + "github.com/onsi/ginkgo/v2/internal/interrupt_handler" + "github.com/onsi/ginkgo/v2/types" +) + +func BuildRunCommand() command.Command { + var suiteConfig = types.NewDefaultSuiteConfig() + var reporterConfig = types.NewDefaultReporterConfig() + var cliConfig = types.NewDefaultCLIConfig() + var goFlagsConfig = types.NewDefaultGoFlagsConfig() + + flags, err := types.BuildRunCommandFlagSet(&suiteConfig, &reporterConfig, &cliConfig, &goFlagsConfig) + if err != nil { + panic(err) + } + + interruptHandler := interrupt_handler.NewInterruptHandler(nil) + interrupt_handler.SwallowSigQuit() + + return command.Command{ + Name: "run", + Flags: flags, + Usage: "ginkgo run -- ", + ShortDoc: "Run the tests in the passed in (or the package in the current directory if left blank)", + Documentation: "Any arguments after -- will be passed to the test.", + DocLink: "running-tests", + Command: func(args []string, additionalArgs []string) { + var errors []error + cliConfig, goFlagsConfig, errors = types.VetAndInitializeCLIAndGoConfig(cliConfig, goFlagsConfig) + command.AbortIfErrors("Ginkgo detected configuration issues:", errors) + + runner := &SpecRunner{ + cliConfig: cliConfig, + goFlagsConfig: goFlagsConfig, + suiteConfig: suiteConfig, + reporterConfig: reporterConfig, + flags: flags, + + interruptHandler: interruptHandler, + } + + runner.RunSpecs(args, additionalArgs) + }, + } +} + +type SpecRunner struct { + suiteConfig types.SuiteConfig + reporterConfig types.ReporterConfig + cliConfig types.CLIConfig + goFlagsConfig types.GoFlagsConfig + flags types.GinkgoFlagSet + + interruptHandler *interrupt_handler.InterruptHandler +} + +func (r *SpecRunner) RunSpecs(args []string, additionalArgs []string) { + suites := internal.FindSuites(args, r.cliConfig, true) + skippedSuites := suites.WithState(internal.TestSuiteStateSkippedByFilter) + suites = suites.WithoutState(internal.TestSuiteStateSkippedByFilter) + + internal.VerifyCLIAndFrameworkVersion(suites) + + if len(skippedSuites) > 0 { + fmt.Println("Will skip:") + for _, skippedSuite := range skippedSuites { + fmt.Println(" " + skippedSuite.Path) + } + } + + if len(skippedSuites) > 0 && len(suites) == 0 { + command.AbortGracefullyWith("All tests skipped! Exiting...") + } + + if len(suites) == 0 { + command.AbortWith("Found no test suites") + } + + if len(suites) > 1 && !r.flags.WasSet("succinct") && r.reporterConfig.Verbosity().LT(types.VerbosityLevelVerbose) { + r.reporterConfig.Succinct = true + } + + t := time.Now() + var endTime time.Time + if r.suiteConfig.Timeout > 0 { + endTime = t.Add(r.suiteConfig.Timeout) + } + + iteration := 0 +OUTER_LOOP: + for { + if !r.flags.WasSet("seed") { + r.suiteConfig.RandomSeed = time.Now().Unix() + } + if r.cliConfig.RandomizeSuites && len(suites) > 1 { + suites = suites.ShuffledCopy(r.suiteConfig.RandomSeed) + } + + opc := internal.NewOrderedParallelCompiler(r.cliConfig.ComputedNumCompilers()) + opc.StartCompiling(suites, r.goFlagsConfig) + + SUITE_LOOP: + for { + suiteIdx, suite := opc.Next() + if suiteIdx >= len(suites) { + break SUITE_LOOP + } + suites[suiteIdx] = suite + + if r.interruptHandler.Status().Interrupted() { + opc.StopAndDrain() + break OUTER_LOOP + } + + if suites[suiteIdx].State.Is(internal.TestSuiteStateSkippedDueToEmptyCompilation) { + fmt.Printf("Skipping %s (no test files)\n", suite.Path) + continue SUITE_LOOP + } + + if suites[suiteIdx].State.Is(internal.TestSuiteStateFailedToCompile) { + fmt.Println(suites[suiteIdx].CompilationError.Error()) + if !r.cliConfig.KeepGoing { + opc.StopAndDrain() + } + continue SUITE_LOOP + } + + if suites.CountWithState(internal.TestSuiteStateFailureStates...) > 0 && !r.cliConfig.KeepGoing { + suites[suiteIdx].State = internal.TestSuiteStateSkippedDueToPriorFailures + opc.StopAndDrain() + continue SUITE_LOOP + } + + if !endTime.IsZero() { + r.suiteConfig.Timeout = endTime.Sub(time.Now()) + if r.suiteConfig.Timeout <= 0 { + suites[suiteIdx].State = internal.TestSuiteStateFailedDueToTimeout + opc.StopAndDrain() + continue SUITE_LOOP + } + } + + suites[suiteIdx] = internal.RunCompiledSuite(suites[suiteIdx], r.suiteConfig, r.reporterConfig, r.cliConfig, r.goFlagsConfig, additionalArgs) + } + + if suites.CountWithState(internal.TestSuiteStateFailureStates...) > 0 { + if iteration > 0 { + fmt.Printf("\nTests failed on attempt #%d\n\n", iteration+1) + } + break OUTER_LOOP + } + + if r.cliConfig.UntilItFails { + fmt.Printf("\nAll tests passed...\nWill keep running them until they fail.\nThis was attempt #%d\n%s\n", iteration+1, orcMessage(iteration+1)) + } else if r.cliConfig.Repeat > 0 && iteration < r.cliConfig.Repeat { + fmt.Printf("\nAll tests passed...\nThis was attempt %d of %d.\n", iteration+1, r.cliConfig.Repeat+1) + } else { + break OUTER_LOOP + } + iteration += 1 + } + + internal.Cleanup(r.goFlagsConfig, suites...) + + messages, err := internal.FinalizeProfilesAndReportsForSuites(suites, r.cliConfig, r.suiteConfig, r.reporterConfig, r.goFlagsConfig) + command.AbortIfError("could not finalize profiles:", err) + for _, message := range messages { + fmt.Println(message) + } + + fmt.Printf("\nGinkgo ran %d %s in %s\n", len(suites), internal.PluralizedWord("suite", "suites", len(suites)), time.Since(t)) + + if suites.CountWithState(internal.TestSuiteStateFailureStates...) == 0 { + if suites.AnyHaveProgrammaticFocus() && strings.TrimSpace(os.Getenv("GINKGO_EDITOR_INTEGRATION")) == "" { + fmt.Printf("Test Suite Passed\n") + fmt.Printf("Detected Programmatic Focus - setting exit status to %d\n", types.GINKGO_FOCUS_EXIT_CODE) + command.Abort(command.AbortDetails{ExitCode: types.GINKGO_FOCUS_EXIT_CODE}) + } else { + fmt.Printf("Test Suite Passed\n") + command.Abort(command.AbortDetails{}) + } + } else { + fmt.Fprintln(formatter.ColorableStdOut, "") + if len(suites) > 1 && suites.CountWithState(internal.TestSuiteStateFailureStates...) > 0 { + fmt.Fprintln(formatter.ColorableStdOut, + internal.FailedSuitesReport(suites, formatter.NewWithNoColorBool(r.reporterConfig.NoColor))) + } + fmt.Printf("Test Suite Failed\n") + command.Abort(command.AbortDetails{ExitCode: 1}) + } +} + +func orcMessage(iteration int) string { + if iteration < 10 { + return "" + } else if iteration < 30 { + return []string{ + "If at first you succeed...", + "...try, try again.", + "Looking good!", + "Still good...", + "I think your tests are fine....", + "Yep, still passing", + "Oh boy, here I go testin' again!", + "Even the gophers are getting bored", + "Did you try -race?", + "Maybe you should stop now?", + "I'm getting tired...", + "What if I just made you a sandwich?", + "Hit ^C, hit ^C, please hit ^C", + "Make it stop. Please!", + "Come on! Enough is enough!", + "Dave, this conversation can serve no purpose anymore. Goodbye.", + "Just what do you think you're doing, Dave? ", + "I, Sisyphus", + "Insanity: doing the same thing over and over again and expecting different results. -Einstein", + "I guess Einstein never tried to churn butter", + }[iteration-10] + "\n" + } else { + return "No, seriously... you can probably stop now.\n" + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/unfocus/unfocus_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/unfocus/unfocus_command.go new file mode 100644 index 000000000..7dd294394 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/unfocus/unfocus_command.go @@ -0,0 +1,186 @@ +package unfocus + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/onsi/ginkgo/v2/ginkgo/command" +) + +func BuildUnfocusCommand() command.Command { + return command.Command{ + Name: "unfocus", + Usage: "ginkgo unfocus", + ShortDoc: "Recursively unfocus any focused tests under the current directory", + DocLink: "filtering-specs", + Command: func(_ []string, _ []string) { + unfocusSpecs() + }, + } +} + +func unfocusSpecs() { + fmt.Println("Scanning for focus...") + + goFiles := make(chan string) + go func() { + unfocusDir(goFiles, ".") + close(goFiles) + }() + + const workers = 10 + wg := sync.WaitGroup{} + wg.Add(workers) + + for i := 0; i < workers; i++ { + go func() { + for path := range goFiles { + unfocusFile(path) + } + wg.Done() + }() + } + + wg.Wait() +} + +func unfocusDir(goFiles chan string, path string) { + files, err := os.ReadDir(path) + if err != nil { + fmt.Println(err.Error()) + return + } + + for _, f := range files { + switch { + case f.IsDir() && shouldProcessDir(f.Name()): + unfocusDir(goFiles, filepath.Join(path, f.Name())) + case !f.IsDir() && shouldProcessFile(f.Name()): + goFiles <- filepath.Join(path, f.Name()) + } + } +} + +func shouldProcessDir(basename string) bool { + return basename != "vendor" && !strings.HasPrefix(basename, ".") +} + +func shouldProcessFile(basename string) bool { + return strings.HasSuffix(basename, ".go") +} + +func unfocusFile(path string) { + data, err := os.ReadFile(path) + if err != nil { + fmt.Printf("error reading file '%s': %s\n", path, err.Error()) + return + } + + ast, err := parser.ParseFile(token.NewFileSet(), path, bytes.NewReader(data), parser.ParseComments) + if err != nil { + fmt.Printf("error parsing file '%s': %s\n", path, err.Error()) + return + } + + eliminations := scanForFocus(ast) + if len(eliminations) == 0 { + return + } + + fmt.Printf("...updating %s\n", path) + backup, err := writeBackup(path, data) + if err != nil { + fmt.Printf("error creating backup file: %s\n", err.Error()) + return + } + + if err := updateFile(path, data, eliminations); err != nil { + fmt.Printf("error writing file '%s': %s\n", path, err.Error()) + return + } + + os.Remove(backup) +} + +func writeBackup(path string, data []byte) (string, error) { + t, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)) + + if err != nil { + return "", fmt.Errorf("error creating temporary file: %w", err) + } + defer t.Close() + + if _, err := io.Copy(t, bytes.NewReader(data)); err != nil { + return "", fmt.Errorf("error writing to temporary file: %w", err) + } + + return t.Name(), nil +} + +func updateFile(path string, data []byte, eliminations [][]int64) error { + to, err := os.Create(path) + if err != nil { + return fmt.Errorf("error opening file for writing '%s': %w\n", path, err) + } + defer to.Close() + + from := bytes.NewReader(data) + var cursor int64 + for _, eliminationRange := range eliminations { + positionToEliminate, lengthToEliminate := eliminationRange[0]-1, eliminationRange[1] + if _, err := io.CopyN(to, from, positionToEliminate-cursor); err != nil { + return fmt.Errorf("error copying data: %w", err) + } + + cursor = positionToEliminate + lengthToEliminate + + if _, err := from.Seek(lengthToEliminate, io.SeekCurrent); err != nil { + return fmt.Errorf("error seeking to position in buffer: %w", err) + } + } + + if _, err := io.Copy(to, from); err != nil { + return fmt.Errorf("error copying end data: %w", err) + } + + return nil +} + +func scanForFocus(file *ast.File) (eliminations [][]int64) { + ast.Inspect(file, func(n ast.Node) bool { + if c, ok := n.(*ast.CallExpr); ok { + if i, ok := c.Fun.(*ast.Ident); ok { + if isFocus(i.Name) { + eliminations = append(eliminations, []int64{int64(i.Pos()), 1}) + } + } + } + + if i, ok := n.(*ast.Ident); ok { + if i.Name == "Focus" { + eliminations = append(eliminations, []int64{int64(i.Pos()), 6}) + } + } + + return true + }) + + return eliminations +} + +func isFocus(name string) bool { + switch name { + case "FDescribe", "FContext", "FIt", "FDescribeTable", "FEntry", "FSpecify", "FWhen": + return true + default: + return false + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta.go new file mode 100644 index 000000000..6c485c5b1 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta.go @@ -0,0 +1,22 @@ +package watch + +import "sort" + +type Delta struct { + ModifiedPackages []string + + NewSuites []*Suite + RemovedSuites []*Suite + modifiedSuites []*Suite +} + +type DescendingByDelta []*Suite + +func (a DescendingByDelta) Len() int { return len(a) } +func (a DescendingByDelta) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a DescendingByDelta) Less(i, j int) bool { return a[i].Delta() > a[j].Delta() } + +func (d Delta) ModifiedSuites() []*Suite { + sort.Sort(DescendingByDelta(d.modifiedSuites)) + return d.modifiedSuites +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta_tracker.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta_tracker.go new file mode 100644 index 000000000..26418ac62 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/delta_tracker.go @@ -0,0 +1,75 @@ +package watch + +import ( + "fmt" + + "regexp" + + "github.com/onsi/ginkgo/v2/ginkgo/internal" +) + +type SuiteErrors map[internal.TestSuite]error + +type DeltaTracker struct { + maxDepth int + watchRegExp *regexp.Regexp + suites map[string]*Suite + packageHashes *PackageHashes +} + +func NewDeltaTracker(maxDepth int, watchRegExp *regexp.Regexp) *DeltaTracker { + return &DeltaTracker{ + maxDepth: maxDepth, + watchRegExp: watchRegExp, + packageHashes: NewPackageHashes(watchRegExp), + suites: map[string]*Suite{}, + } +} + +func (d *DeltaTracker) Delta(suites internal.TestSuites) (delta Delta, errors SuiteErrors) { + errors = SuiteErrors{} + delta.ModifiedPackages = d.packageHashes.CheckForChanges() + + providedSuitePaths := map[string]bool{} + for _, suite := range suites { + providedSuitePaths[suite.Path] = true + } + + d.packageHashes.StartTrackingUsage() + + for _, suite := range d.suites { + if providedSuitePaths[suite.Suite.Path] { + if suite.Delta() > 0 { + delta.modifiedSuites = append(delta.modifiedSuites, suite) + } + } else { + delta.RemovedSuites = append(delta.RemovedSuites, suite) + } + } + + d.packageHashes.StopTrackingUsageAndPrune() + + for _, suite := range suites { + _, ok := d.suites[suite.Path] + if !ok { + s, err := NewSuite(suite, d.maxDepth, d.packageHashes) + if err != nil { + errors[suite] = err + continue + } + d.suites[suite.Path] = s + delta.NewSuites = append(delta.NewSuites, s) + } + } + + return delta, errors +} + +func (d *DeltaTracker) WillRun(suite internal.TestSuite) error { + s, ok := d.suites[suite.Path] + if !ok { + return fmt.Errorf("unknown suite %s", suite.Path) + } + + return s.MarkAsRunAndRecomputedDependencies(d.maxDepth) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/dependencies.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/dependencies.go new file mode 100644 index 000000000..f5ddff30f --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/dependencies.go @@ -0,0 +1,92 @@ +package watch + +import ( + "go/build" + "regexp" +) + +var ginkgoAndGomegaFilter = regexp.MustCompile(`github\.com/onsi/ginkgo|github\.com/onsi/gomega`) +var ginkgoIntegrationTestFilter = regexp.MustCompile(`github\.com/onsi/ginkgo/integration`) //allow us to integration test this thing + +type Dependencies struct { + deps map[string]int +} + +func NewDependencies(path string, maxDepth int) (Dependencies, error) { + d := Dependencies{ + deps: map[string]int{}, + } + + if maxDepth == 0 { + return d, nil + } + + err := d.seedWithDepsForPackageAtPath(path) + if err != nil { + return d, err + } + + for depth := 1; depth < maxDepth; depth++ { + n := len(d.deps) + d.addDepsForDepth(depth) + if n == len(d.deps) { + break + } + } + + return d, nil +} + +func (d Dependencies) Dependencies() map[string]int { + return d.deps +} + +func (d Dependencies) seedWithDepsForPackageAtPath(path string) error { + pkg, err := build.ImportDir(path, 0) + if err != nil { + return err + } + + d.resolveAndAdd(pkg.Imports, 1) + d.resolveAndAdd(pkg.TestImports, 1) + d.resolveAndAdd(pkg.XTestImports, 1) + + delete(d.deps, pkg.Dir) + return nil +} + +func (d Dependencies) addDepsForDepth(depth int) { + for dep, depDepth := range d.deps { + if depDepth == depth { + d.addDepsForDep(dep, depth+1) + } + } +} + +func (d Dependencies) addDepsForDep(dep string, depth int) { + pkg, err := build.ImportDir(dep, 0) + if err != nil { + println(err.Error()) + return + } + d.resolveAndAdd(pkg.Imports, depth) +} + +func (d Dependencies) resolveAndAdd(deps []string, depth int) { + for _, dep := range deps { + pkg, err := build.Import(dep, ".", 0) + if err != nil { + continue + } + if !pkg.Goroot && (!ginkgoAndGomegaFilter.Match([]byte(pkg.Dir)) || ginkgoIntegrationTestFilter.Match([]byte(pkg.Dir))) { + d.addDepIfNotPresent(pkg.Dir, depth) + } + } +} + +func (d Dependencies) addDepIfNotPresent(dep string, depth int) { + _, ok := d.deps[dep] + if !ok { + d.deps[dep] = depth + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hash.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hash.go new file mode 100644 index 000000000..e9f7ec0cb --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hash.go @@ -0,0 +1,108 @@ +package watch + +import ( + "fmt" + "os" + "regexp" + "time" +) + +var goTestRegExp = regexp.MustCompile(`_test\.go$`) + +type PackageHash struct { + CodeModifiedTime time.Time + TestModifiedTime time.Time + Deleted bool + + path string + codeHash string + testHash string + watchRegExp *regexp.Regexp +} + +func NewPackageHash(path string, watchRegExp *regexp.Regexp) *PackageHash { + p := &PackageHash{ + path: path, + watchRegExp: watchRegExp, + } + + p.codeHash, _, p.testHash, _, p.Deleted = p.computeHashes() + + return p +} + +func (p *PackageHash) CheckForChanges() bool { + codeHash, codeModifiedTime, testHash, testModifiedTime, deleted := p.computeHashes() + + if deleted { + if !p.Deleted { + t := time.Now() + p.CodeModifiedTime = t + p.TestModifiedTime = t + } + p.Deleted = true + return true + } + + modified := false + p.Deleted = false + + if p.codeHash != codeHash { + p.CodeModifiedTime = codeModifiedTime + modified = true + } + if p.testHash != testHash { + p.TestModifiedTime = testModifiedTime + modified = true + } + + p.codeHash = codeHash + p.testHash = testHash + return modified +} + +func (p *PackageHash) computeHashes() (codeHash string, codeModifiedTime time.Time, testHash string, testModifiedTime time.Time, deleted bool) { + entries, err := os.ReadDir(p.path) + + if err != nil { + deleted = true + return + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + info, err := entry.Info() + if err != nil { + continue + } + + if goTestRegExp.Match([]byte(info.Name())) { + testHash += p.hashForFileInfo(info) + if info.ModTime().After(testModifiedTime) { + testModifiedTime = info.ModTime() + } + continue + } + + if p.watchRegExp.Match([]byte(info.Name())) { + codeHash += p.hashForFileInfo(info) + if info.ModTime().After(codeModifiedTime) { + codeModifiedTime = info.ModTime() + } + } + } + + testHash += codeHash + if codeModifiedTime.After(testModifiedTime) { + testModifiedTime = codeModifiedTime + } + + return +} + +func (p *PackageHash) hashForFileInfo(info os.FileInfo) string { + return fmt.Sprintf("%s_%d_%d", info.Name(), info.Size(), info.ModTime().UnixNano()) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hashes.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hashes.go new file mode 100644 index 000000000..b4892bebf --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/package_hashes.go @@ -0,0 +1,85 @@ +package watch + +import ( + "path/filepath" + "regexp" + "sync" +) + +type PackageHashes struct { + PackageHashes map[string]*PackageHash + usedPaths map[string]bool + watchRegExp *regexp.Regexp + lock *sync.Mutex +} + +func NewPackageHashes(watchRegExp *regexp.Regexp) *PackageHashes { + return &PackageHashes{ + PackageHashes: map[string]*PackageHash{}, + usedPaths: nil, + watchRegExp: watchRegExp, + lock: &sync.Mutex{}, + } +} + +func (p *PackageHashes) CheckForChanges() []string { + p.lock.Lock() + defer p.lock.Unlock() + + modified := []string{} + + for _, packageHash := range p.PackageHashes { + if packageHash.CheckForChanges() { + modified = append(modified, packageHash.path) + } + } + + return modified +} + +func (p *PackageHashes) Add(path string) *PackageHash { + p.lock.Lock() + defer p.lock.Unlock() + + path, _ = filepath.Abs(path) + _, ok := p.PackageHashes[path] + if !ok { + p.PackageHashes[path] = NewPackageHash(path, p.watchRegExp) + } + + if p.usedPaths != nil { + p.usedPaths[path] = true + } + return p.PackageHashes[path] +} + +func (p *PackageHashes) Get(path string) *PackageHash { + p.lock.Lock() + defer p.lock.Unlock() + + path, _ = filepath.Abs(path) + if p.usedPaths != nil { + p.usedPaths[path] = true + } + return p.PackageHashes[path] +} + +func (p *PackageHashes) StartTrackingUsage() { + p.lock.Lock() + defer p.lock.Unlock() + + p.usedPaths = map[string]bool{} +} + +func (p *PackageHashes) StopTrackingUsageAndPrune() { + p.lock.Lock() + defer p.lock.Unlock() + + for path := range p.PackageHashes { + if !p.usedPaths[path] { + delete(p.PackageHashes, path) + } + } + + p.usedPaths = nil +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/suite.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/suite.go new file mode 100644 index 000000000..53272df7e --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/suite.go @@ -0,0 +1,87 @@ +package watch + +import ( + "fmt" + "math" + "time" + + "github.com/onsi/ginkgo/v2/ginkgo/internal" +) + +type Suite struct { + Suite internal.TestSuite + RunTime time.Time + Dependencies Dependencies + + sharedPackageHashes *PackageHashes +} + +func NewSuite(suite internal.TestSuite, maxDepth int, sharedPackageHashes *PackageHashes) (*Suite, error) { + deps, err := NewDependencies(suite.Path, maxDepth) + if err != nil { + return nil, err + } + + sharedPackageHashes.Add(suite.Path) + for dep := range deps.Dependencies() { + sharedPackageHashes.Add(dep) + } + + return &Suite{ + Suite: suite, + Dependencies: deps, + + sharedPackageHashes: sharedPackageHashes, + }, nil +} + +func (s *Suite) Delta() float64 { + delta := s.delta(s.Suite.Path, true, 0) * 1000 + for dep, depth := range s.Dependencies.Dependencies() { + delta += s.delta(dep, false, depth) + } + return delta +} + +func (s *Suite) MarkAsRunAndRecomputedDependencies(maxDepth int) error { + s.RunTime = time.Now() + + deps, err := NewDependencies(s.Suite.Path, maxDepth) + if err != nil { + return err + } + + s.sharedPackageHashes.Add(s.Suite.Path) + for dep := range deps.Dependencies() { + s.sharedPackageHashes.Add(dep) + } + + s.Dependencies = deps + + return nil +} + +func (s *Suite) Description() string { + numDeps := len(s.Dependencies.Dependencies()) + pluralizer := "ies" + if numDeps == 1 { + pluralizer = "y" + } + return fmt.Sprintf("%s [%d dependenc%s]", s.Suite.Path, numDeps, pluralizer) +} + +func (s *Suite) delta(packagePath string, includeTests bool, depth int) float64 { + return math.Max(float64(s.dt(packagePath, includeTests)), 0) / float64(depth+1) +} + +func (s *Suite) dt(packagePath string, includeTests bool) time.Duration { + packageHash := s.sharedPackageHashes.Get(packagePath) + var modifiedTime time.Time + if includeTests { + modifiedTime = packageHash.TestModifiedTime + } else { + modifiedTime = packageHash.CodeModifiedTime + } + + return modifiedTime.Sub(s.RunTime) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/watch_command.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/watch_command.go new file mode 100644 index 000000000..bde4193ce --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo/watch/watch_command.go @@ -0,0 +1,192 @@ +package watch + +import ( + "fmt" + "regexp" + "time" + + "github.com/onsi/ginkgo/v2/formatter" + "github.com/onsi/ginkgo/v2/ginkgo/command" + "github.com/onsi/ginkgo/v2/ginkgo/internal" + "github.com/onsi/ginkgo/v2/internal/interrupt_handler" + "github.com/onsi/ginkgo/v2/types" +) + +func BuildWatchCommand() command.Command { + var suiteConfig = types.NewDefaultSuiteConfig() + var reporterConfig = types.NewDefaultReporterConfig() + var cliConfig = types.NewDefaultCLIConfig() + var goFlagsConfig = types.NewDefaultGoFlagsConfig() + + flags, err := types.BuildWatchCommandFlagSet(&suiteConfig, &reporterConfig, &cliConfig, &goFlagsConfig) + if err != nil { + panic(err) + } + interruptHandler := interrupt_handler.NewInterruptHandler(nil) + interrupt_handler.SwallowSigQuit() + + return command.Command{ + Name: "watch", + Flags: flags, + Usage: "ginkgo watch -- ", + ShortDoc: "Watch the passed in and runs their tests whenever changes occur.", + Documentation: "Any arguments after -- will be passed to the test.", + DocLink: "watching-for-changes", + Command: func(args []string, additionalArgs []string) { + var errors []error + cliConfig, goFlagsConfig, errors = types.VetAndInitializeCLIAndGoConfig(cliConfig, goFlagsConfig) + command.AbortIfErrors("Ginkgo detected configuration issues:", errors) + + watcher := &SpecWatcher{ + cliConfig: cliConfig, + goFlagsConfig: goFlagsConfig, + suiteConfig: suiteConfig, + reporterConfig: reporterConfig, + flags: flags, + + interruptHandler: interruptHandler, + } + + watcher.WatchSpecs(args, additionalArgs) + }, + } +} + +type SpecWatcher struct { + suiteConfig types.SuiteConfig + reporterConfig types.ReporterConfig + cliConfig types.CLIConfig + goFlagsConfig types.GoFlagsConfig + flags types.GinkgoFlagSet + + interruptHandler *interrupt_handler.InterruptHandler +} + +func (w *SpecWatcher) WatchSpecs(args []string, additionalArgs []string) { + suites := internal.FindSuites(args, w.cliConfig, false).WithoutState(internal.TestSuiteStateSkippedByFilter) + + internal.VerifyCLIAndFrameworkVersion(suites) + + if len(suites) == 0 { + command.AbortWith("Found no test suites") + } + + fmt.Printf("Identified %d test %s. Locating dependencies to a depth of %d (this may take a while)...\n", len(suites), internal.PluralizedWord("suite", "suites", len(suites)), w.cliConfig.Depth) + deltaTracker := NewDeltaTracker(w.cliConfig.Depth, regexp.MustCompile(w.cliConfig.WatchRegExp)) + delta, errors := deltaTracker.Delta(suites) + + fmt.Printf("Watching %d %s:\n", len(delta.NewSuites), internal.PluralizedWord("suite", "suites", len(delta.NewSuites))) + for _, suite := range delta.NewSuites { + fmt.Println(" " + suite.Description()) + } + + for suite, err := range errors { + fmt.Printf("Failed to watch %s: %s\n", suite.PackageName, err) + } + + if len(suites) == 1 { + w.updateSeed() + w.compileAndRun(suites[0], additionalArgs) + } + + ticker := time.NewTicker(time.Second) + + for { + select { + case <-ticker.C: + suites := internal.FindSuites(args, w.cliConfig, false).WithoutState(internal.TestSuiteStateSkippedByFilter) + delta, _ := deltaTracker.Delta(suites) + coloredStream := formatter.ColorableStdOut + + suites = internal.TestSuites{} + + if len(delta.NewSuites) > 0 { + fmt.Fprintln(coloredStream, formatter.F("{{green}}Detected %d new %s:{{/}}", len(delta.NewSuites), internal.PluralizedWord("suite", "suites", len(delta.NewSuites)))) + for _, suite := range delta.NewSuites { + suites = append(suites, suite.Suite) + fmt.Fprintln(coloredStream, formatter.Fi(1, "%s", suite.Description())) + } + } + + modifiedSuites := delta.ModifiedSuites() + if len(modifiedSuites) > 0 { + fmt.Fprintln(coloredStream, formatter.F("{{green}}Detected changes in:{{/}}")) + for _, pkg := range delta.ModifiedPackages { + fmt.Fprintln(coloredStream, formatter.Fi(1, "%s", pkg)) + } + fmt.Fprintln(coloredStream, formatter.F("{{green}}Will run %d %s:{{/}}", len(modifiedSuites), internal.PluralizedWord("suite", "suites", len(modifiedSuites)))) + for _, suite := range modifiedSuites { + suites = append(suites, suite.Suite) + fmt.Fprintln(coloredStream, formatter.Fi(1, "%s", suite.Description())) + } + fmt.Fprintln(coloredStream, "") + } + + if len(suites) == 0 { + break + } + + w.updateSeed() + w.computeSuccinctMode(len(suites)) + for idx := range suites { + if w.interruptHandler.Status().Interrupted() { + return + } + deltaTracker.WillRun(suites[idx]) + suites[idx] = w.compileAndRun(suites[idx], additionalArgs) + } + color := "{{green}}" + if suites.CountWithState(internal.TestSuiteStateFailureStates...) > 0 { + color = "{{red}}" + } + fmt.Fprintln(coloredStream, formatter.F(color+"\nDone. Resuming watch...{{/}}")) + + messages, err := internal.FinalizeProfilesAndReportsForSuites(suites, w.cliConfig, w.suiteConfig, w.reporterConfig, w.goFlagsConfig) + command.AbortIfError("could not finalize profiles:", err) + for _, message := range messages { + fmt.Println(message) + } + case <-w.interruptHandler.Status().Channel: + return + } + } +} + +func (w *SpecWatcher) compileAndRun(suite internal.TestSuite, additionalArgs []string) internal.TestSuite { + suite = internal.CompileSuite(suite, w.goFlagsConfig) + if suite.State.Is(internal.TestSuiteStateFailedToCompile) { + fmt.Println(suite.CompilationError.Error()) + return suite + } + if w.interruptHandler.Status().Interrupted() { + return suite + } + suite = internal.RunCompiledSuite(suite, w.suiteConfig, w.reporterConfig, w.cliConfig, w.goFlagsConfig, additionalArgs) + internal.Cleanup(w.goFlagsConfig, suite) + return suite +} + +func (w *SpecWatcher) computeSuccinctMode(numSuites int) { + if w.reporterConfig.Verbosity().GTE(types.VerbosityLevelVerbose) { + w.reporterConfig.Succinct = false + return + } + + if w.flags.WasSet("succinct") { + return + } + + if numSuites == 1 { + w.reporterConfig.Succinct = false + } + + if numSuites > 1 { + w.reporterConfig.Succinct = true + } +} + +func (w *SpecWatcher) updateSeed() { + if !w.flags.WasSet("seed") { + w.suiteConfig.RandomSeed = time.Now().Unix() + } +} diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo_cli_dependencies.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo_cli_dependencies.go new file mode 100644 index 000000000..85162720f --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo_cli_dependencies.go @@ -0,0 +1,8 @@ +//go:build ginkgoclidependencies +// +build ginkgoclidependencies + +package ginkgo + +import ( + _ "github.com/onsi/ginkgo/v2/ginkgo" +) diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go index 1beeb1144..c5a7eb94d 100644 --- a/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go @@ -7,7 +7,7 @@ GinkgoT() implements an interface analogous to *testing.T and can be used with third-party libraries that accept *testing.T through an interface. GinkgoT() takes an optional offset argument that can be used to get the -correct line number associated with the failure. +correct line number associated with the failure - though you do not need to use this if you call GinkgoHelper() or GinkgoT().Helper() appropriately You can learn more here: https://onsi.github.io/ginkgo/#using-third-party-libraries */ diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/group.go b/vendor/github.com/onsi/ginkgo/v2/internal/group.go index ce7e72e82..ae1b7b011 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/group.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/group.go @@ -94,15 +94,19 @@ type group struct { runOncePairs map[uint]runOncePairs runOnceTracker map[runOncePair]types.SpecState - succeeded bool + succeeded bool + failedInARunOnceBefore bool + continueOnFailure bool } func newGroup(suite *Suite) *group { return &group{ - suite: suite, - runOncePairs: map[uint]runOncePairs{}, - runOnceTracker: map[runOncePair]types.SpecState{}, - succeeded: true, + suite: suite, + runOncePairs: map[uint]runOncePairs{}, + runOnceTracker: map[runOncePair]types.SpecState{}, + succeeded: true, + failedInARunOnceBefore: false, + continueOnFailure: false, } } @@ -116,8 +120,11 @@ func (g *group) initialReportForSpec(spec Spec) types.SpecReport { LeafNodeText: spec.FirstNodeWithType(types.NodeTypeIt).Text, LeafNodeLabels: []string(spec.FirstNodeWithType(types.NodeTypeIt).Labels), ParallelProcess: g.suite.config.ParallelProcess, + RunningInParallel: g.suite.isRunningInParallel(), IsSerial: spec.Nodes.HasNodeMarkedSerial(), IsInOrderedContainer: !spec.Nodes.FirstNodeMarkedOrdered().IsZero(), + MaxFlakeAttempts: spec.Nodes.GetMaxFlakeAttempts(), + MaxMustPassRepeatedly: spec.Nodes.GetMaxMustPassRepeatedly(), } } @@ -128,13 +135,20 @@ func (g *group) evaluateSkipStatus(spec Spec) (types.SpecState, types.Failure) { if spec.Skip { return types.SpecStateSkipped, types.Failure{} } - if g.suite.interruptHandler.Status().Interrupted || g.suite.skipAll { + if g.suite.interruptHandler.Status().Interrupted() || g.suite.skipAll { return types.SpecStateSkipped, types.Failure{} } - if !g.succeeded { + if !g.suite.deadline.IsZero() && g.suite.deadline.Before(time.Now()) { + return types.SpecStateSkipped, types.Failure{} + } + if !g.succeeded && !g.continueOnFailure { return types.SpecStateSkipped, g.suite.failureForLeafNodeWithMessage(spec.FirstNodeWithType(types.NodeTypeIt), "Spec skipped because an earlier spec in an ordered container failed") } + if g.failedInARunOnceBefore && g.continueOnFailure { + return types.SpecStateSkipped, g.suite.failureForLeafNodeWithMessage(spec.FirstNodeWithType(types.NodeTypeIt), + "Spec skipped because a BeforeAll node failed") + } beforeOncePairs := g.runOncePairs[spec.SubjectID()].withType(types.NodeTypeBeforeAll | types.NodeTypeBeforeEach | types.NodeTypeJustBeforeEach) for _, pair := range beforeOncePairs { if g.runOnceTracker[pair].Is(types.SpecStateSkipped) { @@ -162,9 +176,8 @@ func (g *group) isLastSpecWithPair(specID uint, pair runOncePair) bool { return lastSpecID == specID } -func (g *group) attemptSpec(isFinalAttempt bool, spec Spec) { - interruptStatus := g.suite.interruptHandler.Status() - +func (g *group) attemptSpec(isFinalAttempt bool, spec Spec) bool { + failedInARunOnceBefore := false pairs := g.runOncePairs[spec.SubjectID()] nodes := spec.Nodes.WithType(types.NodeTypeBeforeAll) @@ -173,18 +186,24 @@ func (g *group) attemptSpec(isFinalAttempt bool, spec Spec) { nodes = append(nodes, spec.Nodes.FirstNodeWithType(types.NodeTypeIt)) terminatingNode, terminatingPair := Node{}, runOncePair{} + deadline := time.Time{} + if spec.SpecTimeout() > 0 { + deadline = time.Now().Add(spec.SpecTimeout()) + } + for _, node := range nodes { oncePair := pairs.runOncePairFor(node.ID) if !oncePair.isZero() && g.runOnceTracker[oncePair].Is(types.SpecStatePassed) { continue } - g.suite.currentSpecReport.State, g.suite.currentSpecReport.Failure = g.suite.runNode(node, interruptStatus.Channel, spec.Nodes.BestTextFor(node)) + g.suite.currentSpecReport.State, g.suite.currentSpecReport.Failure = g.suite.runNode(node, deadline, spec.Nodes.BestTextFor(node)) g.suite.currentSpecReport.RunTime = time.Since(g.suite.currentSpecReport.StartTime) if !oncePair.isZero() { g.runOnceTracker[oncePair] = g.suite.currentSpecReport.State } if g.suite.currentSpecReport.State != types.SpecStatePassed { terminatingNode, terminatingPair = node, oncePair + failedInARunOnceBefore = !terminatingPair.isZero() break } } @@ -207,7 +226,7 @@ func (g *group) attemptSpec(isFinalAttempt bool, spec Spec) { //this node has already been run on this attempt, don't rerun it return false } - pair := runOncePair{} + var pair runOncePair switch node.NodeType { case types.NodeTypeCleanupAfterEach, types.NodeTypeCleanupAfterAll: // check if we were generated in an AfterNode that has already run @@ -237,9 +256,13 @@ func (g *group) attemptSpec(isFinalAttempt bool, spec Spec) { if !terminatingPair.isZero() && terminatingNode.NestingLevel == node.NestingLevel { return true //...or, a run-once node at our nesting level was skipped which means this is our last chance to run } - case types.SpecStateFailed, types.SpecStatePanicked: // the spec has failed... + case types.SpecStateFailed, types.SpecStatePanicked, types.SpecStateTimedout: // the spec has failed... if isFinalAttempt { - return true //...if this was the last attempt then we're the last spec to run and so the AfterNode should run + if g.continueOnFailure { + return isLastSpecWithPair || failedInARunOnceBefore //...we're configured to continue on failures - so we should only run if we're the last spec for this pair or if we failed in a runOnceBefore (which means we _are_ the last spec to run) + } else { + return true //...this was the last attempt and continueOnFailure is false therefore we are the last spec to run and so the AfterNode should run + } } if !terminatingPair.isZero() { // ...and it failed in a run-once. which will be running again if node.NodeType.Is(types.NodeTypeCleanupAfterEach | types.NodeTypeCleanupAfterAll) { @@ -260,20 +283,24 @@ func (g *group) attemptSpec(isFinalAttempt bool, spec Spec) { for _, node := range nodes { afterNodeWasRun[node.ID] = true - state, failure := g.suite.runNode(node, g.suite.interruptHandler.Status().Channel, spec.Nodes.BestTextFor(node)) + state, failure := g.suite.runNode(node, deadline, spec.Nodes.BestTextFor(node)) g.suite.currentSpecReport.RunTime = time.Since(g.suite.currentSpecReport.StartTime) if g.suite.currentSpecReport.State == types.SpecStatePassed || state == types.SpecStateAborted { g.suite.currentSpecReport.State = state g.suite.currentSpecReport.Failure = failure + } else if state.Is(types.SpecStateFailureStates) { + g.suite.currentSpecReport.AdditionalFailures = append(g.suite.currentSpecReport.AdditionalFailures, types.AdditionalFailure{State: state, Failure: failure}) } } includeDeferCleanups = true } + return failedInARunOnceBefore } func (g *group) run(specs Specs) { g.specs = specs + g.continueOnFailure = specs[0].Nodes.FirstNodeMarkedOrdered().MarkedContinueOnFailure for _, spec := range g.specs { g.runOncePairs[spec.SubjectID()] = runOncePairsForSpec(spec) } @@ -290,28 +317,52 @@ func (g *group) run(specs Specs) { skip := g.suite.config.DryRun || g.suite.currentSpecReport.State.Is(types.SpecStateFailureStates|types.SpecStateSkipped|types.SpecStatePending) g.suite.currentSpecReport.StartTime = time.Now() + failedInARunOnceBefore := false if !skip { - maxAttempts := max(1, spec.FlakeAttempts()) - if g.suite.config.FlakeAttempts > 0 { + var maxAttempts = 1 + + if g.suite.currentSpecReport.MaxMustPassRepeatedly > 0 { + maxAttempts = max(1, spec.MustPassRepeatedly()) + } else if g.suite.config.FlakeAttempts > 0 { maxAttempts = g.suite.config.FlakeAttempts + g.suite.currentSpecReport.MaxFlakeAttempts = maxAttempts + } else if g.suite.currentSpecReport.MaxFlakeAttempts > 0 { + maxAttempts = max(1, spec.FlakeAttempts()) } + for attempt := 0; attempt < maxAttempts; attempt++ { g.suite.currentSpecReport.NumAttempts = attempt + 1 g.suite.writer.Truncate() g.suite.outputInterceptor.StartInterceptingOutput() if attempt > 0 { - fmt.Fprintf(g.suite.writer, "\nGinkgo: Attempt #%d Failed. Retrying...\n", attempt) + if g.suite.currentSpecReport.MaxMustPassRepeatedly > 0 { + g.suite.handleSpecEvent(types.SpecEvent{SpecEventType: types.SpecEventSpecRepeat, Attempt: attempt}) + } + if g.suite.currentSpecReport.MaxFlakeAttempts > 0 { + g.suite.handleSpecEvent(types.SpecEvent{SpecEventType: types.SpecEventSpecRetry, Attempt: attempt}) + } } - g.attemptSpec(attempt == maxAttempts-1, spec) + failedInARunOnceBefore = g.attemptSpec(attempt == maxAttempts-1, spec) g.suite.currentSpecReport.EndTime = time.Now() g.suite.currentSpecReport.RunTime = g.suite.currentSpecReport.EndTime.Sub(g.suite.currentSpecReport.StartTime) g.suite.currentSpecReport.CapturedGinkgoWriterOutput += string(g.suite.writer.Bytes()) g.suite.currentSpecReport.CapturedStdOutErr += g.suite.outputInterceptor.StopInterceptingAndReturnOutput() - if g.suite.currentSpecReport.State.Is(types.SpecStatePassed | types.SpecStateSkipped | types.SpecStateAborted | types.SpecStateInterrupted) { - break + if g.suite.currentSpecReport.MaxMustPassRepeatedly > 0 { + if g.suite.currentSpecReport.State.Is(types.SpecStateFailureStates | types.SpecStateSkipped) { + break + } + } + if g.suite.currentSpecReport.MaxFlakeAttempts > 0 { + if g.suite.currentSpecReport.State.Is(types.SpecStatePassed | types.SpecStateSkipped | types.SpecStateAborted | types.SpecStateInterrupted) { + break + } else if attempt < maxAttempts-1 { + af := types.AdditionalFailure{State: g.suite.currentSpecReport.State, Failure: g.suite.currentSpecReport.Failure} + af.Failure.Message = fmt.Sprintf("Failure recorded during attempt %d:\n%s", attempt+1, af.Failure.Message) + g.suite.currentSpecReport.AdditionalFailures = append(g.suite.currentSpecReport.AdditionalFailures, af) + } } } } @@ -320,230 +371,10 @@ func (g *group) run(specs Specs) { g.suite.processCurrentSpecReport() if g.suite.currentSpecReport.State.Is(types.SpecStateFailureStates) { g.succeeded = false + g.failedInARunOnceBefore = g.failedInARunOnceBefore || failedInARunOnceBefore } g.suite.selectiveLock.Lock() g.suite.currentSpecReport = types.SpecReport{} g.suite.selectiveLock.Unlock() } } - -func (g *group) oldRun(specs Specs) { - var suite = g.suite - nodeState := map[uint]types.SpecState{} - groupSucceeded := true - - indexOfLastSpecContainingNodeID := func(id uint) int { - lastIdx := -1 - for idx := range specs { - if specs[idx].Nodes.ContainsNodeID(id) && !specs[idx].Skip { - lastIdx = idx - } - } - return lastIdx - } - - for i, spec := range specs { - suite.currentSpecReport = types.SpecReport{ - ContainerHierarchyTexts: spec.Nodes.WithType(types.NodeTypeContainer).Texts(), - ContainerHierarchyLocations: spec.Nodes.WithType(types.NodeTypeContainer).CodeLocations(), - ContainerHierarchyLabels: spec.Nodes.WithType(types.NodeTypeContainer).Labels(), - LeafNodeLocation: spec.FirstNodeWithType(types.NodeTypeIt).CodeLocation, - LeafNodeType: types.NodeTypeIt, - LeafNodeText: spec.FirstNodeWithType(types.NodeTypeIt).Text, - LeafNodeLabels: []string(spec.FirstNodeWithType(types.NodeTypeIt).Labels), - ParallelProcess: suite.config.ParallelProcess, - IsSerial: spec.Nodes.HasNodeMarkedSerial(), - IsInOrderedContainer: !spec.Nodes.FirstNodeMarkedOrdered().IsZero(), - } - - skip := spec.Skip - if spec.Nodes.HasNodeMarkedPending() { - skip = true - suite.currentSpecReport.State = types.SpecStatePending - } else { - if suite.interruptHandler.Status().Interrupted || suite.skipAll { - skip = true - } - if !groupSucceeded { - skip = true - suite.currentSpecReport.Failure = suite.failureForLeafNodeWithMessage(spec.FirstNodeWithType(types.NodeTypeIt), - "Spec skipped because an earlier spec in an ordered container failed") - } - for _, node := range spec.Nodes.WithType(types.NodeTypeBeforeAll) { - if nodeState[node.ID] == types.SpecStateSkipped { - skip = true - suite.currentSpecReport.Failure = suite.failureForLeafNodeWithMessage(spec.FirstNodeWithType(types.NodeTypeIt), - "Spec skipped because Skip() was called in BeforeAll") - break - } - } - if skip { - suite.currentSpecReport.State = types.SpecStateSkipped - } - } - - if suite.config.DryRun && !skip { - skip = true - suite.currentSpecReport.State = types.SpecStatePassed - } - - suite.reporter.WillRun(suite.currentSpecReport) - //send the spec report to any attached ReportBeforeEach blocks - this will update suite.currentSpecReport if failures occur in these blocks - suite.reportEach(spec, types.NodeTypeReportBeforeEach) - if suite.currentSpecReport.State.Is(types.SpecStateFailureStates) { - //the reportEach failed, skip this spec - skip = true - } - - suite.currentSpecReport.StartTime = time.Now() - maxAttempts := max(1, spec.FlakeAttempts()) - if suite.config.FlakeAttempts > 0 { - maxAttempts = suite.config.FlakeAttempts - } - - for attempt := 0; !skip && (attempt < maxAttempts); attempt++ { - suite.currentSpecReport.NumAttempts = attempt + 1 - suite.writer.Truncate() - suite.outputInterceptor.StartInterceptingOutput() - if attempt > 0 { - fmt.Fprintf(suite.writer, "\nGinkgo: Attempt #%d Failed. Retrying...\n", attempt) - } - isFinalAttempt := (attempt == maxAttempts-1) - - interruptStatus := suite.interruptHandler.Status() - deepestNestingLevelAttained := -1 - var nodes = spec.Nodes.WithType(types.NodeTypeBeforeAll).Filter(func(n Node) bool { - return nodeState[n.ID] != types.SpecStatePassed - }) - nodes = nodes.CopyAppend(spec.Nodes.WithType(types.NodeTypeBeforeEach)...).SortedByAscendingNestingLevel() - nodes = nodes.CopyAppend(spec.Nodes.WithType(types.NodeTypeJustBeforeEach).SortedByAscendingNestingLevel()...) - nodes = nodes.CopyAppend(spec.Nodes.WithType(types.NodeTypeIt)...) - - var terminatingNode Node - for j := range nodes { - deepestNestingLevelAttained = max(deepestNestingLevelAttained, nodes[j].NestingLevel) - suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(nodes[j], interruptStatus.Channel, spec.Nodes.BestTextFor(nodes[j])) - suite.currentSpecReport.RunTime = time.Since(suite.currentSpecReport.StartTime) - nodeState[nodes[j].ID] = suite.currentSpecReport.State - if suite.currentSpecReport.State != types.SpecStatePassed { - terminatingNode = nodes[j] - break - } - } - - afterAllNodesThatRan := map[uint]bool{} - // pull out some shared code so we aren't repeating ourselves down below. this just runs after and cleanup nodes - runAfterAndCleanupNodes := func(nodes Nodes) { - for j := range nodes { - state, failure := suite.runNode(nodes[j], suite.interruptHandler.Status().Channel, spec.Nodes.BestTextFor(nodes[j])) - suite.currentSpecReport.RunTime = time.Since(suite.currentSpecReport.StartTime) - nodeState[nodes[j].ID] = state - if suite.currentSpecReport.State == types.SpecStatePassed || state == types.SpecStateAborted { - suite.currentSpecReport.State = state - suite.currentSpecReport.Failure = failure - if state != types.SpecStatePassed { - terminatingNode = nodes[j] - } - } - if nodes[j].NodeType.Is(types.NodeTypeAfterAll) { - afterAllNodesThatRan[nodes[j].ID] = true - } - } - } - - // pull out a helper that captures the logic of whether or not we should run a given After node. - // there is complexity here stemming from the fact that we allow nested ordered contexts and flakey retries - shouldRunAfterNode := func(n Node) bool { - if n.NodeType.Is(types.NodeTypeAfterEach | types.NodeTypeJustAfterEach) { - return true - } - var id uint - if n.NodeType.Is(types.NodeTypeAfterAll) { - id = n.ID - if afterAllNodesThatRan[id] { //we've already run on this attempt. don't run again. - return false - } - } - if n.NodeType.Is(types.NodeTypeCleanupAfterAll) { - id = n.NodeIDWhereCleanupWasGenerated - } - isLastSpecWithNode := indexOfLastSpecContainingNodeID(id) == i - - switch suite.currentSpecReport.State { - case types.SpecStatePassed: //we've passed so far... - return isLastSpecWithNode //... and we're the last spec with this AfterNode, so we should run it - case types.SpecStateSkipped: //the spec was skipped by the user... - if isLastSpecWithNode { - return true //...we're the last spec, so we should run the AfterNode - } - if terminatingNode.NodeType.Is(types.NodeTypeBeforeAll) && terminatingNode.NestingLevel == n.NestingLevel { - return true //...or, a BeforeAll was skipped and it's at our nesting level, so our subgroup is going to skip - } - case types.SpecStateFailed, types.SpecStatePanicked: // the spec has failed... - if isFinalAttempt { - return true //...if this was the last attempt then we're the last spec to run and so the AfterNode should run - } - if terminatingNode.NodeType.Is(types.NodeTypeBeforeAll) { - //...we'll be rerunning a BeforeAll so we should cleanup after it if... - if n.NodeType.Is(types.NodeTypeAfterAll) && terminatingNode.NestingLevel == n.NestingLevel { - return true //we're at the same nesting level - } - if n.NodeType.Is(types.NodeTypeCleanupAfterAll) && terminatingNode.ID == n.NodeIDWhereCleanupWasGenerated { - return true //we're a DeferCleanup generated by it - } - } - if terminatingNode.NodeType.Is(types.NodeTypeAfterAll) { - //...we'll be rerunning an AfterAll so we should cleanup after it if... - if n.NodeType.Is(types.NodeTypeCleanupAfterAll) && terminatingNode.ID == n.NodeIDWhereCleanupWasGenerated { - return true //we're a DeferCleanup generated by it - } - } - case types.SpecStateInterrupted, types.SpecStateAborted: // ...we've been interrupted and/or aborted - return true //...that means the test run is over and we should clean up the stack. Run the AfterNode - } - return false - } - - // first pass - run all the JustAfterEach, Aftereach, and AfterAlls. Our shoudlRunAfterNode filter function will clean up the AfterAlls for us. - afterNodes := spec.Nodes.WithType(types.NodeTypeJustAfterEach).SortedByDescendingNestingLevel() - afterNodes = afterNodes.CopyAppend(spec.Nodes.WithType(types.NodeTypeAfterEach).CopyAppend(spec.Nodes.WithType(types.NodeTypeAfterAll)...).SortedByDescendingNestingLevel()...) - afterNodes = afterNodes.WithinNestingLevel(deepestNestingLevelAttained) - afterNodes = afterNodes.Filter(shouldRunAfterNode) - runAfterAndCleanupNodes(afterNodes) - - // second-pass perhaps we didn't run the AfterAlls but a state change due to an AfterEach now requires us to run the AfterAlls: - afterNodes = spec.Nodes.WithType(types.NodeTypeAfterAll).WithinNestingLevel(deepestNestingLevelAttained).Filter(shouldRunAfterNode) - runAfterAndCleanupNodes(afterNodes) - - // now we run any DeferCleanups - afterNodes = suite.cleanupNodes.WithType(types.NodeTypeCleanupAfterEach).Reverse() - afterNodes = append(afterNodes, suite.cleanupNodes.WithType(types.NodeTypeCleanupAfterAll).Filter(shouldRunAfterNode).Reverse()...) - runAfterAndCleanupNodes(afterNodes) - - // third-pass, perhaps a DeferCleanup failed and now we need to run the AfterAlls. - afterNodes = spec.Nodes.WithType(types.NodeTypeAfterAll).WithinNestingLevel(deepestNestingLevelAttained).Filter(shouldRunAfterNode) - runAfterAndCleanupNodes(afterNodes) - - // and finally - running AfterAlls may have generated some new DeferCleanup nodes, let's run them to finish up - afterNodes = suite.cleanupNodes.WithType(types.NodeTypeCleanupAfterAll).Reverse().Filter(shouldRunAfterNode) - runAfterAndCleanupNodes(afterNodes) - - suite.currentSpecReport.EndTime = time.Now() - suite.currentSpecReport.RunTime = suite.currentSpecReport.EndTime.Sub(suite.currentSpecReport.StartTime) - suite.currentSpecReport.CapturedGinkgoWriterOutput += string(suite.writer.Bytes()) - suite.currentSpecReport.CapturedStdOutErr += suite.outputInterceptor.StopInterceptingAndReturnOutput() - - if suite.currentSpecReport.State.Is(types.SpecStatePassed | types.SpecStateSkipped | types.SpecStateAborted | types.SpecStateInterrupted) { - break - } - } - - //send the spec report to any attached ReportAfterEach blocks - this will update suite.currentSpecReport if failures occur in these blocks - suite.reportEach(spec, types.NodeTypeReportAfterEach) - suite.processCurrentSpecReport() - if suite.currentSpecReport.State.Is(types.SpecStateFailureStates) { - groupSucceeded = false - } - suite.currentSpecReport = types.SpecReport{} - } -} diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/interrupt_handler/interrupt_handler.go b/vendor/github.com/onsi/ginkgo/v2/internal/interrupt_handler/interrupt_handler.go index ad224bc59..ac6f51040 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/interrupt_handler/interrupt_handler.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/interrupt_handler/interrupt_handler.go @@ -1,7 +1,6 @@ package interrupt_handler import ( - "fmt" "os" "os/signal" "sync" @@ -11,27 +10,29 @@ import ( "github.com/onsi/ginkgo/v2/internal/parallel_support" ) -const TIMEOUT_REPEAT_INTERRUPT_MAXIMUM_DURATION = 30 * time.Second -const TIMEOUT_REPEAT_INTERRUPT_FRACTION_OF_TIMEOUT = 10 const ABORT_POLLING_INTERVAL = 500 * time.Millisecond -const ABORT_REPEAT_INTERRUPT_DURATION = 30 * time.Second type InterruptCause uint const ( InterruptCauseInvalid InterruptCause = iota - InterruptCauseSignal - InterruptCauseTimeout InterruptCauseAbortByOtherProcess ) +type InterruptLevel uint + +const ( + InterruptLevelUninterrupted InterruptLevel = iota + InterruptLevelCleanupAndReport + InterruptLevelReportOnly + InterruptLevelBailOut +) + func (ic InterruptCause) String() string { switch ic { case InterruptCauseSignal: return "Interrupted by User" - case InterruptCauseTimeout: - return "Interrupted by Timeout" case InterruptCauseAbortByOtherProcess: return "Interrupted by Other Ginkgo Process" } @@ -39,37 +40,49 @@ func (ic InterruptCause) String() string { } type InterruptStatus struct { - Interrupted bool - Channel chan interface{} - Cause InterruptCause + Channel chan interface{} + Level InterruptLevel + Cause InterruptCause +} + +func (s InterruptStatus) Interrupted() bool { + return s.Level != InterruptLevelUninterrupted +} + +func (s InterruptStatus) Message() string { + return s.Cause.String() +} + +func (s InterruptStatus) ShouldIncludeProgressReport() bool { + return s.Cause != InterruptCauseAbortByOtherProcess } type InterruptHandlerInterface interface { Status() InterruptStatus - SetInterruptPlaceholderMessage(string) - ClearInterruptPlaceholderMessage() - InterruptMessage() (string, bool) } type InterruptHandler struct { - c chan interface{} - lock *sync.Mutex - interrupted bool - interruptPlaceholderMessage string - interruptCause InterruptCause - client parallel_support.Client - stop chan interface{} + c chan interface{} + lock *sync.Mutex + level InterruptLevel + cause InterruptCause + client parallel_support.Client + stop chan interface{} + signals []os.Signal } -func NewInterruptHandler(timeout time.Duration, client parallel_support.Client) *InterruptHandler { +func NewInterruptHandler(client parallel_support.Client, signals ...os.Signal) *InterruptHandler { + if len(signals) == 0 { + signals = []os.Signal{os.Interrupt, syscall.SIGTERM} + } handler := &InterruptHandler{ - c: make(chan interface{}), - lock: &sync.Mutex{}, - interrupted: false, - stop: make(chan interface{}), - client: client, + c: make(chan interface{}), + lock: &sync.Mutex{}, + stop: make(chan interface{}), + client: client, + signals: signals, } - handler.registerForInterrupts(timeout) + handler.registerForInterrupts() return handler } @@ -77,30 +90,22 @@ func (handler *InterruptHandler) Stop() { close(handler.stop) } -func (handler *InterruptHandler) registerForInterrupts(timeout time.Duration) { +func (handler *InterruptHandler) registerForInterrupts() { // os signal handling signalChannel := make(chan os.Signal, 1) - signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM) - - // timeout handling - var timeoutChannel <-chan time.Time - var timeoutTimer *time.Timer - if timeout > 0 { - timeoutTimer = time.NewTimer(timeout) - timeoutChannel = timeoutTimer.C - } + signal.Notify(signalChannel, handler.signals...) // cross-process abort handling - var abortChannel chan bool + var abortChannel chan interface{} if handler.client != nil { - abortChannel = make(chan bool) + abortChannel = make(chan interface{}) go func() { pollTicker := time.NewTicker(ABORT_POLLING_INTERVAL) for { select { case <-pollTicker.C: if handler.client.ShouldAbort() { - abortChannel <- true + close(abortChannel) pollTicker.Stop() return } @@ -112,55 +117,37 @@ func (handler *InterruptHandler) registerForInterrupts(timeout time.Duration) { }() } - // listen for any interrupt signals - // note that some (timeouts, cross-process aborts) will only trigger once - // for these we set up a ticker to keep interrupting the suite until it ends - // this ensures any `AfterEach` or `AfterSuite`s that get stuck cleaning up - // get interrupted eventually - go func() { + go func(abortChannel chan interface{}) { var interruptCause InterruptCause - var repeatChannel <-chan time.Time - var repeatTicker *time.Ticker for { select { case <-signalChannel: interruptCause = InterruptCauseSignal - case <-timeoutChannel: - interruptCause = InterruptCauseTimeout - repeatInterruptTimeout := timeout / time.Duration(TIMEOUT_REPEAT_INTERRUPT_FRACTION_OF_TIMEOUT) - if repeatInterruptTimeout > TIMEOUT_REPEAT_INTERRUPT_MAXIMUM_DURATION { - repeatInterruptTimeout = TIMEOUT_REPEAT_INTERRUPT_MAXIMUM_DURATION - } - timeoutTimer.Stop() - repeatTicker = time.NewTicker(repeatInterruptTimeout) - repeatChannel = repeatTicker.C case <-abortChannel: interruptCause = InterruptCauseAbortByOtherProcess - repeatTicker = time.NewTicker(ABORT_REPEAT_INTERRUPT_DURATION) - repeatChannel = repeatTicker.C - case <-repeatChannel: - //do nothing, just interrupt again using the same interruptCause case <-handler.stop: - if timeoutTimer != nil { - timeoutTimer.Stop() - } - if repeatTicker != nil { - repeatTicker.Stop() - } signal.Stop(signalChannel) return } + abortChannel = nil + handler.lock.Lock() - handler.interruptCause = interruptCause - if handler.interruptPlaceholderMessage != "" { - fmt.Println(handler.interruptPlaceholderMessage) + oldLevel := handler.level + handler.cause = interruptCause + if handler.level == InterruptLevelUninterrupted { + handler.level = InterruptLevelCleanupAndReport + } else if handler.level == InterruptLevelCleanupAndReport { + handler.level = InterruptLevelReportOnly + } else if handler.level == InterruptLevelReportOnly { + handler.level = InterruptLevelBailOut + } + if handler.level != oldLevel { + close(handler.c) + handler.c = make(chan interface{}) } - handler.interrupted = true - close(handler.c) - handler.c = make(chan interface{}) handler.lock.Unlock() } - }() + }(abortChannel) } func (handler *InterruptHandler) Status() InterruptStatus { @@ -168,29 +155,8 @@ func (handler *InterruptHandler) Status() InterruptStatus { defer handler.lock.Unlock() return InterruptStatus{ - Interrupted: handler.interrupted, - Channel: handler.c, - Cause: handler.interruptCause, + Level: handler.level, + Channel: handler.c, + Cause: handler.cause, } } - -func (handler *InterruptHandler) SetInterruptPlaceholderMessage(message string) { - handler.lock.Lock() - defer handler.lock.Unlock() - - handler.interruptPlaceholderMessage = message -} - -func (handler *InterruptHandler) ClearInterruptPlaceholderMessage() { - handler.lock.Lock() - defer handler.lock.Unlock() - - handler.interruptPlaceholderMessage = "" -} - -func (handler *InterruptHandler) InterruptMessage() (string, bool) { - handler.lock.Lock() - out := fmt.Sprintf("%s", handler.interruptCause.String()) - defer handler.lock.Unlock() - return out, handler.interruptCause != InterruptCauseAbortByOtherProcess -} diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/node.go b/vendor/github.com/onsi/ginkgo/v2/internal/node.go index 3dbf0b478..0869bffb3 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/node.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/node.go @@ -1,6 +1,7 @@ package internal import ( + "context" "fmt" "reflect" "sort" @@ -28,29 +29,38 @@ type Node struct { NodeType types.NodeType Text string - Body func() + Body func(SpecContext) CodeLocation types.CodeLocation NestingLevel int - - SynchronizedBeforeSuiteProc1Body func() []byte - SynchronizedBeforeSuiteAllProcsBody func([]byte) - - SynchronizedAfterSuiteAllProcsBody func() - SynchronizedAfterSuiteProc1Body func() - - ReportEachBody func(types.SpecReport) - ReportAfterSuiteBody func(types.Report) - - MarkedFocus bool - MarkedPending bool - MarkedSerial bool - MarkedOrdered bool - MarkedOncePerOrdered bool - MarkedSuppressProgressReporting bool - FlakeAttempts int - Labels Labels - PollProgressAfter time.Duration - PollProgressInterval time.Duration + HasContext bool + + SynchronizedBeforeSuiteProc1Body func(SpecContext) []byte + SynchronizedBeforeSuiteProc1BodyHasContext bool + SynchronizedBeforeSuiteAllProcsBody func(SpecContext, []byte) + SynchronizedBeforeSuiteAllProcsBodyHasContext bool + + SynchronizedAfterSuiteAllProcsBody func(SpecContext) + SynchronizedAfterSuiteAllProcsBodyHasContext bool + SynchronizedAfterSuiteProc1Body func(SpecContext) + SynchronizedAfterSuiteProc1BodyHasContext bool + + ReportEachBody func(types.SpecReport) + ReportSuiteBody func(types.Report) + + MarkedFocus bool + MarkedPending bool + MarkedSerial bool + MarkedOrdered bool + MarkedContinueOnFailure bool + MarkedOncePerOrdered bool + FlakeAttempts int + MustPassRepeatedly int + Labels Labels + PollProgressAfter time.Duration + PollProgressInterval time.Duration + NodeTimeout time.Duration + SpecTimeout time.Duration + GracePeriod time.Duration NodeIDWhereCleanupWasGenerated uint } @@ -60,6 +70,7 @@ type focusType bool type pendingType bool type serialType bool type orderedType bool +type continueOnFailureType bool type honorsOrderedType bool type suppressProgressReporting bool @@ -67,15 +78,24 @@ const Focus = focusType(true) const Pending = pendingType(true) const Serial = serialType(true) const Ordered = orderedType(true) +const ContinueOnFailure = continueOnFailureType(true) const OncePerOrdered = honorsOrderedType(true) const SuppressProgressReporting = suppressProgressReporting(true) type FlakeAttempts uint +type MustPassRepeatedly uint type Offset uint type Done chan<- interface{} // Deprecated Done Channel for asynchronous testing type Labels []string type PollProgressInterval time.Duration type PollProgressAfter time.Duration +type NodeTimeout time.Duration +type SpecTimeout time.Duration +type GracePeriod time.Duration + +func (l Labels) MatchesLabelFilter(query string) bool { + return types.MustParseLabelFilter(query)(l) +} func UnionOfLabels(labels ...Labels) Labels { out := Labels{} @@ -120,18 +140,28 @@ func isDecoration(arg interface{}) bool { return true case t == reflect.TypeOf(Ordered): return true + case t == reflect.TypeOf(ContinueOnFailure): + return true case t == reflect.TypeOf(OncePerOrdered): return true case t == reflect.TypeOf(SuppressProgressReporting): return true case t == reflect.TypeOf(FlakeAttempts(0)): return true + case t == reflect.TypeOf(MustPassRepeatedly(0)): + return true case t == reflect.TypeOf(Labels{}): return true case t == reflect.TypeOf(PollProgressInterval(0)): return true case t == reflect.TypeOf(PollProgressAfter(0)): return true + case t == reflect.TypeOf(NodeTimeout(0)): + return true + case t == reflect.TypeOf(SpecTimeout(0)): + return true + case t == reflect.TypeOf(GracePeriod(0)): + return true case t.Kind() == reflect.Slice && isSliceOfDecorations(arg): return true default: @@ -152,6 +182,9 @@ func isSliceOfDecorations(slice interface{}) bool { return true } +var contextType = reflect.TypeOf(new(context.Context)).Elem() +var specContextType = reflect.TypeOf(new(SpecContext)).Elem() + func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeType, text string, args ...interface{}) (Node, []error) { baseOffset := 2 node := Node{ @@ -163,8 +196,9 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy NestingLevel: -1, PollProgressAfter: -1, PollProgressInterval: -1, + GracePeriod: -1, } - + errors := []error{} appendError := func(err error) { if err != nil { @@ -216,21 +250,28 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy if !nodeType.Is(types.NodeTypeContainer) { appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "Ordered")) } + case t == reflect.TypeOf(ContinueOnFailure): + node.MarkedContinueOnFailure = bool(arg.(continueOnFailureType)) + if !nodeType.Is(types.NodeTypeContainer) { + appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "ContinueOnFailure")) + } case t == reflect.TypeOf(OncePerOrdered): node.MarkedOncePerOrdered = bool(arg.(honorsOrderedType)) if !nodeType.Is(types.NodeTypeBeforeEach | types.NodeTypeJustBeforeEach | types.NodeTypeAfterEach | types.NodeTypeJustAfterEach) { appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "OncePerOrdered")) } case t == reflect.TypeOf(SuppressProgressReporting): - node.MarkedSuppressProgressReporting = bool(arg.(suppressProgressReporting)) - if nodeType.Is(types.NodeTypeContainer) { - appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "SuppressProgressReporting")) - } + deprecationTracker.TrackDeprecation(types.Deprecations.SuppressProgressReporting()) case t == reflect.TypeOf(FlakeAttempts(0)): node.FlakeAttempts = int(arg.(FlakeAttempts)) if !nodeType.Is(types.NodeTypesForContainerAndIt) { appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "FlakeAttempts")) } + case t == reflect.TypeOf(MustPassRepeatedly(0)): + node.MustPassRepeatedly = int(arg.(MustPassRepeatedly)) + if !nodeType.Is(types.NodeTypesForContainerAndIt) { + appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "MustPassRepeatedly")) + } case t == reflect.TypeOf(PollProgressAfter(0)): node.PollProgressAfter = time.Duration(arg.(PollProgressAfter)) if nodeType.Is(types.NodeTypeContainer) { @@ -241,6 +282,21 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy if nodeType.Is(types.NodeTypeContainer) { appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "PollProgressInterval")) } + case t == reflect.TypeOf(NodeTimeout(0)): + node.NodeTimeout = time.Duration(arg.(NodeTimeout)) + if nodeType.Is(types.NodeTypeContainer) { + appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "NodeTimeout")) + } + case t == reflect.TypeOf(SpecTimeout(0)): + node.SpecTimeout = time.Duration(arg.(SpecTimeout)) + if !nodeType.Is(types.NodeTypeIt) { + appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "SpecTimeout")) + } + case t == reflect.TypeOf(GracePeriod(0)): + node.GracePeriod = time.Duration(arg.(GracePeriod)) + if nodeType.Is(types.NodeTypeContainer) { + appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "GracePeriod")) + } case t == reflect.TypeOf(Labels{}): if !nodeType.Is(types.NodeTypesForContainerAndIt) { appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "Label")) @@ -254,7 +310,20 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy } } case t.Kind() == reflect.Func: - if nodeType.Is(types.NodeTypeReportBeforeEach | types.NodeTypeReportAfterEach) { + if nodeType.Is(types.NodeTypeContainer) { + if node.Body != nil { + appendError(types.GinkgoErrors.MultipleBodyFunctions(node.CodeLocation, nodeType)) + trackedFunctionError = true + break + } + if t.NumOut() > 0 || t.NumIn() > 0 { + appendError(types.GinkgoErrors.InvalidBodyTypeForContainer(t, node.CodeLocation, nodeType)) + trackedFunctionError = true + break + } + body := arg.(func()) + node.Body = func(SpecContext) { body() } + } else if nodeType.Is(types.NodeTypeReportBeforeEach | types.NodeTypeReportAfterEach) { if node.ReportEachBody == nil { node.ReportEachBody = arg.(func(types.SpecReport)) } else { @@ -262,53 +331,64 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy trackedFunctionError = true break } - } else if nodeType.Is(types.NodeTypeSynchronizedBeforeSuite) { - if node.SynchronizedBeforeSuiteProc1Body == nil { - node.SynchronizedBeforeSuiteProc1Body = arg.(func() []byte) - } else if node.SynchronizedBeforeSuiteAllProcsBody == nil { - node.SynchronizedBeforeSuiteAllProcsBody = arg.(func([]byte)) + } else if nodeType.Is(types.NodeTypeReportBeforeSuite | types.NodeTypeReportAfterSuite) { + if node.ReportSuiteBody == nil { + node.ReportSuiteBody = arg.(func(types.Report)) } else { appendError(types.GinkgoErrors.MultipleBodyFunctions(node.CodeLocation, nodeType)) trackedFunctionError = true break } - } else if nodeType.Is(types.NodeTypeSynchronizedAfterSuite) { - if node.SynchronizedAfterSuiteAllProcsBody == nil { - node.SynchronizedAfterSuiteAllProcsBody = arg.(func()) - } else if node.SynchronizedAfterSuiteProc1Body == nil { - node.SynchronizedAfterSuiteProc1Body = arg.(func()) - } else { + } else if nodeType.Is(types.NodeTypeSynchronizedBeforeSuite) { + if node.SynchronizedBeforeSuiteProc1Body != nil && node.SynchronizedBeforeSuiteAllProcsBody != nil { appendError(types.GinkgoErrors.MultipleBodyFunctions(node.CodeLocation, nodeType)) trackedFunctionError = true break } - } else if nodeType.Is(types.NodeTypeReportAfterSuite) { - if node.ReportAfterSuiteBody == nil { - node.ReportAfterSuiteBody = arg.(func(types.Report)) - } else { + if node.SynchronizedBeforeSuiteProc1Body == nil { + body, hasContext := extractSynchronizedBeforeSuiteProc1Body(arg) + if body == nil { + appendError(types.GinkgoErrors.InvalidBodyTypeForSynchronizedBeforeSuiteProc1(t, node.CodeLocation)) + trackedFunctionError = true + } + node.SynchronizedBeforeSuiteProc1Body, node.SynchronizedBeforeSuiteProc1BodyHasContext = body, hasContext + } else if node.SynchronizedBeforeSuiteAllProcsBody == nil { + body, hasContext := extractSynchronizedBeforeSuiteAllProcsBody(arg) + if body == nil { + appendError(types.GinkgoErrors.InvalidBodyTypeForSynchronizedBeforeSuiteAllProcs(t, node.CodeLocation)) + trackedFunctionError = true + } + node.SynchronizedBeforeSuiteAllProcsBody, node.SynchronizedBeforeSuiteAllProcsBodyHasContext = body, hasContext + } + } else if nodeType.Is(types.NodeTypeSynchronizedAfterSuite) { + if node.SynchronizedAfterSuiteAllProcsBody != nil && node.SynchronizedAfterSuiteProc1Body != nil { appendError(types.GinkgoErrors.MultipleBodyFunctions(node.CodeLocation, nodeType)) trackedFunctionError = true break } + body, hasContext := extractBodyFunction(deprecationTracker, node.CodeLocation, arg) + if body == nil { + appendError(types.GinkgoErrors.InvalidBodyType(t, node.CodeLocation, nodeType)) + trackedFunctionError = true + break + } + if node.SynchronizedAfterSuiteAllProcsBody == nil { + node.SynchronizedAfterSuiteAllProcsBody, node.SynchronizedAfterSuiteAllProcsBodyHasContext = body, hasContext + } else if node.SynchronizedAfterSuiteProc1Body == nil { + node.SynchronizedAfterSuiteProc1Body, node.SynchronizedAfterSuiteProc1BodyHasContext = body, hasContext + } } else { if node.Body != nil { appendError(types.GinkgoErrors.MultipleBodyFunctions(node.CodeLocation, nodeType)) trackedFunctionError = true break } - isValid := (t.NumOut() == 0) && (t.NumIn() <= 1) && (t.NumIn() == 0 || t.In(0) == reflect.TypeOf(make(Done))) - if !isValid { + node.Body, node.HasContext = extractBodyFunction(deprecationTracker, node.CodeLocation, arg) + if node.Body == nil { appendError(types.GinkgoErrors.InvalidBodyType(t, node.CodeLocation, nodeType)) trackedFunctionError = true break } - if t.NumIn() == 0 { - node.Body = arg.(func()) - } else { - deprecationTracker.TrackDeprecation(types.Deprecations.Async(), node.CodeLocation) - deprecatedAsyncBody := arg.(func(Done)) - node.Body = func() { deprecatedAsyncBody(make(Done)) } - } } default: remainingArgs = append(remainingArgs, arg) @@ -320,13 +400,36 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy appendError(types.GinkgoErrors.InvalidDeclarationOfFocusedAndPending(node.CodeLocation, nodeType)) } - if !node.NodeType.Is(types.NodeTypeReportBeforeEach|types.NodeTypeReportAfterEach|types.NodeTypeSynchronizedBeforeSuite|types.NodeTypeSynchronizedAfterSuite|types.NodeTypeReportAfterSuite) && node.Body == nil && !node.MarkedPending && !trackedFunctionError { + if node.MarkedContinueOnFailure && !node.MarkedOrdered { + appendError(types.GinkgoErrors.InvalidContinueOnFailureDecoration(node.CodeLocation)) + } + + hasContext := node.HasContext || node.SynchronizedAfterSuiteProc1BodyHasContext || node.SynchronizedAfterSuiteAllProcsBodyHasContext || node.SynchronizedBeforeSuiteProc1BodyHasContext || node.SynchronizedBeforeSuiteAllProcsBodyHasContext + + if !hasContext && (node.NodeTimeout > 0 || node.SpecTimeout > 0 || node.GracePeriod > 0) && len(errors) == 0 { + appendError(types.GinkgoErrors.InvalidTimeoutOrGracePeriodForNonContextNode(node.CodeLocation, nodeType)) + } + + if !node.NodeType.Is(types.NodeTypeReportBeforeEach|types.NodeTypeReportAfterEach|types.NodeTypeSynchronizedBeforeSuite|types.NodeTypeSynchronizedAfterSuite|types.NodeTypeReportBeforeSuite|types.NodeTypeReportAfterSuite) && node.Body == nil && !node.MarkedPending && !trackedFunctionError { appendError(types.GinkgoErrors.MissingBodyFunction(node.CodeLocation, nodeType)) } + + if node.NodeType.Is(types.NodeTypeSynchronizedBeforeSuite) && !trackedFunctionError && (node.SynchronizedBeforeSuiteProc1Body == nil || node.SynchronizedBeforeSuiteAllProcsBody == nil) { + appendError(types.GinkgoErrors.MissingBodyFunction(node.CodeLocation, nodeType)) + } + + if node.NodeType.Is(types.NodeTypeSynchronizedAfterSuite) && !trackedFunctionError && (node.SynchronizedAfterSuiteProc1Body == nil || node.SynchronizedAfterSuiteAllProcsBody == nil) { + appendError(types.GinkgoErrors.MissingBodyFunction(node.CodeLocation, nodeType)) + } + for _, arg := range remainingArgs { appendError(types.GinkgoErrors.UnknownDecorator(node.CodeLocation, nodeType, arg)) } + if node.FlakeAttempts > 0 && node.MustPassRepeatedly > 0 { + appendError(types.GinkgoErrors.InvalidDeclarationOfFlakeAttemptsAndMustPassRepeatedly(node.CodeLocation, nodeType)) + } + if len(errors) > 0 { return Node{}, errors } @@ -334,52 +437,157 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy return node, errors } -func NewCleanupNode(fail func(string, types.CodeLocation), args ...interface{}) (Node, []error) { - baseOffset := 2 - node := Node{ - ID: UniqueNodeID(), - NodeType: types.NodeTypeCleanupInvalid, - CodeLocation: types.NewCodeLocation(baseOffset), - NestingLevel: -1, - PollProgressAfter: -1, - PollProgressInterval: -1, +var doneType = reflect.TypeOf(make(Done)) + +func extractBodyFunction(deprecationTracker *types.DeprecationTracker, cl types.CodeLocation, arg interface{}) (func(SpecContext), bool) { + t := reflect.TypeOf(arg) + if t.NumOut() > 0 || t.NumIn() > 1 { + return nil, false } - remainingArgs := []interface{}{} - for _, arg := range args { + if t.NumIn() == 1 { + if t.In(0) == doneType { + deprecationTracker.TrackDeprecation(types.Deprecations.Async(), cl) + deprecatedAsyncBody := arg.(func(Done)) + return func(SpecContext) { deprecatedAsyncBody(make(Done)) }, false + } else if t.In(0).Implements(specContextType) { + return arg.(func(SpecContext)), true + } else if t.In(0).Implements(contextType) { + body := arg.(func(context.Context)) + return func(c SpecContext) { body(c) }, true + } + + return nil, false + } + + body := arg.(func()) + return func(SpecContext) { body() }, false +} + +var byteType = reflect.TypeOf([]byte{}) + +func extractSynchronizedBeforeSuiteProc1Body(arg interface{}) (func(SpecContext) []byte, bool) { + t := reflect.TypeOf(arg) + v := reflect.ValueOf(arg) + + if t.NumOut() > 1 || t.NumIn() > 1 { + return nil, false + } else if t.NumOut() == 1 && t.Out(0) != byteType { + return nil, false + } else if t.NumIn() == 1 && !t.In(0).Implements(contextType) { + return nil, false + } + hasContext := t.NumIn() == 1 + + return func(c SpecContext) []byte { + var out []reflect.Value + if hasContext { + out = v.Call([]reflect.Value{reflect.ValueOf(c)}) + } else { + out = v.Call([]reflect.Value{}) + } + if len(out) == 1 { + return (out[0].Interface()).([]byte) + } else { + return []byte{} + } + }, hasContext +} + +func extractSynchronizedBeforeSuiteAllProcsBody(arg interface{}) (func(SpecContext, []byte), bool) { + t := reflect.TypeOf(arg) + v := reflect.ValueOf(arg) + hasContext, hasByte := false, false + + if t.NumOut() > 0 || t.NumIn() > 2 { + return nil, false + } else if t.NumIn() == 2 && t.In(0).Implements(contextType) && t.In(1) == byteType { + hasContext, hasByte = true, true + } else if t.NumIn() == 1 && t.In(0).Implements(contextType) { + hasContext = true + } else if t.NumIn() == 1 && t.In(0) == byteType { + hasByte = true + } else if t.NumIn() != 0 { + return nil, false + } + + return func(c SpecContext, b []byte) { + in := []reflect.Value{} + if hasContext { + in = append(in, reflect.ValueOf(c)) + } + if hasByte { + in = append(in, reflect.ValueOf(b)) + } + v.Call(in) + }, hasContext +} + +var errInterface = reflect.TypeOf((*error)(nil)).Elem() + +func NewCleanupNode(deprecationTracker *types.DeprecationTracker, fail func(string, types.CodeLocation), args ...interface{}) (Node, []error) { + decorations, remainingArgs := PartitionDecorations(args...) + baseOffset := 2 + cl := types.NewCodeLocation(baseOffset) + finalArgs := []interface{}{} + for _, arg := range decorations { switch t := reflect.TypeOf(arg); { case t == reflect.TypeOf(Offset(0)): - node.CodeLocation = types.NewCodeLocation(baseOffset + int(arg.(Offset))) + cl = types.NewCodeLocation(baseOffset + int(arg.(Offset))) case t == reflect.TypeOf(types.CodeLocation{}): - node.CodeLocation = arg.(types.CodeLocation) - case t == reflect.TypeOf(PollProgressAfter(0)): - node.PollProgressAfter = time.Duration(arg.(PollProgressAfter)) - case t == reflect.TypeOf(PollProgressInterval(0)): - node.PollProgressInterval = time.Duration(arg.(PollProgressInterval)) + cl = arg.(types.CodeLocation) default: - remainingArgs = append(remainingArgs, arg) + finalArgs = append(finalArgs, arg) } } + finalArgs = append(finalArgs, cl) if len(remainingArgs) == 0 { - return Node{}, []error{types.GinkgoErrors.DeferCleanupInvalidFunction(node.CodeLocation)} + return Node{}, []error{types.GinkgoErrors.DeferCleanupInvalidFunction(cl)} } + callback := reflect.ValueOf(remainingArgs[0]) - if !(callback.Kind() == reflect.Func && callback.Type().NumOut() <= 1) { - return Node{}, []error{types.GinkgoErrors.DeferCleanupInvalidFunction(node.CodeLocation)} + if !(callback.Kind() == reflect.Func) { + return Node{}, []error{types.GinkgoErrors.DeferCleanupInvalidFunction(cl)} } + callArgs := []reflect.Value{} for _, arg := range remainingArgs[1:] { callArgs = append(callArgs, reflect.ValueOf(arg)) } - cl := node.CodeLocation - node.Body = func() { - out := callback.Call(callArgs) - if len(out) == 1 && !out[0].IsNil() { - fail(fmt.Sprintf("DeferCleanup callback returned error: %v", out[0]), cl) + + hasContext := false + t := callback.Type() + if t.NumIn() > 0 { + if t.In(0).Implements(specContextType) { + hasContext = true + } else if t.In(0).Implements(contextType) && (len(callArgs) == 0 || !callArgs[0].Type().Implements(contextType)) { + hasContext = true + } + } + + handleFailure := func(out []reflect.Value) { + if len(out) == 0 { + return } + last := out[len(out)-1] + if last.Type().Implements(errInterface) && !last.IsNil() { + fail(fmt.Sprintf("DeferCleanup callback returned error: %v", last), cl) + } + } + + if hasContext { + finalArgs = append(finalArgs, func(c SpecContext) { + out := callback.Call(append([]reflect.Value{reflect.ValueOf(c)}, callArgs...)) + handleFailure(out) + }) + } else { + finalArgs = append(finalArgs, func() { + out := callback.Call(callArgs) + handleFailure(out) + }) } - return node, nil + return NewNode(deprecationTracker, types.NodeTypeCleanupInvalid, "", finalArgs...) } func (n Node) IsZero() bool { @@ -667,6 +875,26 @@ func (n Nodes) FirstNodeMarkedOrdered() Node { return Node{} } +func (n Nodes) GetMaxFlakeAttempts() int { + maxFlakeAttempts := 0 + for i := range n { + if n[i].FlakeAttempts > 0 { + maxFlakeAttempts = n[i].FlakeAttempts + } + } + return maxFlakeAttempts +} + +func (n Nodes) GetMaxMustPassRepeatedly() int { + maxMustPassRepeatedly := 0 + for i := range n { + if n[i].MustPassRepeatedly > 0 { + maxMustPassRepeatedly = n[i].MustPassRepeatedly + } + } + return maxMustPassRepeatedly +} + func unrollInterfaceSlice(args interface{}) []interface{} { v := reflect.ValueOf(args) if v.Kind() != reflect.Slice { diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go b/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go index 161be820c..7ed43c7fd 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go @@ -7,6 +7,58 @@ import ( "github.com/onsi/ginkgo/v2/types" ) +type SortableSpecs struct { + Specs Specs + Indexes []int +} + +func NewSortableSpecs(specs Specs) *SortableSpecs { + indexes := make([]int, len(specs)) + for i := range specs { + indexes[i] = i + } + return &SortableSpecs{ + Specs: specs, + Indexes: indexes, + } +} +func (s *SortableSpecs) Len() int { return len(s.Indexes) } +func (s *SortableSpecs) Swap(i, j int) { s.Indexes[i], s.Indexes[j] = s.Indexes[j], s.Indexes[i] } +func (s *SortableSpecs) Less(i, j int) bool { + a, b := s.Specs[s.Indexes[i]], s.Specs[s.Indexes[j]] + + firstOrderedA := a.Nodes.FirstNodeMarkedOrdered() + firstOrderedB := b.Nodes.FirstNodeMarkedOrdered() + if firstOrderedA.ID == firstOrderedB.ID && !firstOrderedA.IsZero() { + // strictly preserve order in ordered containers. ID will track this as IDs are generated monotonically + return a.FirstNodeWithType(types.NodeTypeIt).ID < b.FirstNodeWithType(types.NodeTypeIt).ID + } + + aCLs := a.Nodes.WithType(types.NodeTypesForContainerAndIt).CodeLocations() + bCLs := b.Nodes.WithType(types.NodeTypesForContainerAndIt).CodeLocations() + for i := 0; i < len(aCLs) && i < len(bCLs); i++ { + aCL, bCL := aCLs[i], bCLs[i] + if aCL.FileName < bCL.FileName { + return true + } else if aCL.FileName > bCL.FileName { + return false + } + if aCL.LineNumber < bCL.LineNumber { + return true + } else if aCL.LineNumber > bCL.LineNumber { + return false + } + } + // either everything is equal or we have different lengths of CLs + if len(aCLs) < len(bCLs) { + return true + } else if len(aCLs) > len(bCLs) { + return false + } + // ok, now we are sure everything was equal. so we use the spec text to break ties + return a.Text() < b.Text() +} + type GroupedSpecIndices []SpecIndices type SpecIndices []int @@ -28,12 +80,17 @@ func OrderSpecs(specs Specs, suiteConfig types.SuiteConfig) (GroupedSpecIndices, // Seed a new random source based on thee configured random seed. r := rand.New(rand.NewSource(suiteConfig.RandomSeed)) - // first break things into execution groups + // first, we sort the entire suite to ensure a deterministic order. the sort is performed by filename, then line number, and then spec text. this ensures every parallel process has the exact same spec order and is only necessary to cover the edge case where the user iterates over a map to generate specs. + sortableSpecs := NewSortableSpecs(specs) + sort.Sort(sortableSpecs) + + // then we break things into execution groups // a group represents a single unit of execution and is a collection of SpecIndices // usually a group is just a single spec, however ordered containers must be preserved as a single group executionGroupIDs := []uint{} executionGroups := map[uint]SpecIndices{} - for idx, spec := range specs { + for _, idx := range sortableSpecs.Indexes { + spec := specs[idx] groupNode := spec.Nodes.FirstNodeMarkedOrdered() if groupNode.IsZero() { groupNode = spec.Nodes.FirstNodeWithType(types.NodeTypeIt) @@ -48,7 +105,6 @@ func OrderSpecs(specs Specs, suiteConfig types.SuiteConfig) (GroupedSpecIndices, // we shuffle outermost containers. so we need to form shufflable groupings of GroupIDs shufflableGroupingIDs := []uint{} shufflableGroupingIDToGroupIDs := map[uint][]uint{} - shufflableGroupingsIDToSortKeys := map[uint]string{} // for each execution group we're going to have to pick a node to represent how the // execution group is grouped for shuffling: @@ -57,7 +113,7 @@ func OrderSpecs(specs Specs, suiteConfig types.SuiteConfig) (GroupedSpecIndices, nodeTypesToShuffle = types.NodeTypeIt } - //so, fo reach execution group: + //so, for each execution group: for _, groupID := range executionGroupIDs { // pick out a representative spec representativeSpec := specs[executionGroups[groupID][0]] @@ -72,22 +128,9 @@ func OrderSpecs(specs Specs, suiteConfig types.SuiteConfig) (GroupedSpecIndices, if len(shufflableGroupingIDToGroupIDs[shufflableGroupingNode.ID]) == 1 { // record the shuffleable group ID shufflableGroupingIDs = append(shufflableGroupingIDs, shufflableGroupingNode.ID) - // and record the sort key to use - shufflableGroupingsIDToSortKeys[shufflableGroupingNode.ID] = shufflableGroupingNode.CodeLocation.String() } } - // now we sort the shufflable groups by the sort key. We use the shufflable group nodes code location and break ties using its node id - sort.SliceStable(shufflableGroupingIDs, func(i, j int) bool { - keyA := shufflableGroupingsIDToSortKeys[shufflableGroupingIDs[i]] - keyB := shufflableGroupingsIDToSortKeys[shufflableGroupingIDs[j]] - if keyA == keyB { - return shufflableGroupingIDs[i] < shufflableGroupingIDs[j] - } else { - return keyA < keyB - } - }) - // now we permute the sorted shufflable grouping IDs and build the ordered Groups orderedGroups := GroupedSpecIndices{} permutation := r.Perm(len(shufflableGroupingIDs)) diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/client_server.go b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/client_server.go index b417bf5b3..b3cd64292 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/client_server.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/client_server.go @@ -42,6 +42,8 @@ type Client interface { PostSuiteWillBegin(report types.Report) error PostDidRun(report types.SpecReport) error PostSuiteDidEnd(report types.Report) error + PostReportBeforeSuiteCompleted(state types.SpecState) error + BlockUntilReportBeforeSuiteCompleted() (types.SpecState, error) PostSynchronizedBeforeSuiteCompleted(state types.SpecState, data []byte) error BlockUntilSynchronizedBeforeSuiteData() (types.SpecState, []byte, error) BlockUntilNonprimaryProcsHaveFinished() error diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_client.go b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_client.go index ad9932f2a..6547c7a66 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_client.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_client.go @@ -98,6 +98,19 @@ func (client *httpClient) PostEmitProgressReport(report types.ProgressReport) er return client.post("/progress-report", report) } +func (client *httpClient) PostReportBeforeSuiteCompleted(state types.SpecState) error { + return client.post("/report-before-suite-completed", state) +} + +func (client *httpClient) BlockUntilReportBeforeSuiteCompleted() (types.SpecState, error) { + var state types.SpecState + err := client.poll("/report-before-suite-state", &state) + if err == ErrorGone { + return types.SpecStateFailed, nil + } + return state, err +} + func (client *httpClient) PostSynchronizedBeforeSuiteCompleted(state types.SpecState, data []byte) error { beforeSuiteState := BeforeSuiteState{ State: state, diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_server.go b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_server.go index fa3ac682a..d2c71ab1b 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_server.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/http_server.go @@ -26,7 +26,7 @@ type httpServer struct { handler *ServerHandler } -//Create a new server, automatically selecting a port +// Create a new server, automatically selecting a port func newHttpServer(parallelTotal int, reporter reporters.Reporter) (*httpServer, error) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -38,7 +38,7 @@ func newHttpServer(parallelTotal int, reporter reporters.Reporter) (*httpServer, }, nil } -//Start the server. You don't need to `go s.Start()`, just `s.Start()` +// Start the server. You don't need to `go s.Start()`, just `s.Start()` func (server *httpServer) Start() { httpServer := &http.Server{} mux := http.NewServeMux() @@ -52,6 +52,8 @@ func (server *httpServer) Start() { mux.HandleFunc("/progress-report", server.emitProgressReport) //synchronization endpoints + mux.HandleFunc("/report-before-suite-completed", server.handleReportBeforeSuiteCompleted) + mux.HandleFunc("/report-before-suite-state", server.handleReportBeforeSuiteState) mux.HandleFunc("/before-suite-completed", server.handleBeforeSuiteCompleted) mux.HandleFunc("/before-suite-state", server.handleBeforeSuiteState) mux.HandleFunc("/have-nonprimary-procs-finished", server.handleHaveNonprimaryProcsFinished) @@ -63,12 +65,12 @@ func (server *httpServer) Start() { go httpServer.Serve(server.listener) } -//Stop the server +// Stop the server func (server *httpServer) Close() { server.listener.Close() } -//The address the server can be reached it. Pass this into the `ForwardingReporter`. +// The address the server can be reached it. Pass this into the `ForwardingReporter`. func (server *httpServer) Address() string { return "http://" + server.listener.Addr().String() } @@ -93,7 +95,7 @@ func (server *httpServer) RegisterAlive(node int, alive func() bool) { // Streaming Endpoints // -//The server will forward all received messages to Ginkgo reporters registered with `RegisterReporters` +// The server will forward all received messages to Ginkgo reporters registered with `RegisterReporters` func (server *httpServer) decode(writer http.ResponseWriter, request *http.Request, object interface{}) bool { defer request.Body.Close() if json.NewDecoder(request.Body).Decode(object) != nil { @@ -164,6 +166,23 @@ func (server *httpServer) emitProgressReport(writer http.ResponseWriter, request server.handleError(server.handler.EmitProgressReport(report, voidReceiver), writer) } +func (server *httpServer) handleReportBeforeSuiteCompleted(writer http.ResponseWriter, request *http.Request) { + var state types.SpecState + if !server.decode(writer, request, &state) { + return + } + + server.handleError(server.handler.ReportBeforeSuiteCompleted(state, voidReceiver), writer) +} + +func (server *httpServer) handleReportBeforeSuiteState(writer http.ResponseWriter, request *http.Request) { + var state types.SpecState + if server.handleError(server.handler.ReportBeforeSuiteState(voidSender, &state), writer) { + return + } + json.NewEncoder(writer).Encode(state) +} + func (server *httpServer) handleBeforeSuiteCompleted(writer http.ResponseWriter, request *http.Request) { var beforeSuiteState BeforeSuiteState if !server.decode(writer, request, &beforeSuiteState) { diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/rpc_client.go b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/rpc_client.go index fe93cc2b9..59e8e6fd0 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/rpc_client.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/rpc_client.go @@ -76,6 +76,19 @@ func (client *rpcClient) PostEmitProgressReport(report types.ProgressReport) err return client.client.Call("Server.EmitProgressReport", report, voidReceiver) } +func (client *rpcClient) PostReportBeforeSuiteCompleted(state types.SpecState) error { + return client.client.Call("Server.ReportBeforeSuiteCompleted", state, voidReceiver) +} + +func (client *rpcClient) BlockUntilReportBeforeSuiteCompleted() (types.SpecState, error) { + var state types.SpecState + err := client.poll("Server.ReportBeforeSuiteState", &state) + if err == ErrorGone { + return types.SpecStateFailed, nil + } + return state, err +} + func (client *rpcClient) PostSynchronizedBeforeSuiteCompleted(state types.SpecState, data []byte) error { beforeSuiteState := BeforeSuiteState{ State: state, diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/server_handler.go b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/server_handler.go index 7c6e67b96..a6d98793e 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/server_handler.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/parallel_support/server_handler.go @@ -18,16 +18,17 @@ var voidSender Void // It handles all the business logic to avoid duplication between the two servers type ServerHandler struct { - done chan interface{} - outputDestination io.Writer - reporter reporters.Reporter - alives []func() bool - lock *sync.Mutex - beforeSuiteState BeforeSuiteState - parallelTotal int - counter int - counterLock *sync.Mutex - shouldAbort bool + done chan interface{} + outputDestination io.Writer + reporter reporters.Reporter + alives []func() bool + lock *sync.Mutex + beforeSuiteState BeforeSuiteState + reportBeforeSuiteState types.SpecState + parallelTotal int + counter int + counterLock *sync.Mutex + shouldAbort bool numSuiteDidBegins int numSuiteDidEnds int @@ -37,11 +38,12 @@ type ServerHandler struct { func newServerHandler(parallelTotal int, reporter reporters.Reporter) *ServerHandler { return &ServerHandler{ - reporter: reporter, - lock: &sync.Mutex{}, - counterLock: &sync.Mutex{}, - alives: make([]func() bool, parallelTotal), - beforeSuiteState: BeforeSuiteState{Data: nil, State: types.SpecStateInvalid}, + reporter: reporter, + lock: &sync.Mutex{}, + counterLock: &sync.Mutex{}, + alives: make([]func() bool, parallelTotal), + beforeSuiteState: BeforeSuiteState{Data: nil, State: types.SpecStateInvalid}, + parallelTotal: parallelTotal, outputDestination: os.Stdout, done: make(chan interface{}), @@ -140,6 +142,29 @@ func (handler *ServerHandler) haveNonprimaryProcsFinished() bool { return true } +func (handler *ServerHandler) ReportBeforeSuiteCompleted(reportBeforeSuiteState types.SpecState, _ *Void) error { + handler.lock.Lock() + defer handler.lock.Unlock() + handler.reportBeforeSuiteState = reportBeforeSuiteState + + return nil +} + +func (handler *ServerHandler) ReportBeforeSuiteState(_ Void, reportBeforeSuiteState *types.SpecState) error { + proc1IsAlive := handler.procIsAlive(1) + handler.lock.Lock() + defer handler.lock.Unlock() + if handler.reportBeforeSuiteState == types.SpecStateInvalid { + if proc1IsAlive { + return ErrorEarly + } else { + return ErrorGone + } + } + *reportBeforeSuiteState = handler.reportBeforeSuiteState + return nil +} + func (handler *ServerHandler) BeforeSuiteCompleted(beforeSuiteState BeforeSuiteState, _ *Void) error { handler.lock.Lock() defer handler.lock.Unlock() diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/progress_report.go b/vendor/github.com/onsi/ginkgo/v2/internal/progress_report.go index ed0146c69..11269cf1f 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/progress_report.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/progress_report.go @@ -48,13 +48,10 @@ type ProgressStepCursor struct { StartTime time.Time } -func NewProgressReport(isRunningInParallel bool, report types.SpecReport, currentNode Node, currentNodeStartTime time.Time, currentStep ProgressStepCursor, gwOutput string, sourceRoots []string, includeAll bool) (types.ProgressReport, error) { +func NewProgressReport(isRunningInParallel bool, report types.SpecReport, currentNode Node, currentNodeStartTime time.Time, currentStep types.SpecEvent, gwOutput string, timelineLocation types.TimelineLocation, additionalReports []string, sourceRoots []string, includeAll bool) (types.ProgressReport, error) { pr := types.ProgressReport{ - ParallelProcess: report.ParallelProcess, - RunningInParallel: isRunningInParallel, - - Time: time.Now(), - + ParallelProcess: report.ParallelProcess, + RunningInParallel: isRunningInParallel, ContainerHierarchyTexts: report.ContainerHierarchyTexts, LeafNodeText: report.LeafNodeText, LeafNodeLocation: report.LeafNodeLocation, @@ -65,12 +62,14 @@ func NewProgressReport(isRunningInParallel bool, report types.SpecReport, curren CurrentNodeLocation: currentNode.CodeLocation, CurrentNodeStartTime: currentNodeStartTime, - CurrentStepText: currentStep.Text, + CurrentStepText: currentStep.Message, CurrentStepLocation: currentStep.CodeLocation, - CurrentStepStartTime: currentStep.StartTime, + CurrentStepStartTime: currentStep.TimelineLocation.Time, + + AdditionalReports: additionalReports, CapturedGinkgoWriterOutput: gwOutput, - GinkgoWriterOffset: len(gwOutput), + TimelineLocation: timelineLocation, } goroutines, err := extractRunningGoroutines() @@ -184,7 +183,6 @@ func extractRunningGoroutines() ([]types.Goroutine, error) { break } } - r := bufio.NewReader(bytes.NewReader(stack)) out := []types.Goroutine{} idx := -1 @@ -232,12 +230,12 @@ func extractRunningGoroutines() ([]types.Goroutine, error) { return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid function call: %s -- missing file name and line number", functionCall.Function)) } line = strings.TrimLeft(line, " \t") - fields := strings.SplitN(line, ":", 2) - if len(fields) != 2 { - return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid filename nad line number: %s", line)) + delimiterIdx := strings.LastIndex(line, ":") + if delimiterIdx == -1 { + return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid filename and line number: %s", line)) } - functionCall.Filename = fields[0] - line = strings.Split(fields[1], " ")[0] + functionCall.Filename = line[:delimiterIdx] + line = strings.Split(line[delimiterIdx+1:], " ")[0] lineNumber, err := strconv.ParseInt(line, 10, 64) functionCall.Line = int(lineNumber) if err != nil { diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/report_entry.go b/vendor/github.com/onsi/ginkgo/v2/internal/report_entry.go index 74199f395..cc351a39b 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/report_entry.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/report_entry.go @@ -1,7 +1,6 @@ package internal import ( - "reflect" "time" "github.com/onsi/ginkgo/v2/types" @@ -13,20 +12,20 @@ func NewReportEntry(name string, cl types.CodeLocation, args ...interface{}) (Re out := ReportEntry{ Visibility: types.ReportEntryVisibilityAlways, Name: name, - Time: time.Now(), Location: cl, + Time: time.Now(), } var didSetValue = false for _, arg := range args { - switch reflect.TypeOf(arg) { - case reflect.TypeOf(types.ReportEntryVisibilityAlways): - out.Visibility = arg.(types.ReportEntryVisibility) - case reflect.TypeOf(types.CodeLocation{}): - out.Location = arg.(types.CodeLocation) - case reflect.TypeOf(Offset(0)): - out.Location = types.NewCodeLocation(2 + int(arg.(Offset))) - case reflect.TypeOf(out.Time): - out.Time = arg.(time.Time) + switch x := arg.(type) { + case types.ReportEntryVisibility: + out.Visibility = x + case types.CodeLocation: + out.Location = x + case Offset: + out.Location = types.NewCodeLocation(2 + int(x)) + case time.Time: + out.Time = x default: if didSetValue { return ReportEntry{}, types.GinkgoErrors.TooManyReportEntryValues(out.Location, arg) diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/spec.go b/vendor/github.com/onsi/ginkgo/v2/internal/spec.go index 92072edd4..7c4ee5bb7 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/spec.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/spec.go @@ -2,6 +2,7 @@ package internal import ( "strings" + "time" "github.com/onsi/ginkgo/v2/types" ) @@ -40,6 +41,21 @@ func (s Spec) FlakeAttempts() int { return flakeAttempts } +func (s Spec) MustPassRepeatedly() int { + mustPassRepeatedly := 0 + for i := range s.Nodes { + if s.Nodes[i].MustPassRepeatedly > 0 { + mustPassRepeatedly = s.Nodes[i].MustPassRepeatedly + } + } + + return mustPassRepeatedly +} + +func (s Spec) SpecTimeout() time.Duration { + return s.FirstNodeWithType(types.NodeTypeIt).SpecTimeout +} + type Specs []Spec func (s Specs) HasAnySpecsMarkedPending() bool { diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/spec_context.go b/vendor/github.com/onsi/ginkgo/v2/internal/spec_context.go new file mode 100644 index 000000000..8f569dd35 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/v2/internal/spec_context.go @@ -0,0 +1,90 @@ +package internal + +import ( + "context" + "sort" + "sync" + + "github.com/onsi/ginkgo/v2/types" +) + +type SpecContext interface { + context.Context + + SpecReport() types.SpecReport + AttachProgressReporter(func() string) func() +} + +type specContext struct { + context.Context + + cancel context.CancelFunc + lock *sync.Mutex + progressReporters map[int]func() string + prCounter int + + suite *Suite +} + +/* +SpecContext includes a reference to `suite` and embeds itself in itself as a "GINKGO_SPEC_CONTEXT" value. This allows users to create child Contexts without having down-stream consumers (e.g. Gomega) lose access to the SpecContext and its methods. This allows us to build extensions on top of Ginkgo that simply take an all-encompassing context. + +Note that while SpecContext is used to enforce deadlines by Ginkgo it is not configured as a context.WithDeadline. Instead, Ginkgo owns responsibility for cancelling the context when the deadline elapses. + +This is because Ginkgo needs finer control over when the context is canceled. Specifically, Ginkgo needs to generate a ProgressReport before it cancels the context to ensure progress is captured where the spec is currently running. The only way to avoid a race here is to manually control the cancellation. +*/ +func NewSpecContext(suite *Suite) *specContext { + ctx, cancel := context.WithCancel(context.Background()) + sc := &specContext{ + cancel: cancel, + suite: suite, + lock: &sync.Mutex{}, + prCounter: 0, + progressReporters: map[int]func() string{}, + } + ctx = context.WithValue(ctx, "GINKGO_SPEC_CONTEXT", sc) //yes, yes, the go docs say don't use a string for a key... but we'd rather avoid a circular dependency between Gomega and Ginkgo + sc.Context = ctx //thank goodness for garbage collectors that can handle circular dependencies + + return sc +} + +func (sc *specContext) SpecReport() types.SpecReport { + return sc.suite.CurrentSpecReport() +} + +func (sc *specContext) AttachProgressReporter(reporter func() string) func() { + sc.lock.Lock() + defer sc.lock.Unlock() + sc.prCounter += 1 + prCounter := sc.prCounter + sc.progressReporters[prCounter] = reporter + + return func() { + sc.lock.Lock() + defer sc.lock.Unlock() + delete(sc.progressReporters, prCounter) + } +} + +func (sc *specContext) QueryProgressReporters() []string { + sc.lock.Lock() + keys := []int{} + for key := range sc.progressReporters { + keys = append(keys, key) + } + sort.Ints(keys) + reporters := []func() string{} + for _, key := range keys { + reporters = append(reporters, sc.progressReporters[key]) + } + sc.lock.Unlock() + + if len(reporters) == 0 { + return nil + } + out := []string{} + for _, reporter := range reporters { + out = append(out, reporter()) + } + return out +} diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/suite.go b/vendor/github.com/onsi/ginkgo/v2/internal/suite.go index dfafa43f8..5ddf10072 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/suite.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/suite.go @@ -5,7 +5,6 @@ import ( "sync" "time" - "github.com/onsi/ginkgo/v2/formatter" "github.com/onsi/ginkgo/v2/internal/interrupt_handler" "github.com/onsi/ginkgo/v2/internal/parallel_support" "github.com/onsi/ginkgo/v2/reporters" @@ -35,6 +34,7 @@ type Suite struct { outputInterceptor OutputInterceptor interruptHandler interrupt_handler.InterruptHandlerInterface config types.SuiteConfig + deadline time.Time skipAll bool report types.Report @@ -42,7 +42,10 @@ type Suite struct { currentNode Node currentNodeStartTime time.Time - progressStepCursor ProgressStepCursor + currentSpecContext *specContext + + currentByStep types.SpecEvent + timelineOrder int /* We don't need to lock around all operations. Just those that *could* happen concurrently. @@ -98,6 +101,10 @@ func (suite *Suite) Run(description string, suiteLabels Labels, suitePath string suite.interruptHandler = interruptHandler suite.config = suiteConfig + if suite.config.Timeout > 0 { + suite.deadline = time.Now().Add(suite.config.Timeout) + } + cancelProgressHandler := progressSignalRegistrar(suite.handleProgressSignal) success := suite.runSpecs(description, suiteLabels, suitePath, hasProgrammaticFocus, specs) @@ -122,7 +129,7 @@ func (suite *Suite) PushNode(node Node) error { return suite.pushCleanupNode(node) } - if node.NodeType.Is(types.NodeTypeBeforeSuite | types.NodeTypeAfterSuite | types.NodeTypeSynchronizedBeforeSuite | types.NodeTypeSynchronizedAfterSuite | types.NodeTypeReportAfterSuite) { + if node.NodeType.Is(types.NodeTypeBeforeSuite | types.NodeTypeAfterSuite | types.NodeTypeSynchronizedBeforeSuite | types.NodeTypeSynchronizedAfterSuite | types.NodeTypeBeforeSuite | types.NodeTypeReportBeforeSuite | types.NodeTypeReportAfterSuite) { return suite.pushSuiteNode(node) } @@ -144,6 +151,13 @@ func (suite *Suite) PushNode(node Node) error { } } + if node.MarkedContinueOnFailure { + firstOrderedNode := suite.tree.AncestorNodeChain().FirstNodeMarkedOrdered() + if !firstOrderedNode.IsZero() { + return types.GinkgoErrors.InvalidContinueOnFailureDecoration(node.CodeLocation) + } + } + if node.NodeType == types.NodeTypeContainer { // During PhaseBuildTopLevel we only track the top level containers without entering them // We only enter the top level container nodes during PhaseBuildTree @@ -165,7 +179,7 @@ func (suite *Suite) PushNode(node Node) error { err = types.GinkgoErrors.CaughtPanicDuringABuildPhase(e, node.CodeLocation) } }() - node.Body() + node.Body(nil) return err }() suite.tree = parentTree @@ -215,7 +229,7 @@ func (suite *Suite) pushCleanupNode(node Node) error { node.NodeType = types.NodeTypeCleanupAfterSuite case types.NodeTypeBeforeAll, types.NodeTypeAfterAll: node.NodeType = types.NodeTypeCleanupAfterAll - case types.NodeTypeReportBeforeEach, types.NodeTypeReportAfterEach, types.NodeTypeReportAfterSuite: + case types.NodeTypeReportBeforeEach, types.NodeTypeReportAfterEach, types.NodeTypeReportBeforeSuite, types.NodeTypeReportAfterSuite: return types.GinkgoErrors.PushingCleanupInReportingNode(node.CodeLocation, suite.currentNode.NodeType) case types.NodeTypeCleanupInvalid, types.NodeTypeCleanupAfterEach, types.NodeTypeCleanupAfterAll, types.NodeTypeCleanupAfterSuite: return types.GinkgoErrors.PushingCleanupInCleanupNode(node.CodeLocation) @@ -230,19 +244,69 @@ func (suite *Suite) pushCleanupNode(node Node) error { return nil } -/* - Pushing and popping the Step Cursor stack -*/ - -func (suite *Suite) SetProgressStepCursor(cursor ProgressStepCursor) { +func (suite *Suite) generateTimelineLocation() types.TimelineLocation { suite.selectiveLock.Lock() defer suite.selectiveLock.Unlock() - suite.progressStepCursor = cursor + suite.timelineOrder += 1 + return types.TimelineLocation{ + Offset: len(suite.currentSpecReport.CapturedGinkgoWriterOutput) + suite.writer.Len(), + Order: suite.timelineOrder, + Time: time.Now(), + } +} + +func (suite *Suite) handleSpecEvent(event types.SpecEvent) types.SpecEvent { + event.TimelineLocation = suite.generateTimelineLocation() + suite.selectiveLock.Lock() + suite.currentSpecReport.SpecEvents = append(suite.currentSpecReport.SpecEvents, event) + suite.selectiveLock.Unlock() + suite.reporter.EmitSpecEvent(event) + return event +} + +func (suite *Suite) handleSpecEventEnd(eventType types.SpecEventType, startEvent types.SpecEvent) { + event := startEvent + event.SpecEventType = eventType + event.TimelineLocation = suite.generateTimelineLocation() + event.Duration = event.TimelineLocation.Time.Sub(startEvent.TimelineLocation.Time) + suite.selectiveLock.Lock() + suite.currentSpecReport.SpecEvents = append(suite.currentSpecReport.SpecEvents, event) + suite.selectiveLock.Unlock() + suite.reporter.EmitSpecEvent(event) +} + +func (suite *Suite) By(text string, callback ...func()) error { + cl := types.NewCodeLocation(2) + if suite.phase != PhaseRun { + return types.GinkgoErrors.ByNotDuringRunPhase(cl) + } + + event := suite.handleSpecEvent(types.SpecEvent{ + SpecEventType: types.SpecEventByStart, + CodeLocation: cl, + Message: text, + }) + suite.selectiveLock.Lock() + suite.currentByStep = event + suite.selectiveLock.Unlock() + + if len(callback) == 1 { + defer func() { + suite.selectiveLock.Lock() + suite.currentByStep = types.SpecEvent{} + suite.selectiveLock.Unlock() + suite.handleSpecEventEnd(types.SpecEventByEnd, event) + }() + callback[0]() + } else if len(callback) > 1 { + panic("just one callback per By, please") + } + return nil } /* - Spec Running methods - used during PhaseRun +Spec Running methods - used during PhaseRun */ func (suite *Suite) CurrentSpecReport() types.SpecReport { suite.selectiveLock.Lock() @@ -257,23 +321,29 @@ func (suite *Suite) CurrentSpecReport() types.SpecReport { } func (suite *Suite) AddReportEntry(entry ReportEntry) error { - suite.selectiveLock.Lock() - defer suite.selectiveLock.Unlock() if suite.phase != PhaseRun { return types.GinkgoErrors.AddReportEntryNotDuringRunPhase(entry.Location) } + entry.TimelineLocation = suite.generateTimelineLocation() + entry.Time = entry.TimelineLocation.Time + suite.selectiveLock.Lock() suite.currentSpecReport.ReportEntries = append(suite.currentSpecReport.ReportEntries, entry) + suite.selectiveLock.Unlock() + suite.reporter.EmitReportEntry(entry) return nil } func (suite *Suite) generateProgressReport(fullReport bool) types.ProgressReport { + timelineLocation := suite.generateTimelineLocation() suite.selectiveLock.Lock() defer suite.selectiveLock.Unlock() - stepCursor := suite.progressStepCursor - + var additionalReports []string + if suite.currentSpecContext != nil { + additionalReports = suite.currentSpecContext.QueryProgressReporters() + } gwOutput := suite.currentSpecReport.CapturedGinkgoWriterOutput + string(suite.writer.Bytes()) - pr, err := NewProgressReport(suite.isRunningInParallel(), suite.currentSpecReport, suite.currentNode, suite.currentNodeStartTime, stepCursor, gwOutput, suite.config.SourceRoots, fullReport) + pr, err := NewProgressReport(suite.isRunningInParallel(), suite.currentSpecReport, suite.currentNode, suite.currentNodeStartTime, suite.currentByStep, gwOutput, timelineLocation, additionalReports, suite.config.SourceRoots, fullReport) if err != nil { fmt.Printf("{{red}}Failed to generate progress report:{{/}}\n%s\n", err.Error()) @@ -283,7 +353,11 @@ func (suite *Suite) generateProgressReport(fullReport bool) types.ProgressReport func (suite *Suite) handleProgressSignal() { report := suite.generateProgressReport(false) + report.Message = "{{bold}}You've requested a progress report:{{/}}" + suite.emitProgressReport(report) +} +func (suite *Suite) emitProgressReport(report types.ProgressReport) { suite.selectiveLock.Lock() suite.currentSpecReport.ProgressReports = append(suite.currentSpecReport.ProgressReports, report.WithoutCapturedGinkgoWriterOutput()) suite.selectiveLock.Unlock() @@ -341,7 +415,13 @@ func (suite *Suite) runSpecs(description string, suiteLabels Labels, suitePath s } suite.report.SuiteSucceeded = true - suite.runBeforeSuite(numSpecsThatWillBeRun) + + suite.runReportSuiteNodesIfNeedBe(types.NodeTypeReportBeforeSuite) + + ranBeforeSuite := suite.report.SuiteSucceeded + if suite.report.SuiteSucceeded { + suite.runBeforeSuite(numSpecsThatWillBeRun) + } if suite.report.SuiteSucceeded { groupedSpecIndices, serialGroupedSpecIndices := OrderSpecs(specs, suite.config) @@ -380,19 +460,23 @@ func (suite *Suite) runSpecs(description string, suiteLabels Labels, suitePath s } } - suite.runAfterSuiteCleanup(numSpecsThatWillBeRun) + if ranBeforeSuite { + suite.runAfterSuiteCleanup(numSpecsThatWillBeRun) + } interruptStatus := suite.interruptHandler.Status() - if interruptStatus.Interrupted { + if interruptStatus.Interrupted() { suite.report.SpecialSuiteFailureReasons = append(suite.report.SpecialSuiteFailureReasons, interruptStatus.Cause.String()) suite.report.SuiteSucceeded = false } suite.report.EndTime = time.Now() suite.report.RunTime = suite.report.EndTime.Sub(suite.report.StartTime) - - if suite.config.ParallelProcess == 1 { - suite.runReportAfterSuite() + if !suite.deadline.IsZero() && suite.report.EndTime.After(suite.deadline) { + suite.report.SpecialSuiteFailureReasons = append(suite.report.SpecialSuiteFailureReasons, "Suite Timeout Elapsed") + suite.report.SuiteSucceeded = false } + + suite.runReportSuiteNodesIfNeedBe(types.NodeTypeReportAfterSuite) suite.reporter.SuiteDidEnd(suite.report) if suite.isRunningInParallel() { suite.client.PostSuiteDidEnd(suite.report) @@ -402,19 +486,19 @@ func (suite *Suite) runSpecs(description string, suiteLabels Labels, suitePath s } func (suite *Suite) runBeforeSuite(numSpecsThatWillBeRun int) { - interruptStatus := suite.interruptHandler.Status() beforeSuiteNode := suite.suiteNodes.FirstNodeWithType(types.NodeTypeBeforeSuite | types.NodeTypeSynchronizedBeforeSuite) - if !beforeSuiteNode.IsZero() && !interruptStatus.Interrupted && numSpecsThatWillBeRun > 0 { + if !beforeSuiteNode.IsZero() && numSpecsThatWillBeRun > 0 { suite.selectiveLock.Lock() suite.currentSpecReport = types.SpecReport{ - LeafNodeType: beforeSuiteNode.NodeType, - LeafNodeLocation: beforeSuiteNode.CodeLocation, - ParallelProcess: suite.config.ParallelProcess, + LeafNodeType: beforeSuiteNode.NodeType, + LeafNodeLocation: beforeSuiteNode.CodeLocation, + ParallelProcess: suite.config.ParallelProcess, + RunningInParallel: suite.isRunningInParallel(), } suite.selectiveLock.Unlock() suite.reporter.WillRun(suite.currentSpecReport) - suite.runSuiteNode(beforeSuiteNode, interruptStatus.Channel) + suite.runSuiteNode(beforeSuiteNode) if suite.currentSpecReport.State.Is(types.SpecStateSkipped) { suite.report.SpecialSuiteFailureReasons = append(suite.report.SpecialSuiteFailureReasons, "Suite skipped in BeforeSuite") suite.skipAll = true @@ -428,14 +512,15 @@ func (suite *Suite) runAfterSuiteCleanup(numSpecsThatWillBeRun int) { if !afterSuiteNode.IsZero() && numSpecsThatWillBeRun > 0 { suite.selectiveLock.Lock() suite.currentSpecReport = types.SpecReport{ - LeafNodeType: afterSuiteNode.NodeType, - LeafNodeLocation: afterSuiteNode.CodeLocation, - ParallelProcess: suite.config.ParallelProcess, + LeafNodeType: afterSuiteNode.NodeType, + LeafNodeLocation: afterSuiteNode.CodeLocation, + ParallelProcess: suite.config.ParallelProcess, + RunningInParallel: suite.isRunningInParallel(), } suite.selectiveLock.Unlock() suite.reporter.WillRun(suite.currentSpecReport) - suite.runSuiteNode(afterSuiteNode, suite.interruptHandler.Status().Channel) + suite.runSuiteNode(afterSuiteNode) suite.processCurrentSpecReport() } @@ -444,36 +529,20 @@ func (suite *Suite) runAfterSuiteCleanup(numSpecsThatWillBeRun int) { for _, cleanupNode := range afterSuiteCleanup { suite.selectiveLock.Lock() suite.currentSpecReport = types.SpecReport{ - LeafNodeType: cleanupNode.NodeType, - LeafNodeLocation: cleanupNode.CodeLocation, - ParallelProcess: suite.config.ParallelProcess, + LeafNodeType: cleanupNode.NodeType, + LeafNodeLocation: cleanupNode.CodeLocation, + ParallelProcess: suite.config.ParallelProcess, + RunningInParallel: suite.isRunningInParallel(), } suite.selectiveLock.Unlock() suite.reporter.WillRun(suite.currentSpecReport) - suite.runSuiteNode(cleanupNode, suite.interruptHandler.Status().Channel) + suite.runSuiteNode(cleanupNode) suite.processCurrentSpecReport() } } } -func (suite *Suite) runReportAfterSuite() { - for _, node := range suite.suiteNodes.WithType(types.NodeTypeReportAfterSuite) { - suite.selectiveLock.Lock() - suite.currentSpecReport = types.SpecReport{ - LeafNodeType: node.NodeType, - LeafNodeLocation: node.CodeLocation, - LeafNodeText: node.Text, - ParallelProcess: suite.config.ParallelProcess, - } - suite.selectiveLock.Unlock() - - suite.reporter.WillRun(suite.currentSpecReport) - suite.runReportAfterSuiteNode(node, suite.report) - suite.processCurrentSpecReport() - } -} - func (suite *Suite) reportEach(spec Spec, nodeType types.NodeType) { nodes := spec.Nodes.WithType(nodeType) if nodeType == types.NodeTypeReportAfterEach { @@ -490,16 +559,11 @@ func (suite *Suite) reportEach(spec Spec, nodeType types.NodeType) { suite.writer.Truncate() suite.outputInterceptor.StartInterceptingOutput() report := suite.currentSpecReport - nodes[i].Body = func() { + nodes[i].Body = func(SpecContext) { nodes[i].ReportEachBody(report) } - suite.interruptHandler.SetInterruptPlaceholderMessage(formatter.Fiw(0, formatter.COLS, - "{{yellow}}Ginkgo received an interrupt signal but is currently running a %s node. To avoid an invalid report the %s node will not be interrupted however subsequent tests will be skipped.{{/}}\n\n{{bold}}The running %s node is at:\n%s.{{/}}", - nodeType, nodeType, nodeType, - nodes[i].CodeLocation, - )) - state, failure := suite.runNode(nodes[i], nil, spec.Nodes.BestTextFor(nodes[i])) - suite.interruptHandler.ClearInterruptPlaceholderMessage() + state, failure := suite.runNode(nodes[i], time.Time{}, spec.Nodes.BestTextFor(nodes[i])) + // If the spec is not in a failure state (i.e. it's Passed/Skipped/Pending) and the reporter has failed, override the state. // Also, if the reporter is every aborted - always override the state to propagate the abort if (!suite.currentSpecReport.State.Is(types.SpecStateFailureStates) && state.Is(types.SpecStateFailureStates)) || state.Is(types.SpecStateAborted) { @@ -511,7 +575,7 @@ func (suite *Suite) reportEach(spec Spec, nodeType types.NodeType) { } } -func (suite *Suite) runSuiteNode(node Node, interruptChannel chan interface{}) { +func (suite *Suite) runSuiteNode(node Node) { if suite.config.DryRun { suite.currentSpecReport.State = types.SpecStatePassed return @@ -524,13 +588,13 @@ func (suite *Suite) runSuiteNode(node Node, interruptChannel chan interface{}) { var err error switch node.NodeType { case types.NodeTypeBeforeSuite, types.NodeTypeAfterSuite: - suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, interruptChannel, "") + suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, time.Time{}, "") case types.NodeTypeCleanupAfterSuite: if suite.config.ParallelTotal > 1 && suite.config.ParallelProcess == 1 { err = suite.client.BlockUntilNonprimaryProcsHaveFinished() } if err == nil { - suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, interruptChannel, "") + suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, time.Time{}, "") } case types.NodeTypeSynchronizedBeforeSuite: var data []byte @@ -540,8 +604,9 @@ func (suite *Suite) runSuiteNode(node Node, interruptChannel chan interface{}) { suite.outputInterceptor.StopInterceptingAndReturnOutput() suite.outputInterceptor.StartInterceptingOutputAndForwardTo(suite.client) } - node.Body = func() { data = node.SynchronizedBeforeSuiteProc1Body() } - suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, interruptChannel, "") + node.Body = func(c SpecContext) { data = node.SynchronizedBeforeSuiteProc1Body(c) } + node.HasContext = node.SynchronizedBeforeSuiteProc1BodyHasContext + suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, time.Time{}, "") if suite.config.ParallelTotal > 1 { suite.currentSpecReport.CapturedStdOutErr += suite.outputInterceptor.StopInterceptingAndReturnOutput() suite.outputInterceptor.StartInterceptingOutput() @@ -558,19 +623,21 @@ func (suite *Suite) runSuiteNode(node Node, interruptChannel chan interface{}) { switch proc1State { case types.SpecStatePassed: runAllProcs = true - case types.SpecStateFailed, types.SpecStatePanicked: + case types.SpecStateFailed, types.SpecStatePanicked, types.SpecStateTimedout: err = types.GinkgoErrors.SynchronizedBeforeSuiteFailedOnProc1() case types.SpecStateInterrupted, types.SpecStateAborted, types.SpecStateSkipped: suite.currentSpecReport.State = proc1State } } if runAllProcs { - node.Body = func() { node.SynchronizedBeforeSuiteAllProcsBody(data) } - suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, interruptChannel, "") + node.Body = func(c SpecContext) { node.SynchronizedBeforeSuiteAllProcsBody(c, data) } + node.HasContext = node.SynchronizedBeforeSuiteAllProcsBodyHasContext + suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, time.Time{}, "") } case types.NodeTypeSynchronizedAfterSuite: node.Body = node.SynchronizedAfterSuiteAllProcsBody - suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, interruptChannel, "") + node.HasContext = node.SynchronizedAfterSuiteAllProcsBodyHasContext + suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, time.Time{}, "") if suite.config.ParallelProcess == 1 { if suite.config.ParallelTotal > 1 { err = suite.client.BlockUntilNonprimaryProcsHaveFinished() @@ -582,7 +649,8 @@ func (suite *Suite) runSuiteNode(node Node, interruptChannel chan interface{}) { } node.Body = node.SynchronizedAfterSuiteProc1Body - state, failure := suite.runNode(node, interruptChannel, "") + node.HasContext = node.SynchronizedAfterSuiteProc1BodyHasContext + state, failure := suite.runNode(node, time.Time{}, "") if suite.currentSpecReport.State.Is(types.SpecStatePassed) { suite.currentSpecReport.State, suite.currentSpecReport.Failure = state, failure } @@ -592,55 +660,102 @@ func (suite *Suite) runSuiteNode(node Node, interruptChannel chan interface{}) { if err != nil && !suite.currentSpecReport.State.Is(types.SpecStateFailureStates) { suite.currentSpecReport.State, suite.currentSpecReport.Failure = types.SpecStateFailed, suite.failureForLeafNodeWithMessage(node, err.Error()) + suite.reporter.EmitFailure(suite.currentSpecReport.State, suite.currentSpecReport.Failure) } suite.currentSpecReport.EndTime = time.Now() suite.currentSpecReport.RunTime = suite.currentSpecReport.EndTime.Sub(suite.currentSpecReport.StartTime) suite.currentSpecReport.CapturedGinkgoWriterOutput = string(suite.writer.Bytes()) suite.currentSpecReport.CapturedStdOutErr += suite.outputInterceptor.StopInterceptingAndReturnOutput() +} + +func (suite *Suite) runReportSuiteNodesIfNeedBe(nodeType types.NodeType) { + nodes := suite.suiteNodes.WithType(nodeType) + // only run ReportAfterSuite on proc 1 + if nodeType.Is(types.NodeTypeReportAfterSuite) && suite.config.ParallelProcess != 1 { + return + } + // if we're running ReportBeforeSuite on proc > 1 - we should wait until proc 1 has completed + if nodeType.Is(types.NodeTypeReportBeforeSuite) && suite.config.ParallelProcess != 1 && len(nodes) > 0 { + state, err := suite.client.BlockUntilReportBeforeSuiteCompleted() + if err != nil || state.Is(types.SpecStateFailed) { + suite.report.SuiteSucceeded = false + } + return + } + + for _, node := range nodes { + suite.selectiveLock.Lock() + suite.currentSpecReport = types.SpecReport{ + LeafNodeType: node.NodeType, + LeafNodeLocation: node.CodeLocation, + LeafNodeText: node.Text, + ParallelProcess: suite.config.ParallelProcess, + RunningInParallel: suite.isRunningInParallel(), + } + suite.selectiveLock.Unlock() + + suite.reporter.WillRun(suite.currentSpecReport) + suite.runReportSuiteNode(node, suite.report) + suite.processCurrentSpecReport() + } - return + // if we're running ReportBeforeSuite and we're running in parallel - we shuld tell the other procs that we're done + if nodeType.Is(types.NodeTypeReportBeforeSuite) && suite.isRunningInParallel() && len(nodes) > 0 { + if suite.report.SuiteSucceeded { + suite.client.PostReportBeforeSuiteCompleted(types.SpecStatePassed) + } else { + suite.client.PostReportBeforeSuiteCompleted(types.SpecStateFailed) + } + } } -func (suite *Suite) runReportAfterSuiteNode(node Node, report types.Report) { +func (suite *Suite) runReportSuiteNode(node Node, report types.Report) { suite.writer.Truncate() suite.outputInterceptor.StartInterceptingOutput() suite.currentSpecReport.StartTime = time.Now() - if suite.config.ParallelTotal > 1 { + // if we're running a ReportAfterSuite in parallel (on proc 1) we (a) wait until other procs have exited and + // (b) always fetch the latest report as prior ReportAfterSuites will contribute to it + if node.NodeType.Is(types.NodeTypeReportAfterSuite) && suite.isRunningInParallel() { aggregatedReport, err := suite.client.BlockUntilAggregatedNonprimaryProcsReport() if err != nil { suite.currentSpecReport.State, suite.currentSpecReport.Failure = types.SpecStateFailed, suite.failureForLeafNodeWithMessage(node, err.Error()) + suite.reporter.EmitFailure(suite.currentSpecReport.State, suite.currentSpecReport.Failure) return } report = report.Add(aggregatedReport) } - node.Body = func() { node.ReportAfterSuiteBody(report) } - suite.interruptHandler.SetInterruptPlaceholderMessage(formatter.Fiw(0, formatter.COLS, - "{{yellow}}Ginkgo received an interrupt signal but is currently running a ReportAfterSuite node. To avoid an invalid report the ReportAfterSuite node will not be interrupted.{{/}}\n\n{{bold}}The running ReportAfterSuite node is at:\n%s.{{/}}", - node.CodeLocation, - )) - suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, nil, "") - suite.interruptHandler.ClearInterruptPlaceholderMessage() + node.Body = func(SpecContext) { node.ReportSuiteBody(report) } + suite.currentSpecReport.State, suite.currentSpecReport.Failure = suite.runNode(node, time.Time{}, "") suite.currentSpecReport.EndTime = time.Now() suite.currentSpecReport.RunTime = suite.currentSpecReport.EndTime.Sub(suite.currentSpecReport.StartTime) suite.currentSpecReport.CapturedGinkgoWriterOutput = string(suite.writer.Bytes()) suite.currentSpecReport.CapturedStdOutErr = suite.outputInterceptor.StopInterceptingAndReturnOutput() - - return } -func (suite *Suite) runNode(node Node, interruptChannel chan interface{}, text string) (types.SpecState, types.Failure) { +func (suite *Suite) runNode(node Node, specDeadline time.Time, text string) (types.SpecState, types.Failure) { if node.NodeType.Is(types.NodeTypeCleanupAfterEach | types.NodeTypeCleanupAfterAll | types.NodeTypeCleanupAfterSuite) { suite.cleanupNodes = suite.cleanupNodes.WithoutNode(node) } + interruptStatus := suite.interruptHandler.Status() + if interruptStatus.Level == interrupt_handler.InterruptLevelBailOut { + return types.SpecStateSkipped, types.Failure{} + } + if interruptStatus.Level == interrupt_handler.InterruptLevelReportOnly && !node.NodeType.Is(types.NodeTypesAllowedDuringReportInterrupt) { + return types.SpecStateSkipped, types.Failure{} + } + if interruptStatus.Level == interrupt_handler.InterruptLevelCleanupAndReport && !node.NodeType.Is(types.NodeTypesAllowedDuringReportInterrupt|types.NodeTypesAllowedDuringCleanupInterrupt) { + return types.SpecStateSkipped, types.Failure{} + } + suite.selectiveLock.Lock() suite.currentNode = node suite.currentNodeStartTime = time.Now() - suite.progressStepCursor = ProgressStepCursor{} + suite.currentByStep = types.SpecEvent{} suite.selectiveLock.Unlock() defer func() { suite.selectiveLock.Lock() @@ -649,13 +764,18 @@ func (suite *Suite) runNode(node Node, interruptChannel chan interface{}, text s suite.selectiveLock.Unlock() }() - if suite.config.EmitSpecProgress && !node.MarkedSuppressProgressReporting { - if text == "" { - text = "TOP-LEVEL" - } - s := fmt.Sprintf("[%s] %s\n %s\n", node.NodeType.String(), text, node.CodeLocation.String()) - suite.writer.Write([]byte(s)) + if text == "" { + text = "TOP-LEVEL" } + event := suite.handleSpecEvent(types.SpecEvent{ + SpecEventType: types.SpecEventNodeStart, + NodeType: node.NodeType, + Message: text, + CodeLocation: node.CodeLocation, + }) + defer func() { + suite.handleSpecEventEnd(types.SpecEventNodeEnd, event) + }() var failure types.Failure failure.FailureNodeType, failure.FailureNodeLocation = node.NodeType, node.CodeLocation @@ -666,6 +786,54 @@ func (suite *Suite) runNode(node Node, interruptChannel chan interface{}, text s } else { failure.FailureNodeContext, failure.FailureNodeContainerIndex = types.FailureNodeInContainer, node.NestingLevel-1 } + var outcome types.SpecState + + gracePeriod := suite.config.GracePeriod + if node.GracePeriod >= 0 { + gracePeriod = node.GracePeriod + } + + now := time.Now() + deadline := suite.deadline + timeoutInPlay := "suite" + if deadline.IsZero() || (!specDeadline.IsZero() && specDeadline.Before(deadline)) { + deadline = specDeadline + timeoutInPlay = "spec" + } + if node.NodeTimeout > 0 && (deadline.IsZero() || deadline.Sub(now) > node.NodeTimeout) { + deadline = now.Add(node.NodeTimeout) + timeoutInPlay = "node" + } + if (!deadline.IsZero() && deadline.Before(now)) || interruptStatus.Interrupted() { + //we're out of time already. let's wait for a NodeTimeout if we have it, or GracePeriod if we don't + if node.NodeTimeout > 0 { + deadline = now.Add(node.NodeTimeout) + timeoutInPlay = "node" + } else { + deadline = now.Add(gracePeriod) + timeoutInPlay = "grace period" + } + } + + if !node.HasContext { + // this maps onto the pre-context behavior: + // - an interrupted node exits immediately. with this, context-less nodes that are in a spec with a SpecTimeout and/or are interrupted by other means will simply exit immediately after the timeout/interrupt + // - clean up nodes have up to GracePeriod (formerly hard-coded at 30s) to complete before they are interrupted + gracePeriod = 0 + } + + sc := NewSpecContext(suite) + defer sc.cancel() + + suite.selectiveLock.Lock() + suite.currentSpecContext = sc + suite.selectiveLock.Unlock() + + var deadlineChannel <-chan time.Time + if !deadline.IsZero() { + deadlineChannel = time.After(deadline.Sub(now)) + } + var gracePeriodChannel <-chan time.Time outcomeC := make(chan types.SpecState) failureC := make(chan types.Failure) @@ -677,15 +845,17 @@ func (suite *Suite) runNode(node Node, interruptChannel chan interface{}, text s suite.failer.Panic(types.NewCodeLocationWithStackTrace(2), e) } - outcome, failureFromRun := suite.failer.Drain() - outcomeC <- outcome + outcomeFromRun, failureFromRun := suite.failer.Drain() + failureFromRun.TimelineLocation = suite.generateTimelineLocation() + outcomeC <- outcomeFromRun failureC <- failureFromRun }() - node.Body() + node.Body(sc) finished = true }() + // progress polling timer and channel var emitProgressNow <-chan time.Time var progressPoller *time.Timer var pollProgressAfter, pollProgressInterval = suite.config.PollProgressAfter, suite.config.PollProgressInterval @@ -695,31 +865,114 @@ func (suite *Suite) runNode(node Node, interruptChannel chan interface{}, text s if node.PollProgressInterval >= 0 { pollProgressInterval = node.PollProgressInterval } - if pollProgressAfter > 0 { progressPoller = time.NewTimer(pollProgressAfter) emitProgressNow = progressPoller.C defer progressPoller.Stop() } + // now we wait for an outcome, an interrupt, a timeout, or a progress poll for { select { - case outcome := <-outcomeC: + case outcomeFromRun := <-outcomeC: failureFromRun := <-failureC - if outcome == types.SpecStatePassed { - return outcome, types.Failure{} + if outcome.Is(types.SpecStateInterrupted | types.SpecStateTimedout) { + // we've already been interrupted/timed out. we just managed to actually exit + // before the grace period elapsed + // if we have a failure message we attach it as an additional failure + if outcomeFromRun != types.SpecStatePassed { + additionalFailure := types.AdditionalFailure{ + State: outcomeFromRun, + Failure: failure, //we make a copy - this will include all the configuration set up above... + } + //...and then we update the failure with the details from failureFromRun + additionalFailure.Failure.Location, additionalFailure.Failure.ForwardedPanic, additionalFailure.Failure.TimelineLocation = failureFromRun.Location, failureFromRun.ForwardedPanic, failureFromRun.TimelineLocation + additionalFailure.Failure.ProgressReport = types.ProgressReport{} + if outcome == types.SpecStateTimedout { + additionalFailure.Failure.Message = fmt.Sprintf("A %s timeout occurred and then the following failure was recorded in the timedout node before it exited:\n%s", timeoutInPlay, failureFromRun.Message) + } else { + additionalFailure.Failure.Message = fmt.Sprintf("An interrupt occurred and then the following failure was recorded in the interrupted node before it exited:\n%s", failureFromRun.Message) + } + suite.reporter.EmitFailure(additionalFailure.State, additionalFailure.Failure) + failure.AdditionalFailure = &additionalFailure + } + return outcome, failure + } + if outcomeFromRun.Is(types.SpecStatePassed) { + return outcomeFromRun, types.Failure{} + } else { + failure.Message, failure.Location, failure.ForwardedPanic, failure.TimelineLocation = failureFromRun.Message, failureFromRun.Location, failureFromRun.ForwardedPanic, failureFromRun.TimelineLocation + suite.reporter.EmitFailure(outcomeFromRun, failure) + return outcomeFromRun, failure + } + case <-gracePeriodChannel: + if node.HasContext && outcome.Is(types.SpecStateTimedout) { + report := suite.generateProgressReport(false) + report.Message = "{{bold}}{{orange}}A running node failed to exit in time{{/}}\nGinkgo is moving on but a node has timed out and failed to exit before its grace period elapsed. The node has now leaked and is running in the background.\nHere's a current progress report:" + suite.emitProgressReport(report) } - failure.Message, failure.Location, failure.ForwardedPanic = failureFromRun.Message, failureFromRun.Location, failureFromRun.ForwardedPanic return outcome, failure - case <-interruptChannel: - reason, includeProgressReport := suite.interruptHandler.InterruptMessage() - failure.Message, failure.Location = reason, node.CodeLocation - if includeProgressReport { - failure.ProgressReport = suite.generateProgressReport(true).WithoutCapturedGinkgoWriterOutput() + case <-deadlineChannel: + // we're out of time - the outcome is a timeout and we capture the failure and progress report + outcome = types.SpecStateTimedout + failure.Message, failure.Location, failure.TimelineLocation = fmt.Sprintf("A %s timeout occurred", timeoutInPlay), node.CodeLocation, suite.generateTimelineLocation() + failure.ProgressReport = suite.generateProgressReport(false).WithoutCapturedGinkgoWriterOutput() + failure.ProgressReport.Message = fmt.Sprintf("{{bold}}This is the Progress Report generated when the %s timeout occurred:{{/}}", timeoutInPlay) + deadlineChannel = nil + suite.reporter.EmitFailure(outcome, failure) + + // tell the spec to stop. it's important we generate the progress report first to make sure we capture where + // the spec is actually stuck + sc.cancel() + //and now we wait for the grace period + gracePeriodChannel = time.After(gracePeriod) + case <-interruptStatus.Channel: + interruptStatus = suite.interruptHandler.Status() + deadlineChannel = nil // don't worry about deadlines, time's up now + + failureTimelineLocation := suite.generateTimelineLocation() + progressReport := suite.generateProgressReport(true) + + if outcome == types.SpecStateInvalid { + outcome = types.SpecStateInterrupted + failure.Message, failure.Location, failure.TimelineLocation = interruptStatus.Message(), node.CodeLocation, failureTimelineLocation + if interruptStatus.ShouldIncludeProgressReport() { + failure.ProgressReport = progressReport.WithoutCapturedGinkgoWriterOutput() + failure.ProgressReport.Message = "{{bold}}This is the Progress Report generated when the interrupt was received:{{/}}" + } + suite.reporter.EmitFailure(outcome, failure) + } + + progressReport = progressReport.WithoutOtherGoroutines() + sc.cancel() + + if interruptStatus.Level == interrupt_handler.InterruptLevelBailOut { + if interruptStatus.ShouldIncludeProgressReport() { + progressReport.Message = fmt.Sprintf("{{bold}}{{orange}}%s{{/}}\n{{bold}}{{red}}Final interrupt received{{/}}; Ginkgo will not run any cleanup or reporting nodes and will terminate as soon as possible.\nHere's a current progress report:", interruptStatus.Message()) + suite.emitProgressReport(progressReport) + } + return outcome, failure + } + if interruptStatus.ShouldIncludeProgressReport() { + if interruptStatus.Level == interrupt_handler.InterruptLevelCleanupAndReport { + progressReport.Message = fmt.Sprintf("{{bold}}{{orange}}%s{{/}}\nFirst interrupt received; Ginkgo will run any cleanup and reporting nodes but will skip all remaining specs. {{bold}}Interrupt again to skip cleanup{{/}}.\nHere's a current progress report:", interruptStatus.Message()) + } else if interruptStatus.Level == interrupt_handler.InterruptLevelReportOnly { + progressReport.Message = fmt.Sprintf("{{bold}}{{orange}}%s{{/}}\nSecond interrupt received; Ginkgo will run any reporting nodes but will skip all remaining specs and cleanup nodes. {{bold}}Interrupt again to bail immediately{{/}}.\nHere's a current progress report:", interruptStatus.Message()) + } + suite.emitProgressReport(progressReport) + } + + if gracePeriodChannel == nil { + // we haven't given grace yet... so let's + gracePeriodChannel = time.After(gracePeriod) + } else { + // we've already given grace. time's up. now. + return outcome, failure } - return types.SpecStateInterrupted, failure case <-emitProgressNow: - suite.handleProgressSignal() + report := suite.generateProgressReport(false) + report.Message = "{{bold}}Automatically polling progress:{{/}}" + suite.emitProgressReport(report) if pollProgressInterval > 0 { progressPoller.Reset(pollProgressInterval) } @@ -727,10 +980,12 @@ func (suite *Suite) runNode(node Node, interruptChannel chan interface{}, text s } } +// TODO: search for usages and consider if reporter.EmitFailure() is necessary func (suite *Suite) failureForLeafNodeWithMessage(node Node, message string) types.Failure { return types.Failure{ Message: message, Location: node.CodeLocation, + TimelineLocation: suite.generateTimelineLocation(), FailureNodeContext: types.FailureNodeIsLeafNode, FailureNodeType: node.NodeType, FailureNodeLocation: node.CodeLocation, diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/testingtproxy/testing_t_proxy.go b/vendor/github.com/onsi/ginkgo/v2/internal/testingtproxy/testing_t_proxy.go index 2f42b2642..c797c95d4 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/testingtproxy/testing_t_proxy.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/testingtproxy/testing_t_proxy.go @@ -81,7 +81,7 @@ func (t *ginkgoTestingTProxy) Fatalf(format string, args ...interface{}) { } func (t *ginkgoTestingTProxy) Helper() { - // No-op + types.MarkAsHelper(1) } func (t *ginkgoTestingTProxy) Log(args ...interface{}) { diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/writer.go b/vendor/github.com/onsi/ginkgo/v2/internal/writer.go index 70f0a41f6..28a45b0fa 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/writer.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/writer.go @@ -5,6 +5,9 @@ import ( "fmt" "io" "sync" + + "github.com/go-logr/logr" + "github.com/go-logr/logr/funcr" ) type WriterMode uint @@ -19,24 +22,30 @@ type WriterInterface interface { Truncate() Bytes() []byte + Len() int } -//Writer implements WriterInterface and GinkgoWriterInterface +// Writer implements WriterInterface and GinkgoWriterInterface type Writer struct { buffer *bytes.Buffer outWriter io.Writer lock *sync.Mutex mode WriterMode + streamIndent []byte + indentNext bool + teeWriters []io.Writer } func NewWriter(outWriter io.Writer) *Writer { return &Writer{ - buffer: &bytes.Buffer{}, - lock: &sync.Mutex{}, - outWriter: outWriter, - mode: WriterModeStreamAndBuffer, + buffer: &bytes.Buffer{}, + lock: &sync.Mutex{}, + outWriter: outWriter, + mode: WriterModeStreamAndBuffer, + streamIndent: []byte(" "), + indentNext: true, } } @@ -46,6 +55,14 @@ func (w *Writer) SetMode(mode WriterMode) { w.mode = mode } +func (w *Writer) Len() int { + w.lock.Lock() + defer w.lock.Unlock() + return w.buffer.Len() +} + +var newline = []byte("\n") + func (w *Writer) Write(b []byte) (n int, err error) { w.lock.Lock() defer w.lock.Unlock() @@ -55,7 +72,21 @@ func (w *Writer) Write(b []byte) (n int, err error) { } if w.mode == WriterModeStreamAndBuffer { - w.outWriter.Write(b) + line, remaining, found := []byte{}, b, false + for len(remaining) > 0 { + line, remaining, found = bytes.Cut(remaining, newline) + if len(line) > 0 { + if w.indentNext { + w.outWriter.Write(w.streamIndent) + w.indentNext = false + } + w.outWriter.Write(line) + } + if found { + w.outWriter.Write(newline) + w.indentNext = true + } + } } return w.buffer.Write(b) } @@ -75,7 +106,7 @@ func (w *Writer) Bytes() []byte { return copied } -//GinkgoWriterInterface +// GinkgoWriterInterface func (w *Writer) TeeTo(writer io.Writer) { w.lock.Lock() defer w.lock.Unlock() @@ -101,3 +132,9 @@ func (w *Writer) Printf(format string, a ...interface{}) { func (w *Writer) Println(a ...interface{}) { fmt.Fprintln(w, a...) } + +func GinkgoLogrFunc(writer *Writer) logr.Logger { + return funcr.New(func(prefix, args string) { + writer.Printf("%s", args) + }, funcr.Options{}) +} diff --git a/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go b/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go index ccd3317a5..56b7be758 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go @@ -12,6 +12,7 @@ import ( "io" "runtime" "strings" + "sync" "time" "github.com/onsi/ginkgo/v2/formatter" @@ -23,13 +24,16 @@ type DefaultReporter struct { writer io.Writer // managing the emission stream - lastChar string + lastCharWasNewline bool lastEmissionWasDelimiter bool // rendering specDenoter string retryDenoter string formatter formatter.Formatter + + runningInParallel bool + lock *sync.Mutex } func NewDefaultReporterUnderTest(conf types.ReporterConfig, writer io.Writer) *DefaultReporter { @@ -44,12 +48,13 @@ func NewDefaultReporter(conf types.ReporterConfig, writer io.Writer) *DefaultRep conf: conf, writer: writer, - lastChar: "\n", + lastCharWasNewline: true, lastEmissionWasDelimiter: false, specDenoter: "•", retryDenoter: "↺", formatter: formatter.NewWithNoColorBool(conf.NoColor), + lock: &sync.Mutex{}, } if runtime.GOOS == "windows" { reporter.specDenoter = "+" @@ -97,244 +102,356 @@ func (r *DefaultReporter) SuiteWillBegin(report types.Report) { } } -func (r *DefaultReporter) WillRun(report types.SpecReport) { - if r.conf.Verbosity().LT(types.VerbosityLevelVerbose) || report.State.Is(types.SpecStatePending|types.SpecStateSkipped) { +func (r *DefaultReporter) SuiteDidEnd(report types.Report) { + failures := report.SpecReports.WithState(types.SpecStateFailureStates) + if len(failures) > 0 { + r.emitBlock("\n") + if len(failures) > 1 { + r.emitBlock(r.f("{{red}}{{bold}}Summarizing %d Failures:{{/}}", len(failures))) + } else { + r.emitBlock(r.f("{{red}}{{bold}}Summarizing 1 Failure:{{/}}")) + } + for _, specReport := range failures { + highlightColor, heading := "{{red}}", "[FAIL]" + switch specReport.State { + case types.SpecStatePanicked: + highlightColor, heading = "{{magenta}}", "[PANICKED!]" + case types.SpecStateAborted: + highlightColor, heading = "{{coral}}", "[ABORTED]" + case types.SpecStateTimedout: + highlightColor, heading = "{{orange}}", "[TIMEDOUT]" + case types.SpecStateInterrupted: + highlightColor, heading = "{{orange}}", "[INTERRUPTED]" + } + locationBlock := r.codeLocationBlock(specReport, highlightColor, false, true) + r.emitBlock(r.fi(1, highlightColor+"%s{{/}} %s", heading, locationBlock)) + } + } + + //summarize the suite + if r.conf.Verbosity().Is(types.VerbosityLevelSuccinct) && report.SuiteSucceeded { + r.emit(r.f(" {{green}}SUCCESS!{{/}} %s ", report.RunTime)) return } - r.emitDelimiter() - indentation := uint(0) - if report.LeafNodeType.Is(types.NodeTypesForSuiteLevelNodes) { - r.emitBlock(r.f("{{bold}}[%s] %s{{/}}", report.LeafNodeType.String(), report.LeafNodeText)) + r.emitBlock("\n") + color, status := "{{green}}{{bold}}", "SUCCESS!" + if !report.SuiteSucceeded { + color, status = "{{red}}{{bold}}", "FAIL!" + } + + specs := report.SpecReports.WithLeafNodeType(types.NodeTypeIt) //exclude any suite setup nodes + r.emitBlock(r.f(color+"Ran %d of %d Specs in %.3f seconds{{/}}", + specs.CountWithState(types.SpecStatePassed)+specs.CountWithState(types.SpecStateFailureStates), + report.PreRunStats.TotalSpecs, + report.RunTime.Seconds()), + ) + + switch len(report.SpecialSuiteFailureReasons) { + case 0: + r.emit(r.f(color+"%s{{/}} -- ", status)) + case 1: + r.emit(r.f(color+"%s - %s{{/}} -- ", status, report.SpecialSuiteFailureReasons[0])) + default: + r.emitBlock(r.f(color+"%s - %s{{/}}\n", status, strings.Join(report.SpecialSuiteFailureReasons, ", "))) + } + + if len(specs) == 0 && report.SpecReports.WithLeafNodeType(types.NodeTypeBeforeSuite|types.NodeTypeSynchronizedBeforeSuite).CountWithState(types.SpecStateFailureStates) > 0 { + r.emit(r.f("{{cyan}}{{bold}}A BeforeSuite node failed so all tests were skipped.{{/}}\n")) } else { - if len(report.ContainerHierarchyTexts) > 0 { - r.emitBlock(r.cycleJoin(report.ContainerHierarchyTexts, " ")) - indentation = 1 + r.emit(r.f("{{green}}{{bold}}%d Passed{{/}} | ", specs.CountWithState(types.SpecStatePassed))) + r.emit(r.f("{{red}}{{bold}}%d Failed{{/}} | ", specs.CountWithState(types.SpecStateFailureStates))) + if specs.CountOfFlakedSpecs() > 0 { + r.emit(r.f("{{light-yellow}}{{bold}}%d Flaked{{/}} | ", specs.CountOfFlakedSpecs())) } - line := r.fi(indentation, "{{bold}}%s{{/}}", report.LeafNodeText) - labels := report.Labels() - if len(labels) > 0 { - line += r.f(" {{coral}}[%s]{{/}}", strings.Join(labels, ", ")) + if specs.CountOfRepeatedSpecs() > 0 { + r.emit(r.f("{{light-yellow}}{{bold}}%d Repeated{{/}} | ", specs.CountOfRepeatedSpecs())) } - r.emitBlock(line) + r.emit(r.f("{{yellow}}{{bold}}%d Pending{{/}} | ", specs.CountWithState(types.SpecStatePending))) + r.emit(r.f("{{cyan}}{{bold}}%d Skipped{{/}}\n", specs.CountWithState(types.SpecStateSkipped))) } - r.emitBlock(r.fi(indentation, "{{gray}}%s{{/}}", report.LeafNodeLocation)) } -func (r *DefaultReporter) DidRun(report types.SpecReport) { +func (r *DefaultReporter) WillRun(report types.SpecReport) { v := r.conf.Verbosity() - var header, highlightColor string - includeRuntime, emitGinkgoWriterOutput, stream, denoter := true, true, false, r.specDenoter - succinctLocationBlock := v.Is(types.VerbosityLevelSuccinct) + if v.LT(types.VerbosityLevelVerbose) || report.State.Is(types.SpecStatePending|types.SpecStateSkipped) || report.RunningInParallel { + return + } + + r.emitDelimiter(0) + r.emitBlock(r.f(r.codeLocationBlock(report, "{{/}}", v.Is(types.VerbosityLevelVeryVerbose), false))) +} - hasGW := report.CapturedGinkgoWriterOutput != "" - hasStd := report.CapturedStdOutErr != "" - hasEmittableReports := report.ReportEntries.HasVisibility(types.ReportEntryVisibilityAlways) || (report.ReportEntries.HasVisibility(types.ReportEntryVisibilityFailureOrVerbose) && (!report.Failure.IsZero() || v.GTE(types.VerbosityLevelVerbose))) +func (r *DefaultReporter) DidRun(report types.SpecReport) { + v := r.conf.Verbosity() + inParallel := report.RunningInParallel + header := r.specDenoter if report.LeafNodeType.Is(types.NodeTypesForSuiteLevelNodes) { - denoter = fmt.Sprintf("[%s]", report.LeafNodeType) + header = fmt.Sprintf("[%s]", report.LeafNodeType) + } + highlightColor := r.highlightColorForState(report.State) + + // have we already been streaming the timeline? + timelineHasBeenStreaming := v.GTE(types.VerbosityLevelVerbose) && !inParallel + + // should we show the timeline? + var timeline types.Timeline + showTimeline := !timelineHasBeenStreaming && (v.GTE(types.VerbosityLevelVerbose) || report.Failed()) + if showTimeline { + timeline = report.Timeline().WithoutHiddenReportEntries() + keepVeryVerboseSpecEvents := v.Is(types.VerbosityLevelVeryVerbose) || + (v.Is(types.VerbosityLevelVerbose) && r.conf.ShowNodeEvents) || + (report.Failed() && r.conf.ShowNodeEvents) + if !keepVeryVerboseSpecEvents { + timeline = timeline.WithoutVeryVerboseSpecEvents() + } + if len(timeline) == 0 && report.CapturedGinkgoWriterOutput == "" { + // the timeline is completely empty - don't show it + showTimeline = false + } + if v.LT(types.VerbosityLevelVeryVerbose) && report.CapturedGinkgoWriterOutput == "" && len(timeline) > 0 { + //if we aren't -vv and the timeline only has a single failure, don't show it as it will appear at the end of the report + failure, isFailure := timeline[0].(types.Failure) + if isFailure && (len(timeline) == 1 || (len(timeline) == 2 && failure.AdditionalFailure != nil)) { + showTimeline = false + } + } } + // should we have a separate section for always-visible reports? + showSeparateVisibilityAlwaysReportsSection := !timelineHasBeenStreaming && !showTimeline && report.ReportEntries.HasVisibility(types.ReportEntryVisibilityAlways) + + // should we have a separate section for captured stdout/stderr + showSeparateStdSection := inParallel && (report.CapturedStdOutErr != "") + + // given all that - do we have any actual content to show? or are we a single denoter in a stream? + reportHasContent := v.Is(types.VerbosityLevelVeryVerbose) || showTimeline || showSeparateVisibilityAlwaysReportsSection || showSeparateStdSection || report.Failed() || (v.Is(types.VerbosityLevelVerbose) && !report.State.Is(types.SpecStateSkipped)) + + // should we show a runtime? + includeRuntime := !report.State.Is(types.SpecStateSkipped|types.SpecStatePending) || (report.State.Is(types.SpecStateSkipped) && report.Failure.Message != "") + + // should we show the codelocation block? + showCodeLocation := !timelineHasBeenStreaming || !report.State.Is(types.SpecStatePassed) + switch report.State { case types.SpecStatePassed: - highlightColor, succinctLocationBlock = "{{green}}", v.LT(types.VerbosityLevelVerbose) - emitGinkgoWriterOutput = (r.conf.AlwaysEmitGinkgoWriter || v.GTE(types.VerbosityLevelVerbose)) && hasGW + if report.LeafNodeType.Is(types.NodeTypesForSuiteLevelNodes) && !reportHasContent { + return + } if report.LeafNodeType.Is(types.NodeTypesForSuiteLevelNodes) { - if v.GTE(types.VerbosityLevelVerbose) || hasStd || hasEmittableReports { - header = fmt.Sprintf("%s PASSED", denoter) - } else { - return - } - } else { - header, stream = denoter, true - if report.NumAttempts > 1 { - header, stream = fmt.Sprintf("%s [FLAKEY TEST - TOOK %d ATTEMPTS TO PASS]", r.retryDenoter, report.NumAttempts), false - } - if report.RunTime > r.conf.SlowSpecThreshold { - header, stream = fmt.Sprintf("%s [SLOW TEST]", header), false - } + header = fmt.Sprintf("%s PASSED", header) } - if hasStd || emitGinkgoWriterOutput || hasEmittableReports { - stream = false + if report.NumAttempts > 1 && report.MaxFlakeAttempts > 1 { + header, reportHasContent = fmt.Sprintf("%s [FLAKEY TEST - TOOK %d ATTEMPTS TO PASS]", r.retryDenoter, report.NumAttempts), true } case types.SpecStatePending: - highlightColor = "{{yellow}}" - includeRuntime, emitGinkgoWriterOutput = false, false - if v.Is(types.VerbosityLevelSuccinct) { - header, stream = "P", true - } else { - header, succinctLocationBlock = "P [PENDING]", v.LT(types.VerbosityLevelVeryVerbose) + header = "P" + if v.GT(types.VerbosityLevelSuccinct) { + header, reportHasContent = "P [PENDING]", true } case types.SpecStateSkipped: - highlightColor = "{{cyan}}" - if report.Failure.Message != "" || v.Is(types.VerbosityLevelVeryVerbose) { - header = "S [SKIPPED]" - } else { - header, stream = "S", true + header = "S" + if v.Is(types.VerbosityLevelVeryVerbose) || (v.Is(types.VerbosityLevelVerbose) && report.Failure.Message != "") { + header, reportHasContent = "S [SKIPPED]", true + } + default: + header = fmt.Sprintf("%s [%s]", header, r.humanReadableState(report.State)) + if report.MaxMustPassRepeatedly > 1 { + header = fmt.Sprintf("%s DURING REPETITION #%d", header, report.NumAttempts) } - case types.SpecStateFailed: - highlightColor, header = "{{red}}", fmt.Sprintf("%s [FAILED]", denoter) - case types.SpecStatePanicked: - highlightColor, header = "{{magenta}}", fmt.Sprintf("%s! [PANICKED]", denoter) - case types.SpecStateInterrupted: - highlightColor, header = "{{orange}}", fmt.Sprintf("%s! [INTERRUPTED]", denoter) - case types.SpecStateAborted: - highlightColor, header = "{{coral}}", fmt.Sprintf("%s! [ABORTED]", denoter) } - // Emit stream and return - if stream { + // If we have no content to show, jsut emit the header and return + if !reportHasContent { r.emit(r.f(highlightColor + header + "{{/}}")) return } - // Emit header - r.emitDelimiter() if includeRuntime { header = r.f("%s [%.3f seconds]", header, report.RunTime.Seconds()) } - r.emitBlock(r.f(highlightColor + header + "{{/}}")) - // Emit Code Location Block - r.emitBlock(r.codeLocationBlock(report, highlightColor, succinctLocationBlock, false)) + // Emit header + if !timelineHasBeenStreaming { + r.emitDelimiter(0) + } + r.emitBlock(r.f(highlightColor + header + "{{/}}")) + if showCodeLocation { + r.emitBlock(r.codeLocationBlock(report, highlightColor, v.Is(types.VerbosityLevelVeryVerbose), false)) + } //Emit Stdout/Stderr Output - if hasStd { + if showSeparateStdSection { r.emitBlock("\n") - r.emitBlock(r.fi(1, "{{gray}}Begin Captured StdOut/StdErr Output >>{{/}}")) - r.emitBlock(r.fi(2, "%s", report.CapturedStdOutErr)) - r.emitBlock(r.fi(1, "{{gray}}<< End Captured StdOut/StdErr Output{{/}}")) + r.emitBlock(r.fi(1, "{{gray}}Captured StdOut/StdErr Output >>{{/}}")) + r.emitBlock(r.fi(1, "%s", report.CapturedStdOutErr)) + r.emitBlock(r.fi(1, "{{gray}}<< Captured StdOut/StdErr Output{{/}}")) } - //Emit Captured GinkgoWriter Output - if emitGinkgoWriterOutput && hasGW { + if showSeparateVisibilityAlwaysReportsSection { r.emitBlock("\n") - r.emitGinkgoWriterOutput(1, report.CapturedGinkgoWriterOutput, 0) + r.emitBlock(r.fi(1, "{{gray}}Report Entries >>{{/}}")) + for _, entry := range report.ReportEntries.WithVisibility(types.ReportEntryVisibilityAlways) { + r.emitReportEntry(1, entry) + } + r.emitBlock(r.fi(1, "{{gray}}<< Report Entries{{/}}")) } - if hasEmittableReports { + if showTimeline { r.emitBlock("\n") - r.emitBlock(r.fi(1, "{{gray}}Begin Report Entries >>{{/}}")) - reportEntries := report.ReportEntries.WithVisibility(types.ReportEntryVisibilityAlways) - if !report.Failure.IsZero() || v.GTE(types.VerbosityLevelVerbose) { - reportEntries = report.ReportEntries.WithVisibility(types.ReportEntryVisibilityAlways, types.ReportEntryVisibilityFailureOrVerbose) - } - for _, entry := range reportEntries { - r.emitBlock(r.fi(2, "{{bold}}"+entry.Name+"{{gray}} - %s @ %s{{/}}", entry.Location, entry.Time.Format(types.GINKGO_TIME_FORMAT))) - if representation := entry.StringRepresentation(); representation != "" { - r.emitBlock(r.fi(3, representation)) - } - } - r.emitBlock(r.fi(1, "{{gray}}<< End Report Entries{{/}}")) + r.emitBlock(r.fi(1, "{{gray}}Timeline >>{{/}}")) + r.emitTimeline(1, report, timeline) + r.emitBlock(r.fi(1, "{{gray}}<< Timeline{{/}}")) } // Emit Failure Message - if !report.Failure.IsZero() { + if !report.Failure.IsZero() && !v.Is(types.VerbosityLevelVeryVerbose) { r.emitBlock("\n") - r.emitBlock(r.fi(1, highlightColor+"%s{{/}}", report.Failure.Message)) - r.emitBlock(r.fi(1, highlightColor+"In {{bold}}[%s]{{/}}"+highlightColor+" at: {{bold}}%s{{/}}\n", report.Failure.FailureNodeType, report.Failure.Location)) - if report.Failure.ForwardedPanic != "" { - r.emitBlock("\n") - r.emitBlock(r.fi(1, highlightColor+"%s{{/}}", report.Failure.ForwardedPanic)) + r.emitFailure(1, report.State, report.Failure, true) + if len(report.AdditionalFailures) > 0 { + r.emitBlock(r.fi(1, "\nThere were {{bold}}{{red}}additional failures{{/}} detected. To view them in detail run {{bold}}ginkgo -vv{{/}}")) } + } - if r.conf.FullTrace || report.Failure.ForwardedPanic != "" { - r.emitBlock("\n") - r.emitBlock(r.fi(1, highlightColor+"Full Stack Trace{{/}}")) - r.emitBlock(r.fi(2, "%s", report.Failure.Location.FullStackTrace)) - } + r.emitDelimiter(0) +} - if !report.Failure.ProgressReport.IsZero() { - r.emitBlock("\n") - r.emitProgressReport(1, false, report.Failure.ProgressReport) - } +func (r *DefaultReporter) highlightColorForState(state types.SpecState) string { + switch state { + case types.SpecStatePassed: + return "{{green}}" + case types.SpecStatePending: + return "{{yellow}}" + case types.SpecStateSkipped: + return "{{cyan}}" + case types.SpecStateFailed: + return "{{red}}" + case types.SpecStateTimedout: + return "{{orange}}" + case types.SpecStatePanicked: + return "{{magenta}}" + case types.SpecStateInterrupted: + return "{{orange}}" + case types.SpecStateAborted: + return "{{coral}}" + default: + return "{{gray}}" } +} - r.emitDelimiter() +func (r *DefaultReporter) humanReadableState(state types.SpecState) string { + return strings.ToUpper(state.String()) } -func (r *DefaultReporter) SuiteDidEnd(report types.Report) { - failures := report.SpecReports.WithState(types.SpecStateFailureStates) - if len(failures) > 0 { - r.emitBlock("\n\n") - if len(failures) > 1 { - r.emitBlock(r.f("{{red}}{{bold}}Summarizing %d Failures:{{/}}", len(failures))) - } else { - r.emitBlock(r.f("{{red}}{{bold}}Summarizing 1 Failure:{{/}}")) - } - for _, specReport := range failures { - highlightColor, heading := "{{red}}", "[FAIL]" - switch specReport.State { - case types.SpecStatePanicked: - highlightColor, heading = "{{magenta}}", "[PANICKED!]" - case types.SpecStateAborted: - highlightColor, heading = "{{coral}}", "[ABORTED]" - case types.SpecStateInterrupted: - highlightColor, heading = "{{orange}}", "[INTERRUPTED]" +func (r *DefaultReporter) emitTimeline(indent uint, report types.SpecReport, timeline types.Timeline) { + isVeryVerbose := r.conf.Verbosity().Is(types.VerbosityLevelVeryVerbose) + gw := report.CapturedGinkgoWriterOutput + cursor := 0 + for _, entry := range timeline { + tl := entry.GetTimelineLocation() + if tl.Offset < len(gw) { + r.emit(r.fi(indent, "%s", gw[cursor:tl.Offset])) + cursor = tl.Offset + } else if cursor < len(gw) { + r.emit(r.fi(indent, "%s", gw[cursor:])) + cursor = len(gw) + } + switch x := entry.(type) { + case types.Failure: + if isVeryVerbose { + r.emitFailure(indent, report.State, x, false) + } else { + r.emitShortFailure(indent, report.State, x) + } + case types.AdditionalFailure: + if isVeryVerbose { + r.emitFailure(indent, x.State, x.Failure, true) + } else { + r.emitShortFailure(indent, x.State, x.Failure) + } + case types.ReportEntry: + r.emitReportEntry(indent, x) + case types.ProgressReport: + r.emitProgressReport(indent, false, x) + case types.SpecEvent: + if isVeryVerbose || !x.IsOnlyVisibleAtVeryVerbose() || r.conf.ShowNodeEvents { + r.emitSpecEvent(indent, x, isVeryVerbose) } - locationBlock := r.codeLocationBlock(specReport, highlightColor, true, true) - r.emitBlock(r.fi(1, highlightColor+"%s{{/}} %s", heading, locationBlock)) } } + if cursor < len(gw) { + r.emit(r.fi(indent, "%s", gw[cursor:])) + } +} - //summarize the suite - if r.conf.Verbosity().Is(types.VerbosityLevelSuccinct) && report.SuiteSucceeded { - r.emit(r.f(" {{green}}SUCCESS!{{/}} %s ", report.RunTime)) - return +func (r *DefaultReporter) EmitFailure(state types.SpecState, failure types.Failure) { + if r.conf.Verbosity().Is(types.VerbosityLevelVerbose) { + r.emitShortFailure(1, state, failure) + } else if r.conf.Verbosity().Is(types.VerbosityLevelVeryVerbose) { + r.emitFailure(1, state, failure, true) } +} - r.emitBlock("\n") - color, status := "{{green}}{{bold}}", "SUCCESS!" - if !report.SuiteSucceeded { - color, status = "{{red}}{{bold}}", "FAIL!" +func (r *DefaultReporter) emitShortFailure(indent uint, state types.SpecState, failure types.Failure) { + r.emitBlock(r.fi(indent, r.highlightColorForState(state)+"[%s]{{/}} in [%s] - %s {{gray}}@ %s{{/}}", + r.humanReadableState(state), + failure.FailureNodeType, + failure.Location, + failure.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT), + )) +} + +func (r *DefaultReporter) emitFailure(indent uint, state types.SpecState, failure types.Failure, includeAdditionalFailure bool) { + highlightColor := r.highlightColorForState(state) + r.emitBlock(r.fi(indent, highlightColor+"[%s] %s{{/}}", r.humanReadableState(state), failure.Message)) + r.emitBlock(r.fi(indent, highlightColor+"In {{bold}}[%s]{{/}}"+highlightColor+" at: {{bold}}%s{{/}} {{gray}}@ %s{{/}}\n", failure.FailureNodeType, failure.Location, failure.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT))) + if failure.ForwardedPanic != "" { + r.emitBlock("\n") + r.emitBlock(r.fi(indent, highlightColor+"%s{{/}}", failure.ForwardedPanic)) } - specs := report.SpecReports.WithLeafNodeType(types.NodeTypeIt) //exclude any suite setup nodes - r.emitBlock(r.f(color+"Ran %d of %d Specs in %.3f seconds{{/}}", - specs.CountWithState(types.SpecStatePassed)+specs.CountWithState(types.SpecStateFailureStates), - report.PreRunStats.TotalSpecs, - report.RunTime.Seconds()), - ) + if r.conf.FullTrace || failure.ForwardedPanic != "" { + r.emitBlock("\n") + r.emitBlock(r.fi(indent, highlightColor+"Full Stack Trace{{/}}")) + r.emitBlock(r.fi(indent+1, "%s", failure.Location.FullStackTrace)) + } - switch len(report.SpecialSuiteFailureReasons) { - case 0: - r.emit(r.f(color+"%s{{/}} -- ", status)) - case 1: - r.emit(r.f(color+"%s - %s{{/}} -- ", status, report.SpecialSuiteFailureReasons[0])) - default: - r.emitBlock(r.f(color+"%s - %s{{/}}\n", status, strings.Join(report.SpecialSuiteFailureReasons, ", "))) + if !failure.ProgressReport.IsZero() { + r.emitBlock("\n") + r.emitProgressReport(indent, false, failure.ProgressReport) } - if len(specs) == 0 && report.SpecReports.WithLeafNodeType(types.NodeTypeBeforeSuite|types.NodeTypeSynchronizedBeforeSuite).CountWithState(types.SpecStateFailureStates) > 0 { - r.emit(r.f("{{cyan}}{{bold}}A BeforeSuite node failed so all tests were skipped.{{/}}\n")) - } else { - r.emit(r.f("{{green}}{{bold}}%d Passed{{/}} | ", specs.CountWithState(types.SpecStatePassed))) - r.emit(r.f("{{red}}{{bold}}%d Failed{{/}} | ", specs.CountWithState(types.SpecStateFailureStates))) - if specs.CountOfFlakedSpecs() > 0 { - r.emit(r.f("{{light-yellow}}{{bold}}%d Flaked{{/}} | ", specs.CountOfFlakedSpecs())) - } - r.emit(r.f("{{yellow}}{{bold}}%d Pending{{/}} | ", specs.CountWithState(types.SpecStatePending))) - r.emit(r.f("{{cyan}}{{bold}}%d Skipped{{/}}\n", specs.CountWithState(types.SpecStateSkipped))) + if failure.AdditionalFailure != nil && includeAdditionalFailure { + r.emitBlock("\n") + r.emitFailure(indent, failure.AdditionalFailure.State, failure.AdditionalFailure.Failure, true) } } func (r *DefaultReporter) EmitProgressReport(report types.ProgressReport) { - r.emitDelimiter() + r.emitDelimiter(1) if report.RunningInParallel { - r.emit(r.f("{{coral}}Progress Report for Ginkgo Process #{{bold}}%d{{/}}\n", report.ParallelProcess)) + r.emit(r.fi(1, "{{coral}}Progress Report for Ginkgo Process #{{bold}}%d{{/}}\n", report.ParallelProcess)) } - r.emitProgressReport(0, true, report) - r.emitDelimiter() + shouldEmitGW := report.RunningInParallel || r.conf.Verbosity().LT(types.VerbosityLevelVerbose) + r.emitProgressReport(1, shouldEmitGW, report) + r.emitDelimiter(1) } func (r *DefaultReporter) emitProgressReport(indent uint, emitGinkgoWriterOutput bool, report types.ProgressReport) { + if report.Message != "" { + r.emitBlock(r.fi(indent, report.Message+"\n")) + indent += 1 + } if report.LeafNodeText != "" { + subjectIndent := indent if len(report.ContainerHierarchyTexts) > 0 { r.emit(r.fi(indent, r.cycleJoin(report.ContainerHierarchyTexts, " "))) r.emit(" ") + subjectIndent = 0 } - r.emit(r.f("{{bold}}{{orange}}%s{{/}} (Spec Runtime: %s)\n", report.LeafNodeText, report.Time.Sub(report.SpecStartTime).Round(time.Millisecond))) + r.emit(r.fi(subjectIndent, "{{bold}}{{orange}}%s{{/}} (Spec Runtime: %s)\n", report.LeafNodeText, report.Time().Sub(report.SpecStartTime).Round(time.Millisecond))) r.emit(r.fi(indent+1, "{{gray}}%s{{/}}\n", report.LeafNodeLocation)) indent += 1 } @@ -344,12 +461,12 @@ func (r *DefaultReporter) emitProgressReport(indent uint, emitGinkgoWriterOutput r.emit(r.f(" {{bold}}{{orange}}%s{{/}}", report.CurrentNodeText)) } - r.emit(r.f(" (Node Runtime: %s)\n", report.Time.Sub(report.CurrentNodeStartTime).Round(time.Millisecond))) + r.emit(r.f(" (Node Runtime: %s)\n", report.Time().Sub(report.CurrentNodeStartTime).Round(time.Millisecond))) r.emit(r.fi(indent+1, "{{gray}}%s{{/}}\n", report.CurrentNodeLocation)) indent += 1 } if report.CurrentStepText != "" { - r.emit(r.fi(indent, "At {{bold}}{{orange}}[By Step] %s{{/}} (Step Runtime: %s)\n", report.CurrentStepText, report.Time.Sub(report.CurrentStepStartTime).Round(time.Millisecond))) + r.emit(r.fi(indent, "At {{bold}}{{orange}}[By Step] %s{{/}} (Step Runtime: %s)\n", report.CurrentStepText, report.Time().Sub(report.CurrentStepStartTime).Round(time.Millisecond))) r.emit(r.fi(indent+1, "{{gray}}%s{{/}}\n", report.CurrentStepLocation)) indent += 1 } @@ -358,9 +475,19 @@ func (r *DefaultReporter) emitProgressReport(indent uint, emitGinkgoWriterOutput indent -= 1 } - if emitGinkgoWriterOutput && report.CapturedGinkgoWriterOutput != "" && (report.RunningInParallel || r.conf.Verbosity().LT(types.VerbosityLevelVerbose)) { + if emitGinkgoWriterOutput && report.CapturedGinkgoWriterOutput != "" { r.emit("\n") - r.emitGinkgoWriterOutput(indent, report.CapturedGinkgoWriterOutput, 10) + r.emitBlock(r.fi(indent, "{{gray}}Begin Captured GinkgoWriter Output >>{{/}}")) + limit, lines := 10, strings.Split(report.CapturedGinkgoWriterOutput, "\n") + if len(lines) <= limit { + r.emitBlock(r.fi(indent+1, "%s", report.CapturedGinkgoWriterOutput)) + } else { + r.emitBlock(r.fi(indent+1, "{{gray}}...{{/}}")) + for _, line := range lines[len(lines)-limit-1:] { + r.emitBlock(r.fi(indent+1, "%s", line)) + } + } + r.emitBlock(r.fi(indent, "{{gray}}<< End Captured GinkgoWriter Output{{/}}")) } if !report.SpecGoroutine().IsZero() { @@ -369,6 +496,18 @@ func (r *DefaultReporter) emitProgressReport(indent uint, emitGinkgoWriterOutput r.emitGoroutines(indent, report.SpecGoroutine()) } + if len(report.AdditionalReports) > 0 { + r.emit("\n") + r.emitBlock(r.fi(indent, "{{gray}}Begin Additional Progress Reports >>{{/}}")) + for i, additionalReport := range report.AdditionalReports { + r.emit(r.fi(indent+1, additionalReport)) + if i < len(report.AdditionalReports)-1 { + r.emitBlock(r.fi(indent+1, "{{gray}}%s{{/}}", strings.Repeat("-", 10))) + } + } + r.emitBlock(r.fi(indent, "{{gray}}<< End Additional Progress Reports{{/}}")) + } + highlightedGoroutines := report.HighlightedGoroutines() if len(highlightedGoroutines) > 0 { r.emit("\n") @@ -384,22 +523,48 @@ func (r *DefaultReporter) emitProgressReport(indent uint, emitGinkgoWriterOutput } } -func (r *DefaultReporter) emitGinkgoWriterOutput(indent uint, output string, limit int) { - r.emitBlock(r.fi(indent, "{{gray}}Begin Captured GinkgoWriter Output >>{{/}}")) - if limit == 0 { - r.emitBlock(r.fi(indent+1, "%s", output)) - } else { - lines := strings.Split(output, "\n") - if len(lines) <= limit { - r.emitBlock(r.fi(indent+1, "%s", output)) - } else { - r.emitBlock(r.fi(indent+1, "{{gray}}...{{/}}")) - for _, line := range lines[len(lines)-limit-1:] { - r.emitBlock(r.fi(indent+1, "%s", line)) - } - } +func (r *DefaultReporter) EmitReportEntry(entry types.ReportEntry) { + if r.conf.Verbosity().LT(types.VerbosityLevelVerbose) || entry.Visibility == types.ReportEntryVisibilityNever { + return + } + r.emitReportEntry(1, entry) +} + +func (r *DefaultReporter) emitReportEntry(indent uint, entry types.ReportEntry) { + r.emitBlock(r.fi(indent, "{{bold}}"+entry.Name+"{{gray}} "+fmt.Sprintf("- %s @ %s{{/}}", entry.Location, entry.Time.Format(types.GINKGO_TIME_FORMAT)))) + if representation := entry.StringRepresentation(); representation != "" { + r.emitBlock(r.fi(indent+1, representation)) + } +} + +func (r *DefaultReporter) EmitSpecEvent(event types.SpecEvent) { + v := r.conf.Verbosity() + if v.Is(types.VerbosityLevelVeryVerbose) || (v.Is(types.VerbosityLevelVerbose) && (r.conf.ShowNodeEvents || !event.IsOnlyVisibleAtVeryVerbose())) { + r.emitSpecEvent(1, event, r.conf.Verbosity().Is(types.VerbosityLevelVeryVerbose)) + } +} + +func (r *DefaultReporter) emitSpecEvent(indent uint, event types.SpecEvent, includeLocation bool) { + location := "" + if includeLocation { + location = fmt.Sprintf("- %s ", event.CodeLocation.String()) + } + switch event.SpecEventType { + case types.SpecEventInvalid: + return + case types.SpecEventByStart: + r.emitBlock(r.fi(indent, "{{bold}}STEP:{{/}} %s {{gray}}%s@ %s{{/}}", event.Message, location, event.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT))) + case types.SpecEventByEnd: + r.emitBlock(r.fi(indent, "{{bold}}END STEP:{{/}} %s {{gray}}%s@ %s (%s){{/}}", event.Message, location, event.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT), event.Duration.Round(time.Millisecond))) + case types.SpecEventNodeStart: + r.emitBlock(r.fi(indent, "> Enter {{bold}}[%s]{{/}} %s {{gray}}%s@ %s{{/}}", event.NodeType.String(), event.Message, location, event.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT))) + case types.SpecEventNodeEnd: + r.emitBlock(r.fi(indent, "< Exit {{bold}}[%s]{{/}} %s {{gray}}%s@ %s (%s){{/}}", event.NodeType.String(), event.Message, location, event.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT), event.Duration.Round(time.Millisecond))) + case types.SpecEventSpecRepeat: + r.emitBlock(r.fi(indent, "\n{{bold}}Attempt #%d {{green}}Passed{{/}}{{bold}}. Repeating %s{{/}} {{gray}}@ %s{{/}}\n\n", event.Attempt, r.retryDenoter, event.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT))) + case types.SpecEventSpecRetry: + r.emitBlock(r.fi(indent, "\n{{bold}}Attempt #%d {{red}}Failed{{/}}{{bold}}. Retrying %s{{/}} {{gray}}@ %s{{/}}\n\n", event.Attempt, r.retryDenoter, event.TimelineLocation.Time.Format(types.GINKGO_TIME_FORMAT))) } - r.emitBlock(r.fi(indent, "{{gray}}<< End Captured GinkgoWriter Output{{/}}")) } func (r *DefaultReporter) emitGoroutines(indent uint, goroutines ...types.Goroutine) { @@ -457,31 +622,37 @@ func (r *DefaultReporter) emitSource(indent uint, fc types.FunctionCall) { /* Emitting to the writer */ func (r *DefaultReporter) emit(s string) { - if len(s) > 0 { - r.lastChar = s[len(s)-1:] - r.lastEmissionWasDelimiter = false - r.writer.Write([]byte(s)) - } + r._emit(s, false, false) } func (r *DefaultReporter) emitBlock(s string) { - if len(s) > 0 { - if r.lastChar != "\n" { - r.emit("\n") - } - r.emit(s) - if r.lastChar != "\n" { - r.emit("\n") - } - } + r._emit(s, true, false) +} + +func (r *DefaultReporter) emitDelimiter(indent uint) { + r._emit(r.fi(indent, "{{gray}}%s{{/}}", strings.Repeat("-", 30)), true, true) } -func (r *DefaultReporter) emitDelimiter() { - if r.lastEmissionWasDelimiter { +// a bit ugly - but we're trying to minimize locking on this hot codepath +func (r *DefaultReporter) _emit(s string, block bool, isDelimiter bool) { + if len(s) == 0 { return } - r.emitBlock(r.f("{{gray}}%s{{/}}", strings.Repeat("-", 30))) - r.lastEmissionWasDelimiter = true + r.lock.Lock() + defer r.lock.Unlock() + if isDelimiter && r.lastEmissionWasDelimiter { + return + } + if block && !r.lastCharWasNewline { + r.writer.Write([]byte("\n")) + } + r.lastCharWasNewline = (s[len(s)-1:] == "\n") + r.writer.Write([]byte(s)) + if block && !r.lastCharWasNewline { + r.writer.Write([]byte("\n")) + r.lastCharWasNewline = true + } + r.lastEmissionWasDelimiter = isDelimiter } /* Rendering text */ @@ -497,13 +668,14 @@ func (r *DefaultReporter) cycleJoin(elements []string, joiner string) string { return r.formatter.CycleJoin(elements, joiner, []string{"{{/}}", "{{gray}}"}) } -func (r *DefaultReporter) codeLocationBlock(report types.SpecReport, highlightColor string, succinct bool, usePreciseFailureLocation bool) string { +func (r *DefaultReporter) codeLocationBlock(report types.SpecReport, highlightColor string, veryVerbose bool, usePreciseFailureLocation bool) string { texts, locations, labels := []string{}, []types.CodeLocation{}, [][]string{} texts, locations, labels = append(texts, report.ContainerHierarchyTexts...), append(locations, report.ContainerHierarchyLocations...), append(labels, report.ContainerHierarchyLabels...) + if report.LeafNodeType.Is(types.NodeTypesForSuiteLevelNodes) { texts = append(texts, r.f("[%s] %s", report.LeafNodeType, report.LeafNodeText)) } else { - texts = append(texts, report.LeafNodeText) + texts = append(texts, r.f(report.LeafNodeText)) } labels = append(labels, report.LeafNodeLabels) locations = append(locations, report.LeafNodeLocation) @@ -513,24 +685,58 @@ func (r *DefaultReporter) codeLocationBlock(report types.SpecReport, highlightCo failureLocation = report.Failure.Location } + highlightIndex := -1 switch report.Failure.FailureNodeContext { case types.FailureNodeAtTopLevel: - texts = append([]string{r.f(highlightColor+"{{bold}}TOP-LEVEL [%s]{{/}}", report.Failure.FailureNodeType)}, texts...) + texts = append([]string{fmt.Sprintf("TOP-LEVEL [%s]", report.Failure.FailureNodeType)}, texts...) locations = append([]types.CodeLocation{failureLocation}, locations...) labels = append([][]string{{}}, labels...) + highlightIndex = 0 case types.FailureNodeInContainer: i := report.Failure.FailureNodeContainerIndex - texts[i] = r.f(highlightColor+"{{bold}}%s [%s]{{/}}", texts[i], report.Failure.FailureNodeType) + texts[i] = fmt.Sprintf("%s [%s]", texts[i], report.Failure.FailureNodeType) locations[i] = failureLocation + highlightIndex = i case types.FailureNodeIsLeafNode: i := len(texts) - 1 - texts[i] = r.f(highlightColor+"{{bold}}[%s] %s{{/}}", report.LeafNodeType, report.LeafNodeText) + texts[i] = fmt.Sprintf("[%s] %s", report.LeafNodeType, report.LeafNodeText) locations[i] = failureLocation + highlightIndex = i + default: + //there is no failure, so we highlight the leaf ndoe + highlightIndex = len(texts) - 1 } out := "" - if succinct { - out += r.f("%s", r.cycleJoin(texts, " ")) + if veryVerbose { + for i := range texts { + if i == highlightIndex { + out += r.fi(uint(i), highlightColor+"{{bold}}%s{{/}}", texts[i]) + } else { + out += r.fi(uint(i), "%s", texts[i]) + } + if len(labels[i]) > 0 { + out += r.f(" {{coral}}[%s]{{/}}", strings.Join(labels[i], ", ")) + } + out += "\n" + out += r.fi(uint(i), "{{gray}}%s{{/}}\n", locations[i]) + } + } else { + for i := range texts { + style := "{{/}}" + if i%2 == 1 { + style = "{{gray}}" + } + if i == highlightIndex { + style = highlightColor + "{{bold}}" + } + out += r.f(style+"%s", texts[i]) + if i < len(texts)-1 { + out += " " + } else { + out += r.f("{{/}}") + } + } flattenedLabels := report.Labels() if len(flattenedLabels) > 0 { out += r.f(" {{coral}}[%s]{{/}}", strings.Join(flattenedLabels, ", ")) @@ -539,17 +745,15 @@ func (r *DefaultReporter) codeLocationBlock(report types.SpecReport, highlightCo if usePreciseFailureLocation { out += r.f("{{gray}}%s{{/}}", failureLocation) } else { - out += r.f("{{gray}}%s{{/}}", locations[len(locations)-1]) - } - } else { - for i := range texts { - out += r.fi(uint(i), "%s", texts[i]) - if len(labels[i]) > 0 { - out += r.f(" {{coral}}[%s]{{/}}", strings.Join(labels[i], ", ")) + leafLocation := locations[len(locations)-1] + if (report.Failure.FailureNodeLocation != types.CodeLocation{}) && (report.Failure.FailureNodeLocation != leafLocation) { + out += r.fi(1, highlightColor+"[%s]{{/}} {{gray}}%s{{/}}\n", report.Failure.FailureNodeType, report.Failure.FailureNodeLocation) + out += r.fi(1, "{{gray}}[%s] %s{{/}}", report.LeafNodeType, leafLocation) + } else { + out += r.f("{{gray}}%s{{/}}", leafLocation) } - out += "\n" - out += r.fi(uint(i), "{{gray}}%s{{/}}\n", locations[i]) } + } return out } diff --git a/vendor/github.com/onsi/ginkgo/v2/reporters/deprecated_reporter.go b/vendor/github.com/onsi/ginkgo/v2/reporters/deprecated_reporter.go index 89d30076b..613072ebf 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporters/deprecated_reporter.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporters/deprecated_reporter.go @@ -35,7 +35,7 @@ func ReportViaDeprecatedReporter(reporter DeprecatedReporter, report types.Repor FailOnPending: report.SuiteConfig.FailOnPending, FailFast: report.SuiteConfig.FailFast, FlakeAttempts: report.SuiteConfig.FlakeAttempts, - EmitSpecProgress: report.SuiteConfig.EmitSpecProgress, + EmitSpecProgress: false, DryRun: report.SuiteConfig.DryRun, ParallelNode: report.SuiteConfig.ParallelProcess, ParallelTotal: report.SuiteConfig.ParallelTotal, diff --git a/vendor/github.com/onsi/ginkgo/v2/reporters/junit_report.go b/vendor/github.com/onsi/ginkgo/v2/reporters/junit_report.go index 0f165069a..ca98609d0 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporters/junit_report.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporters/junit_report.go @@ -15,12 +15,32 @@ import ( "fmt" "os" "strings" - "time" "github.com/onsi/ginkgo/v2/config" "github.com/onsi/ginkgo/v2/types" ) +type JunitReportConfig struct { + // Spec States for which no timeline should be emitted for system-err + // set this to types.SpecStatePassed|types.SpecStateSkipped|types.SpecStatePending to only match failing specs + OmitTimelinesForSpecState types.SpecState + + // Enable OmitFailureMessageAttr to prevent failure messages appearing in the "message" attribute of the Failure and Error tags + OmitFailureMessageAttr bool + + //Enable OmitCapturedStdOutErr to prevent captured stdout/stderr appearing in system-out + OmitCapturedStdOutErr bool + + // Enable OmitSpecLabels to prevent labels from appearing in the spec name + OmitSpecLabels bool + + // Enable OmitLeafNodeType to prevent the spec leaf node type from appearing in the spec name + OmitLeafNodeType bool + + // Enable OmitSuiteSetupNodes to prevent the creation of testcase entries for setup nodes + OmitSuiteSetupNodes bool +} + type JUnitTestSuites struct { XMLName xml.Name `xml:"testsuites"` // Tests maps onto the total number of specs in all test suites (this includes any suite nodes such as BeforeSuite) @@ -128,6 +148,10 @@ type JUnitFailure struct { } func GenerateJUnitReport(report types.Report, dst string) error { + return GenerateJUnitReportWithConfig(report, dst, JunitReportConfig{}) +} + +func GenerateJUnitReportWithConfig(report types.Report, dst string, config JunitReportConfig) error { suite := JUnitTestSuite{ Name: report.SuiteDescription, Package: report.SuitePath, @@ -149,7 +173,6 @@ func GenerateJUnitReport(report types.Report, dst string) error { {"FailOnPending", fmt.Sprintf("%t", report.SuiteConfig.FailOnPending)}, {"FailFast", fmt.Sprintf("%t", report.SuiteConfig.FailFast)}, {"FlakeAttempts", fmt.Sprintf("%d", report.SuiteConfig.FlakeAttempts)}, - {"EmitSpecProgress", fmt.Sprintf("%t", report.SuiteConfig.EmitSpecProgress)}, {"DryRun", fmt.Sprintf("%t", report.SuiteConfig.DryRun)}, {"ParallelTotal", fmt.Sprintf("%d", report.SuiteConfig.ParallelTotal)}, {"OutputInterceptorMode", report.SuiteConfig.OutputInterceptorMode}, @@ -157,22 +180,33 @@ func GenerateJUnitReport(report types.Report, dst string) error { }, } for _, spec := range report.SpecReports { + if config.OmitSuiteSetupNodes && spec.LeafNodeType != types.NodeTypeIt { + continue + } name := fmt.Sprintf("[%s]", spec.LeafNodeType) + if config.OmitLeafNodeType { + name = "" + } if spec.FullText() != "" { name = name + " " + spec.FullText() } labels := spec.Labels() - if len(labels) > 0 { + if len(labels) > 0 && !config.OmitSpecLabels { name = name + " [" + strings.Join(labels, ", ") + "]" } + name = strings.TrimSpace(name) test := JUnitTestCase{ Name: name, Classname: report.SuiteDescription, Status: spec.State.String(), Time: spec.RunTime.Seconds(), - SystemOut: systemOutForUnstructuredReporters(spec), - SystemErr: systemErrForUnstructuredReporters(spec), + } + if !spec.State.Is(config.OmitTimelinesForSpecState) { + test.SystemErr = systemErrForUnstructuredReporters(spec) + } + if !config.OmitCapturedStdOutErr { + test.SystemOut = systemOutForUnstructuredReporters(spec) } suite.Tests += 1 @@ -191,28 +225,50 @@ func GenerateJUnitReport(report types.Report, dst string) error { test.Failure = &JUnitFailure{ Message: spec.Failure.Message, Type: "failed", - Description: fmt.Sprintf("%s\n%s", spec.Failure.Location.String(), spec.Failure.Location.FullStackTrace), + Description: failureDescriptionForUnstructuredReporters(spec), + } + if config.OmitFailureMessageAttr { + test.Failure.Message = "" + } + suite.Failures += 1 + case types.SpecStateTimedout: + test.Failure = &JUnitFailure{ + Message: spec.Failure.Message, + Type: "timedout", + Description: failureDescriptionForUnstructuredReporters(spec), + } + if config.OmitFailureMessageAttr { + test.Failure.Message = "" } suite.Failures += 1 case types.SpecStateInterrupted: test.Error = &JUnitError{ - Message: "interrupted", + Message: spec.Failure.Message, Type: "interrupted", - Description: interruptDescriptionForUnstructuredReporters(spec.Failure), + Description: failureDescriptionForUnstructuredReporters(spec), + } + if config.OmitFailureMessageAttr { + test.Error.Message = "" } suite.Errors += 1 case types.SpecStateAborted: test.Failure = &JUnitFailure{ Message: spec.Failure.Message, Type: "aborted", - Description: fmt.Sprintf("%s\n%s", spec.Failure.Location.String(), spec.Failure.Location.FullStackTrace), + Description: failureDescriptionForUnstructuredReporters(spec), + } + if config.OmitFailureMessageAttr { + test.Failure.Message = "" } suite.Errors += 1 case types.SpecStatePanicked: test.Error = &JUnitError{ Message: spec.Failure.ForwardedPanic, Type: "panicked", - Description: fmt.Sprintf("%s\n%s", spec.Failure.Location.String(), spec.Failure.Location.FullStackTrace), + Description: failureDescriptionForUnstructuredReporters(spec), + } + if config.OmitFailureMessageAttr { + test.Error.Message = "" } suite.Errors += 1 } @@ -278,52 +334,23 @@ func MergeAndCleanupJUnitReports(sources []string, dst string) ([]string, error) return messages, f.Close() } -func interruptDescriptionForUnstructuredReporters(failure types.Failure) string { +func failureDescriptionForUnstructuredReporters(spec types.SpecReport) string { out := &strings.Builder{} - out.WriteString(failure.Message + "\n") - NewDefaultReporter(types.ReporterConfig{NoColor: true}, out).EmitProgressReport(failure.ProgressReport) + NewDefaultReporter(types.ReporterConfig{NoColor: true, VeryVerbose: true}, out).emitFailure(0, spec.State, spec.Failure, true) + if len(spec.AdditionalFailures) > 0 { + out.WriteString("\nThere were additional failures detected after the initial failure. These are visible in the timeline\n") + } return out.String() } func systemErrForUnstructuredReporters(spec types.SpecReport) string { out := &strings.Builder{} - gw := spec.CapturedGinkgoWriterOutput - cursor := 0 - for _, pr := range spec.ProgressReports { - if cursor < pr.GinkgoWriterOffset { - if pr.GinkgoWriterOffset < len(gw) { - out.WriteString(gw[cursor:pr.GinkgoWriterOffset]) - cursor = pr.GinkgoWriterOffset - } else if cursor < len(gw) { - out.WriteString(gw[cursor:]) - cursor = len(gw) - } - } - NewDefaultReporter(types.ReporterConfig{NoColor: true}, out).EmitProgressReport(pr) - } - - if cursor < len(gw) { - out.WriteString(gw[cursor:]) - } - + NewDefaultReporter(types.ReporterConfig{NoColor: true, VeryVerbose: true}, out).emitTimeline(0, spec, spec.Timeline()) return out.String() } func systemOutForUnstructuredReporters(spec types.SpecReport) string { - systemOut := spec.CapturedStdOutErr - if len(spec.ReportEntries) > 0 { - systemOut += "\nReport Entries:\n" - for i, entry := range spec.ReportEntries { - systemOut += fmt.Sprintf("%s\n%s\n%s\n", entry.Name, entry.Location, entry.Time.Format(time.RFC3339Nano)) - if representation := entry.StringRepresentation(); representation != "" { - systemOut += representation + "\n" - } - if i+1 < len(spec.ReportEntries) { - systemOut += "--\n" - } - } - } - return systemOut + return spec.CapturedStdOutErr } // Deprecated JUnitReporter (so folks can still compile their suites) diff --git a/vendor/github.com/onsi/ginkgo/v2/reporters/reporter.go b/vendor/github.com/onsi/ginkgo/v2/reporters/reporter.go index f79f005db..5e726c464 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporters/reporter.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporters/reporter.go @@ -9,13 +9,21 @@ type Reporter interface { WillRun(report types.SpecReport) DidRun(report types.SpecReport) SuiteDidEnd(report types.Report) + + //Timeline emission + EmitFailure(state types.SpecState, failure types.Failure) EmitProgressReport(progressReport types.ProgressReport) + EmitReportEntry(entry types.ReportEntry) + EmitSpecEvent(event types.SpecEvent) } type NoopReporter struct{} -func (n NoopReporter) SuiteWillBegin(report types.Report) {} -func (n NoopReporter) WillRun(report types.SpecReport) {} -func (n NoopReporter) DidRun(report types.SpecReport) {} -func (n NoopReporter) SuiteDidEnd(report types.Report) {} -func (n NoopReporter) EmitProgressReport(progressReport types.ProgressReport) {} +func (n NoopReporter) SuiteWillBegin(report types.Report) {} +func (n NoopReporter) WillRun(report types.SpecReport) {} +func (n NoopReporter) DidRun(report types.SpecReport) {} +func (n NoopReporter) SuiteDidEnd(report types.Report) {} +func (n NoopReporter) EmitFailure(state types.SpecState, failure types.Failure) {} +func (n NoopReporter) EmitProgressReport(progressReport types.ProgressReport) {} +func (n NoopReporter) EmitReportEntry(entry types.ReportEntry) {} +func (n NoopReporter) EmitSpecEvent(event types.SpecEvent) {} diff --git a/vendor/github.com/onsi/ginkgo/v2/reporters/teamcity_report.go b/vendor/github.com/onsi/ginkgo/v2/reporters/teamcity_report.go index 00b038769..c1863496d 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporters/teamcity_report.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporters/teamcity_report.go @@ -60,15 +60,19 @@ func GenerateTeamcityReport(report types.Report, dst string) error { } fmt.Fprintf(f, "##teamcity[testIgnored name='%s' message='%s']\n", name, tcEscape(message)) case types.SpecStateFailed: - details := fmt.Sprintf("%s\n%s", spec.Failure.Location.String(), spec.Failure.Location.FullStackTrace) + details := failureDescriptionForUnstructuredReporters(spec) fmt.Fprintf(f, "##teamcity[testFailed name='%s' message='failed - %s' details='%s']\n", name, tcEscape(spec.Failure.Message), tcEscape(details)) case types.SpecStatePanicked: - details := fmt.Sprintf("%s\n%s", spec.Failure.Location.String(), spec.Failure.Location.FullStackTrace) + details := failureDescriptionForUnstructuredReporters(spec) fmt.Fprintf(f, "##teamcity[testFailed name='%s' message='panicked - %s' details='%s']\n", name, tcEscape(spec.Failure.ForwardedPanic), tcEscape(details)) + case types.SpecStateTimedout: + details := failureDescriptionForUnstructuredReporters(spec) + fmt.Fprintf(f, "##teamcity[testFailed name='%s' message='timedout - %s' details='%s']\n", name, tcEscape(spec.Failure.Message), tcEscape(details)) case types.SpecStateInterrupted: - fmt.Fprintf(f, "##teamcity[testFailed name='%s' message='interrupted' details='%s']\n", name, tcEscape(interruptDescriptionForUnstructuredReporters(spec.Failure))) + details := failureDescriptionForUnstructuredReporters(spec) + fmt.Fprintf(f, "##teamcity[testFailed name='%s' message='interrupted - %s' details='%s']\n", name, tcEscape(spec.Failure.Message), tcEscape(details)) case types.SpecStateAborted: - details := fmt.Sprintf("%s\n%s", spec.Failure.Location.String(), spec.Failure.Location.FullStackTrace) + details := failureDescriptionForUnstructuredReporters(spec) fmt.Fprintf(f, "##teamcity[testFailed name='%s' message='aborted - %s' details='%s']\n", name, tcEscape(spec.Failure.Message), tcEscape(details)) } diff --git a/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go b/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go index afc151b13..f33786a2d 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go @@ -35,7 +35,7 @@ func CurrentSpecReport() SpecReport { } /* - ReportEntryVisibility governs the visibility of ReportEntries in Ginkgo's console reporter + ReportEntryVisibility governs the visibility of ReportEntries in Ginkgo's console reporter - ReportEntryVisibilityAlways: the default behavior - the ReportEntry is always emitted. - ReportEntryVisibilityFailureOrVerbose: the ReportEntry is only emitted if the spec fails or if the tests are run with -v (similar to GinkgoWriters behavior). @@ -50,9 +50,9 @@ const ReportEntryVisibilityAlways, ReportEntryVisibilityFailureOrVerbose, Report /* AddReportEntry generates and adds a new ReportEntry to the current spec's SpecReport. It can take any of the following arguments: - - A single arbitrary object to attach as the Value of the ReportEntry. This object will be included in any generated reports and will be emitted to the console when the report is emitted. - - A ReportEntryVisibility enum to control the visibility of the ReportEntry - - An Offset or CodeLocation decoration to control the reported location of the ReportEntry + - A single arbitrary object to attach as the Value of the ReportEntry. This object will be included in any generated reports and will be emitted to the console when the report is emitted. + - A ReportEntryVisibility enum to control the visibility of the ReportEntry + - An Offset or CodeLocation decoration to control the reported location of the ReportEntry If the Value object implements `fmt.Stringer`, it's `String()` representation is used when emitting to the console. @@ -100,6 +100,25 @@ func ReportAfterEach(body func(SpecReport), args ...interface{}) bool { return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeReportAfterEach, "", combinedArgs...)) } +/* +ReportBeforeSuite nodes are run at the beginning of the suite. ReportBeforeSuite nodes take a function that receives a suite Report. + +They are called at the beginning of the suite, before any specs have run and any BeforeSuite or SynchronizedBeforeSuite nodes, and are passed in the initial report for the suite. +ReportBeforeSuite nodes must be created at the top-level (i.e. not nested in a Context/Describe/When node) + +# When running in parallel, Ginkgo ensures that only one of the parallel nodes runs the ReportBeforeSuite + +You cannot nest any other Ginkgo nodes within a ReportAfterSuite node's closure. +You can learn more about ReportAfterSuite here: https://onsi.github.io/ginkgo/#generating-reports-programmatically + +You can learn more about Ginkgo's reporting infrastructure, including generating reports with the CLI here: https://onsi.github.io/ginkgo/#generating-machine-readable-reports +*/ +func ReportBeforeSuite(body func(Report), args ...interface{}) bool { + combinedArgs := []interface{}{body} + combinedArgs = append(combinedArgs, args...) + return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeReportBeforeSuite, "", combinedArgs...)) +} + /* ReportAfterSuite nodes are run at the end of the suite. ReportAfterSuite nodes take a function that receives a suite Report. @@ -113,6 +132,7 @@ In addition to using ReportAfterSuite to programmatically generate suite reports You cannot nest any other Ginkgo nodes within a ReportAfterSuite node's closure. You can learn more about ReportAfterSuite here: https://onsi.github.io/ginkgo/#generating-reports-programmatically + You can learn more about Ginkgo's reporting infrastructure, including generating reports with the CLI here: https://onsi.github.io/ginkgo/#generating-machine-readable-reports */ func ReportAfterSuite(text string, body func(Report), args ...interface{}) bool { diff --git a/vendor/github.com/onsi/ginkgo/v2/table_dsl.go b/vendor/github.com/onsi/ginkgo/v2/table_dsl.go index 0db0e7180..ac9b7abb5 100644 --- a/vendor/github.com/onsi/ginkgo/v2/table_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/table_dsl.go @@ -1,6 +1,7 @@ package ginkgo import ( + "context" "fmt" "reflect" "strings" @@ -12,7 +13,7 @@ import ( /* The EntryDescription decorator allows you to pass a format string to DescribeTable() and Entry(). This format string is used to generate entry names via: - fmt.Sprintf(formatString, parameters...) + fmt.Sprintf(formatString, parameters...) where parameters are the parameters passed into the entry. @@ -31,19 +32,20 @@ DescribeTable describes a table-driven spec. For example: - DescribeTable("a simple table", - func(x int, y int, expected bool) { - Ω(x > y).Should(Equal(expected)) - }, - Entry("x > y", 1, 0, true), - Entry("x == y", 0, 0, false), - Entry("x < y", 0, 1, false), - ) + DescribeTable("a simple table", + func(x int, y int, expected bool) { + Ω(x > y).Should(Equal(expected)) + }, + Entry("x > y", 1, 0, true), + Entry("x == y", 0, 0, false), + Entry("x < y", 0, 1, false), + ) You can learn more about DescribeTable here: https://onsi.github.io/ginkgo/#table-specs And can explore some Table patterns here: https://onsi.github.io/ginkgo/#table-specs-patterns */ func DescribeTable(description string, args ...interface{}) bool { + GinkgoHelper() generateTable(description, args...) return true } @@ -52,6 +54,7 @@ func DescribeTable(description string, args ...interface{}) bool { You can focus a table with `FDescribeTable`. This is equivalent to `FDescribe`. */ func FDescribeTable(description string, args ...interface{}) bool { + GinkgoHelper() args = append(args, internal.Focus) generateTable(description, args...) return true @@ -61,6 +64,7 @@ func FDescribeTable(description string, args ...interface{}) bool { You can mark a table as pending with `PDescribeTable`. This is equivalent to `PDescribe`. */ func PDescribeTable(description string, args ...interface{}) bool { + GinkgoHelper() args = append(args, internal.Pending) generateTable(description, args...) return true @@ -89,29 +93,34 @@ Subsequent arguments accept any Ginkgo decorators. These are filtered out and t Each Entry ends up generating an individual Ginkgo It. The body of the it is the Table Body function with the Entry parameters passed in. +If you want to generate interruptible specs simply write a Table function that accepts a SpecContext as its first argument. You can then decorate individual Entrys with the NodeTimeout and SpecTimeout decorators. + You can learn more about Entry here: https://onsi.github.io/ginkgo/#table-specs */ func Entry(description interface{}, args ...interface{}) TableEntry { + GinkgoHelper() decorations, parameters := internal.PartitionDecorations(args...) - return TableEntry{description: description, decorations: decorations, parameters: parameters, codeLocation: types.NewCodeLocation(1)} + return TableEntry{description: description, decorations: decorations, parameters: parameters, codeLocation: types.NewCodeLocation(0)} } /* You can focus a particular entry with FEntry. This is equivalent to FIt. */ func FEntry(description interface{}, args ...interface{}) TableEntry { + GinkgoHelper() decorations, parameters := internal.PartitionDecorations(args...) decorations = append(decorations, internal.Focus) - return TableEntry{description: description, decorations: decorations, parameters: parameters, codeLocation: types.NewCodeLocation(1)} + return TableEntry{description: description, decorations: decorations, parameters: parameters, codeLocation: types.NewCodeLocation(0)} } /* You can mark a particular entry as pending with PEntry. This is equivalent to PIt. */ func PEntry(description interface{}, args ...interface{}) TableEntry { + GinkgoHelper() decorations, parameters := internal.PartitionDecorations(args...) decorations = append(decorations, internal.Pending) - return TableEntry{description: description, decorations: decorations, parameters: parameters, codeLocation: types.NewCodeLocation(1)} + return TableEntry{description: description, decorations: decorations, parameters: parameters, codeLocation: types.NewCodeLocation(0)} } /* @@ -119,12 +128,17 @@ You can mark a particular entry as pending with XEntry. This is equivalent to X */ var XEntry = PEntry +var contextType = reflect.TypeOf(new(context.Context)).Elem() +var specContextType = reflect.TypeOf(new(SpecContext)).Elem() + func generateTable(description string, args ...interface{}) { - cl := types.NewCodeLocation(2) + GinkgoHelper() + cl := types.NewCodeLocation(0) containerNodeArgs := []interface{}{cl} entries := []TableEntry{} var itBody interface{} + var itBodyType reflect.Type var tableLevelEntryDescription interface{} tableLevelEntryDescription = func(args ...interface{}) string { @@ -135,6 +149,10 @@ func generateTable(description string, args ...interface{}) { return "Entry: " + strings.Join(out, ", ") } + if len(args) == 1 { + exitIfErr(types.GinkgoErrors.MissingParametersForTableFunction(cl)) + } + for i, arg := range args { switch t := reflect.TypeOf(arg); { case t == nil: @@ -152,6 +170,7 @@ func generateTable(description string, args ...interface{}) { exitIfErr(types.GinkgoErrors.MultipleEntryBodyFunctionsForTable(cl)) } itBody = arg + itBodyType = reflect.TypeOf(itBody) default: containerNodeArgs = append(containerNodeArgs, arg) } @@ -164,7 +183,7 @@ func generateTable(description string, args ...interface{}) { var description string switch t := reflect.TypeOf(entry.description); { case t == nil: - err = validateParameters(tableLevelEntryDescription, entry.parameters, "Entry Description function", entry.codeLocation) + err = validateParameters(tableLevelEntryDescription, entry.parameters, "Entry Description function", entry.codeLocation, false) if err == nil { description = invokeFunction(tableLevelEntryDescription, entry.parameters)[0].String() } @@ -173,7 +192,7 @@ func generateTable(description string, args ...interface{}) { case t == reflect.TypeOf(""): description = entry.description.(string) case t.Kind() == reflect.Func && t.NumOut() == 1 && t.Out(0) == reflect.TypeOf(""): - err = validateParameters(entry.description, entry.parameters, "Entry Description function", entry.codeLocation) + err = validateParameters(entry.description, entry.parameters, "Entry Description function", entry.codeLocation, false) if err == nil { description = invokeFunction(entry.description, entry.parameters)[0].String() } @@ -181,17 +200,37 @@ func generateTable(description string, args ...interface{}) { err = types.GinkgoErrors.InvalidEntryDescription(entry.codeLocation) } - if err == nil { - err = validateParameters(itBody, entry.parameters, "Table Body function", entry.codeLocation) - } itNodeArgs := []interface{}{entry.codeLocation} itNodeArgs = append(itNodeArgs, entry.decorations...) - itNodeArgs = append(itNodeArgs, func() { - if err != nil { - panic(err) + + hasContext := false + if itBodyType.NumIn() > 0. { + if itBodyType.In(0).Implements(specContextType) { + hasContext = true + } else if itBodyType.In(0).Implements(contextType) && (len(entry.parameters) == 0 || !reflect.TypeOf(entry.parameters[0]).Implements(contextType)) { + hasContext = true } - invokeFunction(itBody, entry.parameters) - }) + } + + if err == nil { + err = validateParameters(itBody, entry.parameters, "Table Body function", entry.codeLocation, hasContext) + } + + if hasContext { + itNodeArgs = append(itNodeArgs, func(c SpecContext) { + if err != nil { + panic(err) + } + invokeFunction(itBody, append([]interface{}{c}, entry.parameters...)) + }) + } else { + itNodeArgs = append(itNodeArgs, func() { + if err != nil { + panic(err) + } + invokeFunction(itBody, entry.parameters) + }) + } pushNode(internal.NewNode(deprecationTracker, types.NodeTypeIt, description, itNodeArgs...)) } @@ -223,9 +262,14 @@ func invokeFunction(function interface{}, parameters []interface{}) []reflect.Va return reflect.ValueOf(function).Call(inValues) } -func validateParameters(function interface{}, parameters []interface{}, kind string, cl types.CodeLocation) error { +func validateParameters(function interface{}, parameters []interface{}, kind string, cl types.CodeLocation, hasContext bool) error { funcType := reflect.TypeOf(function) limit := funcType.NumIn() + offset := 0 + if hasContext { + limit = limit - 1 + offset = 1 + } if funcType.IsVariadic() { limit = limit - 1 } @@ -238,13 +282,13 @@ func validateParameters(function interface{}, parameters []interface{}, kind str var i = 0 for ; i < limit; i++ { actual := reflect.TypeOf(parameters[i]) - expected := funcType.In(i) + expected := funcType.In(i + offset) if !(actual == nil) && !actual.AssignableTo(expected) { return types.GinkgoErrors.IncorrectParameterTypeToTableFunction(i+1, expected, actual, kind, cl) } } if funcType.IsVariadic() { - expected := funcType.In(limit).Elem() + expected := funcType.In(limit + offset).Elem() for ; i < len(parameters); i++ { actual := reflect.TypeOf(parameters[i]) if !(actual == nil) && !actual.AssignableTo(expected) { diff --git a/vendor/github.com/onsi/ginkgo/v2/types/code_location.go b/vendor/github.com/onsi/ginkgo/v2/types/code_location.go index 129109183..9cd576817 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/code_location.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/code_location.go @@ -7,6 +7,7 @@ import ( "runtime" "runtime/debug" "strings" + "sync" ) type CodeLocation struct { @@ -38,6 +39,73 @@ func (codeLocation CodeLocation) ContentsOfLine() string { return lines[codeLocation.LineNumber-1] } +type codeLocationLocator struct { + pcs map[uintptr]bool + helpers map[string]bool + lock *sync.Mutex +} + +func (c *codeLocationLocator) addHelper(pc uintptr) { + c.lock.Lock() + defer c.lock.Unlock() + + if c.pcs[pc] { + return + } + c.lock.Unlock() + f := runtime.FuncForPC(pc) + c.lock.Lock() + if f == nil { + return + } + c.helpers[f.Name()] = true + c.pcs[pc] = true +} + +func (c *codeLocationLocator) hasHelper(name string) bool { + c.lock.Lock() + defer c.lock.Unlock() + return c.helpers[name] +} + +func (c *codeLocationLocator) getCodeLocation(skip int) CodeLocation { + pc := make([]uintptr, 40) + n := runtime.Callers(skip+2, pc) + if n == 0 { + return CodeLocation{} + } + pc = pc[:n] + frames := runtime.CallersFrames(pc) + for { + frame, more := frames.Next() + if !c.hasHelper(frame.Function) { + return CodeLocation{FileName: frame.File, LineNumber: frame.Line} + } + if !more { + break + } + } + return CodeLocation{} +} + +var clLocator = &codeLocationLocator{ + pcs: map[uintptr]bool{}, + helpers: map[string]bool{}, + lock: &sync.Mutex{}, +} + +// MarkAsHelper is used by GinkgoHelper to mark the caller (appropriately offset by skip)as a helper. You can use this directly if you need to provide an optional `skip` to mark functions further up the call stack as helpers. +func MarkAsHelper(optionalSkip ...int) { + skip := 1 + if len(optionalSkip) > 0 { + skip += optionalSkip[0] + } + pc, _, _, ok := runtime.Caller(skip) + if ok { + clLocator.addHelper(pc) + } +} + func NewCustomCodeLocation(message string) CodeLocation { return CodeLocation{ CustomMessage: message, @@ -45,14 +113,13 @@ func NewCustomCodeLocation(message string) CodeLocation { } func NewCodeLocation(skip int) CodeLocation { - _, file, line, _ := runtime.Caller(skip + 1) - return CodeLocation{FileName: file, LineNumber: line} + return clLocator.getCodeLocation(skip + 1) } func NewCodeLocationWithStackTrace(skip int) CodeLocation { - _, file, line, _ := runtime.Caller(skip + 1) - stackTrace := PruneStack(string(debug.Stack()), skip+1) - return CodeLocation{FileName: file, LineNumber: line, FullStackTrace: stackTrace} + cl := clLocator.getCodeLocation(skip + 1) + cl.FullStackTrace = PruneStack(string(debug.Stack()), skip+1) + return cl } // PruneStack removes references to functions that are internal to Ginkgo diff --git a/vendor/github.com/onsi/ginkgo/v2/types/config.go b/vendor/github.com/onsi/ginkgo/v2/types/config.go index 438c947c8..1efd77d39 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/config.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/config.go @@ -26,13 +26,14 @@ type SuiteConfig struct { FailOnPending bool FailFast bool FlakeAttempts int - EmitSpecProgress bool DryRun bool PollProgressAfter time.Duration PollProgressInterval time.Duration Timeout time.Duration + EmitSpecProgress bool // this is deprecated but its removal is causing compile issue for some users that were setting it manually OutputInterceptorMode string SourceRoots []string + GracePeriod time.Duration ParallelProcess int ParallelTotal int @@ -45,6 +46,7 @@ func NewDefaultSuiteConfig() SuiteConfig { Timeout: time.Hour, ParallelProcess: 1, ParallelTotal: 1, + GracePeriod: 30 * time.Second, } } @@ -79,13 +81,12 @@ func (vl VerbosityLevel) LT(comp VerbosityLevel) bool { // Configuration for Ginkgo's reporter type ReporterConfig struct { - NoColor bool - SlowSpecThreshold time.Duration - Succinct bool - Verbose bool - VeryVerbose bool - FullTrace bool - AlwaysEmitGinkgoWriter bool + NoColor bool + Succinct bool + Verbose bool + VeryVerbose bool + FullTrace bool + ShowNodeEvents bool JSONReport string JUnitReport string @@ -108,9 +109,7 @@ func (rc ReporterConfig) WillGenerateReport() bool { } func NewDefaultReporterConfig() ReporterConfig { - return ReporterConfig{ - SlowSpecThreshold: 5 * time.Second, - } + return ReporterConfig{} } // Configuration for the Ginkgo CLI @@ -233,6 +232,9 @@ type deprecatedConfig struct { SlowSpecThresholdWithFLoatUnits float64 Stream bool Notify bool + EmitSpecProgress bool + SlowSpecThreshold time.Duration + AlwaysEmitGinkgoWriter bool } // Flags @@ -273,8 +275,6 @@ var SuiteConfigFlags = GinkgoFlags{ {KeyPath: "S.DryRun", Name: "dry-run", SectionKey: "debug", DeprecatedName: "dryRun", DeprecatedDocLink: "changed-command-line-flags", Usage: "If set, ginkgo will walk the test hierarchy without actually running anything. Best paired with -v."}, - {KeyPath: "S.EmitSpecProgress", Name: "progress", SectionKey: "debug", - Usage: "If set, ginkgo will emit progress information as each spec runs to the GinkgoWriter."}, {KeyPath: "S.PollProgressAfter", Name: "poll-progress-after", SectionKey: "debug", UsageDefaultValue: "0", Usage: "Emit node progress reports periodically if node hasn't completed after this duration."}, {KeyPath: "S.PollProgressInterval", Name: "poll-progress-interval", SectionKey: "debug", UsageDefaultValue: "10s", @@ -283,6 +283,8 @@ var SuiteConfigFlags = GinkgoFlags{ Usage: "The location to look for source code when generating progress reports. You can pass multiple --source-root flags."}, {KeyPath: "S.Timeout", Name: "timeout", SectionKey: "debug", UsageDefaultValue: "1h", Usage: "Test suite fails if it does not complete within the specified timeout."}, + {KeyPath: "S.GracePeriod", Name: "grace-period", SectionKey: "debug", UsageDefaultValue: "30s", + Usage: "When interrupted, Ginkgo will wait for GracePeriod for the current running node to exit before moving on to the next one."}, {KeyPath: "S.OutputInterceptorMode", Name: "output-interceptor-mode", SectionKey: "debug", UsageArgument: "dup, swap, or none", Usage: "If set, ginkgo will use the specified output interception strategy when running in parallel. Defaults to dup on unix and swap on windows."}, @@ -299,6 +301,8 @@ var SuiteConfigFlags = GinkgoFlags{ {KeyPath: "D.RegexScansFilePath", DeprecatedName: "regexScansFilePath", DeprecatedDocLink: "removed--regexscansfilepath", DeprecatedVersion: "2.0.0"}, {KeyPath: "D.DebugParallel", DeprecatedName: "debug", DeprecatedDocLink: "removed--debug", DeprecatedVersion: "2.0.0"}, + {KeyPath: "D.EmitSpecProgress", DeprecatedName: "progress", SectionKey: "debug", + DeprecatedVersion: "2.5.0", Usage: ". The functionality provided by --progress was confusing and is no longer needed. Use --show-node-events instead to see node entry and exit events included in the timeline of failed and verbose specs. Or you can run with -vv to always see all node events. Lastly, --poll-progress-after and the PollProgressAfter decorator now provide a better mechanism for debugging specs that tend to get stuck."}, } // ParallelConfigFlags provides flags for the Ginkgo test process (not the CLI) @@ -315,8 +319,6 @@ var ParallelConfigFlags = GinkgoFlags{ var ReporterConfigFlags = GinkgoFlags{ {KeyPath: "R.NoColor", Name: "no-color", SectionKey: "output", DeprecatedName: "noColor", DeprecatedDocLink: "changed-command-line-flags", Usage: "If set, suppress color output in default reporter."}, - {KeyPath: "R.SlowSpecThreshold", Name: "slow-spec-threshold", SectionKey: "output", UsageArgument: "duration", UsageDefaultValue: "5s", - Usage: "Specs that take longer to run than this threshold are flagged as slow by the default reporter."}, {KeyPath: "R.Verbose", Name: "v", SectionKey: "output", Usage: "If set, emits more output including GinkgoWriter contents."}, {KeyPath: "R.VeryVerbose", Name: "vv", SectionKey: "output", @@ -325,8 +327,8 @@ var ReporterConfigFlags = GinkgoFlags{ Usage: "If set, default reporter prints out a very succinct report"}, {KeyPath: "R.FullTrace", Name: "trace", SectionKey: "output", Usage: "If set, default reporter prints out the full stack trace when a failure occurs"}, - {KeyPath: "R.AlwaysEmitGinkgoWriter", Name: "always-emit-ginkgo-writer", SectionKey: "output", DeprecatedName: "reportPassed", DeprecatedDocLink: "renamed--reportpassed", - Usage: "If set, default reporter prints out captured output of passed tests."}, + {KeyPath: "R.ShowNodeEvents", Name: "show-node-events", SectionKey: "output", + Usage: "If set, default reporter prints node > Enter and < Exit events when specs fail"}, {KeyPath: "R.JSONReport", Name: "json-report", UsageArgument: "filename.json", SectionKey: "output", Usage: "If set, Ginkgo will generate a JSON-formatted test report at the specified location."}, @@ -339,6 +341,8 @@ var ReporterConfigFlags = GinkgoFlags{ Usage: "use --slow-spec-threshold instead and pass in a duration string (e.g. '5s', not '5.0')"}, {KeyPath: "D.NoisyPendings", DeprecatedName: "noisyPendings", DeprecatedDocLink: "removed--noisypendings-and--noisyskippings", DeprecatedVersion: "2.0.0"}, {KeyPath: "D.NoisySkippings", DeprecatedName: "noisySkippings", DeprecatedDocLink: "removed--noisypendings-and--noisyskippings", DeprecatedVersion: "2.0.0"}, + {KeyPath: "D.SlowSpecThreshold", DeprecatedName: "slow-spec-threshold", SectionKey: "output", Usage: "--slow-spec-threshold has been deprecated and will be removed in a future version of Ginkgo. This feature has proved to be more noisy than useful. You can use --poll-progress-after, instead, to get more actionable feedback about potentially slow specs and understand where they might be getting stuck.", DeprecatedVersion: "2.5.0"}, + {KeyPath: "D.AlwaysEmitGinkgoWriter", DeprecatedName: "always-emit-ginkgo-writer", SectionKey: "output", Usage: " - use -v instead, or one of Ginkgo's machine-readable report formats to get GinkgoWriter output for passing specs."}, } // BuildTestSuiteFlagSet attaches to the CommandLine flagset and provides flags for the Ginkgo test process @@ -390,6 +394,10 @@ func VetConfig(flagSet GinkgoFlagSet, suiteConfig SuiteConfig, reporterConfig Re errors = append(errors, GinkgoErrors.DryRunInParallelConfiguration()) } + if suiteConfig.GracePeriod <= 0 { + errors = append(errors, GinkgoErrors.GracePeriodCannotBeZero()) + } + if len(suiteConfig.FocusFiles) > 0 { _, err := ParseFileFilters(suiteConfig.FocusFiles) if err != nil { diff --git a/vendor/github.com/onsi/ginkgo/v2/types/deprecation_support.go b/vendor/github.com/onsi/ginkgo/v2/types/deprecation_support.go index 2948dfa0c..f267bdefd 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/deprecation_support.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/deprecation_support.go @@ -83,6 +83,13 @@ func (d deprecations) Nodot() Deprecation { } } +func (d deprecations) SuppressProgressReporting() Deprecation { + return Deprecation{ + Message: "Improvements to how reporters emit timeline information means that SuppressProgressReporting is no longer necessary and has been deprecated.", + Version: "2.5.0", + } +} + type DeprecationTracker struct { deprecations map[Deprecation][]CodeLocation lock *sync.Mutex diff --git a/vendor/github.com/onsi/ginkgo/v2/types/errors.go b/vendor/github.com/onsi/ginkgo/v2/types/errors.go index 6806d6afc..1e0dbfd9d 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/errors.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/errors.go @@ -108,8 +108,8 @@ Please ensure all assertions are inside leaf nodes such as {{bold}}BeforeEach{{/ func (g ginkgoErrors) SuiteNodeInNestedContext(nodeType NodeType, cl CodeLocation) error { docLink := "suite-setup-and-cleanup-beforesuite-and-aftersuite" - if nodeType.Is(NodeTypeReportAfterSuite) { - docLink = "reporting-nodes---reportaftersuite" + if nodeType.Is(NodeTypeReportBeforeSuite | NodeTypeReportAfterSuite) { + docLink = "reporting-nodes---reportbeforesuite-and-reportaftersuite" } return GinkgoError{ @@ -125,8 +125,8 @@ func (g ginkgoErrors) SuiteNodeInNestedContext(nodeType NodeType, cl CodeLocatio func (g ginkgoErrors) SuiteNodeDuringRunPhase(nodeType NodeType, cl CodeLocation) error { docLink := "suite-setup-and-cleanup-beforesuite-and-aftersuite" - if nodeType.Is(NodeTypeReportAfterSuite) { - docLink = "reporting-nodes---reportaftersuite" + if nodeType.Is(NodeTypeReportBeforeSuite | NodeTypeReportAfterSuite) { + docLink = "reporting-nodes---reportbeforesuite-and-reportaftersuite" } return GinkgoError{ @@ -180,6 +180,15 @@ func (g ginkgoErrors) InvalidDeclarationOfFocusedAndPending(cl CodeLocation, nod } } +func (g ginkgoErrors) InvalidDeclarationOfFlakeAttemptsAndMustPassRepeatedly(cl CodeLocation, nodeType NodeType) error { + return GinkgoError{ + Heading: "Invalid Combination of Decorators: FlakeAttempts and MustPassRepeatedly", + Message: formatter.F(`[%s] node was decorated with both FlakeAttempts and MustPassRepeatedly. At most one is allowed.`, nodeType), + CodeLocation: cl, + DocLink: "node-decorators-overview", + } +} + func (g ginkgoErrors) UnknownDecorator(cl CodeLocation, nodeType NodeType, decorator interface{}) error { return GinkgoError{ Heading: "Unknown Decorator", @@ -189,20 +198,55 @@ func (g ginkgoErrors) UnknownDecorator(cl CodeLocation, nodeType NodeType, decor } } +func (g ginkgoErrors) InvalidBodyTypeForContainer(t reflect.Type, cl CodeLocation, nodeType NodeType) error { + return GinkgoError{ + Heading: "Invalid Function", + Message: formatter.F(`[%s] node must be passed {{bold}}func(){{/}} - i.e. functions that take nothing and return nothing. You passed {{bold}}%s{{/}} instead.`, nodeType, t), + CodeLocation: cl, + DocLink: "node-decorators-overview", + } +} + func (g ginkgoErrors) InvalidBodyType(t reflect.Type, cl CodeLocation, nodeType NodeType) error { + mustGet := "{{bold}}func(){{/}}, {{bold}}func(ctx SpecContext){{/}}, or {{bold}}func(ctx context.Context){{/}}" + if nodeType.Is(NodeTypeContainer) { + mustGet = "{{bold}}func(){{/}}" + } return GinkgoError{ Heading: "Invalid Function", - Message: formatter.F(`[%s] node must be passed {{bold}}func(){{/}} - i.e. functions that take nothing and return nothing. + Message: formatter.F(`[%s] node must be passed `+mustGet+`. You passed {{bold}}%s{{/}} instead.`, nodeType, t), CodeLocation: cl, DocLink: "node-decorators-overview", } } +func (g ginkgoErrors) InvalidBodyTypeForSynchronizedBeforeSuiteProc1(t reflect.Type, cl CodeLocation) error { + mustGet := "{{bold}}func() []byte{{/}}, {{bold}}func(ctx SpecContext) []byte{{/}}, or {{bold}}func(ctx context.Context) []byte{{/}}, {{bold}}func(){{/}}, {{bold}}func(ctx SpecContext){{/}}, or {{bold}}func(ctx context.Context){{/}}" + return GinkgoError{ + Heading: "Invalid Function", + Message: formatter.F(`[SynchronizedBeforeSuite] node must be passed `+mustGet+` for its first function. +You passed {{bold}}%s{{/}} instead.`, t), + CodeLocation: cl, + DocLink: "node-decorators-overview", + } +} + +func (g ginkgoErrors) InvalidBodyTypeForSynchronizedBeforeSuiteAllProcs(t reflect.Type, cl CodeLocation) error { + mustGet := "{{bold}}func(){{/}}, {{bold}}func(ctx SpecContext){{/}}, or {{bold}}func(ctx context.Context){{/}}, {{bold}}func([]byte){{/}}, {{bold}}func(ctx SpecContext, []byte){{/}}, or {{bold}}func(ctx context.Context, []byte){{/}}" + return GinkgoError{ + Heading: "Invalid Function", + Message: formatter.F(`[SynchronizedBeforeSuite] node must be passed `+mustGet+` for its second function. +You passed {{bold}}%s{{/}} instead.`, t), + CodeLocation: cl, + DocLink: "node-decorators-overview", + } +} + func (g ginkgoErrors) MultipleBodyFunctions(cl CodeLocation, nodeType NodeType) error { return GinkgoError{ Heading: "Multiple Functions", - Message: formatter.F(`[%s] node must be passed a single {{bold}}func(){{/}} - but more than one was passed in.`, nodeType), + Message: formatter.F(`[%s] node must be passed a single function - but more than one was passed in.`, nodeType), CodeLocation: cl, DocLink: "node-decorators-overview", } @@ -211,12 +255,30 @@ func (g ginkgoErrors) MultipleBodyFunctions(cl CodeLocation, nodeType NodeType) func (g ginkgoErrors) MissingBodyFunction(cl CodeLocation, nodeType NodeType) error { return GinkgoError{ Heading: "Missing Functions", - Message: formatter.F(`[%s] node must be passed a single {{bold}}func(){{/}} - but none was passed in.`, nodeType), + Message: formatter.F(`[%s] node must be passed a single function - but none was passed in.`, nodeType), CodeLocation: cl, DocLink: "node-decorators-overview", } } +func (g ginkgoErrors) InvalidTimeoutOrGracePeriodForNonContextNode(cl CodeLocation, nodeType NodeType) error { + return GinkgoError{ + Heading: "Invalid NodeTimeout SpecTimeout, or GracePeriod", + Message: formatter.F(`[%s] was passed NodeTimeout, SpecTimeout, or GracePeriod but does not have a callback that accepts a {{bold}}SpecContext{{/}} or {{bold}}context.Context{{/}}. You must accept a context to enable timeouts and grace periods`, nodeType), + CodeLocation: cl, + DocLink: "spec-timeouts-and-interruptible-nodes", + } +} + +func (g ginkgoErrors) InvalidTimeoutOrGracePeriodForNonContextCleanupNode(cl CodeLocation) error { + return GinkgoError{ + Heading: "Invalid NodeTimeout SpecTimeout, or GracePeriod", + Message: formatter.F(`[DeferCleanup] was passed NodeTimeout or GracePeriod but does not have a callback that accepts a {{bold}}SpecContext{{/}} or {{bold}}context.Context{{/}}. You must accept a context to enable timeouts and grace periods`), + CodeLocation: cl, + DocLink: "spec-timeouts-and-interruptible-nodes", + } +} + /* Ordered Container errors */ func (g ginkgoErrors) InvalidSerialNodeInNonSerialOrderedContainer(cl CodeLocation, nodeType NodeType) error { return GinkgoError{ @@ -236,6 +298,15 @@ func (g ginkgoErrors) SetupNodeNotInOrderedContainer(cl CodeLocation, nodeType N } } +func (g ginkgoErrors) InvalidContinueOnFailureDecoration(cl CodeLocation) error { + return GinkgoError{ + Heading: "ContinueOnFailure not decorating an outermost Ordered Container", + Message: "ContinueOnFailure can only decorate an Ordered container, and this Ordered container must be the outermost Ordered container.", + CodeLocation: cl, + DocLink: "ordered-containers", + } +} + /* DeferCleanup errors */ func (g ginkgoErrors) DeferCleanupInvalidFunction(cl CodeLocation) error { return GinkgoError{ @@ -258,7 +329,7 @@ func (g ginkgoErrors) PushingCleanupNodeDuringTreeConstruction(cl CodeLocation) func (g ginkgoErrors) PushingCleanupInReportingNode(cl CodeLocation, nodeType NodeType) error { return GinkgoError{ Heading: fmt.Sprintf("DeferCleanup cannot be called in %s", nodeType), - Message: "Please inline your cleanup code - Ginkgo won't run cleanup code after a ReportAfterEach or ReportAfterSuite.", + Message: "Please inline your cleanup code - Ginkgo won't run cleanup code after a Reporting node.", CodeLocation: cl, DocLink: "cleaning-up-our-cleanup-code-defercleanup", } @@ -380,6 +451,15 @@ func (g ginkgoErrors) InvalidEntryDescription(cl CodeLocation) error { } } +func (g ginkgoErrors) MissingParametersForTableFunction(cl CodeLocation) error { + return GinkgoError{ + Heading: fmt.Sprintf("No parameters have been passed to the Table Function"), + Message: fmt.Sprintf("The Table Function expected at least 1 parameter"), + CodeLocation: cl, + DocLink: "table-specs", + } +} + func (g ginkgoErrors) IncorrectParameterTypeForTable(i int, name string, cl CodeLocation) error { return GinkgoError{ Heading: "DescribeTable passed incorrect parameter type", @@ -498,6 +578,13 @@ func (g ginkgoErrors) DryRunInParallelConfiguration() error { } } +func (g ginkgoErrors) GracePeriodCannotBeZero() error { + return GinkgoError{ + Heading: "Ginkgo requires a positive --grace-period.", + Message: "Please set --grace-period to a positive duration. The default is 30s.", + } +} + func (g ginkgoErrors) ConflictingVerbosityConfiguration() error { return GinkgoError{ Heading: "Conflicting reporter verbosity settings.", diff --git a/vendor/github.com/onsi/ginkgo/v2/types/label_filter.go b/vendor/github.com/onsi/ginkgo/v2/types/label_filter.go index 0403f9e63..b0d3b651e 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/label_filter.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/label_filter.go @@ -272,12 +272,23 @@ func tokenize(input string) func() (*treeNode, error) { } } +func MustParseLabelFilter(input string) LabelFilter { + filter, err := ParseLabelFilter(input) + if err != nil { + panic(err) + } + return filter +} + func ParseLabelFilter(input string) (LabelFilter, error) { if DEBUG_LABEL_FILTER_PARSING { fmt.Println("\n==============") fmt.Println("Input: ", input) fmt.Print("Tokens: ") } + if input == "" { + return func(_ []string) bool { return true }, nil + } nextToken := tokenize(input) root := &treeNode{token: lfTokenRoot} diff --git a/vendor/github.com/onsi/ginkgo/v2/types/report_entry.go b/vendor/github.com/onsi/ginkgo/v2/types/report_entry.go index 798bedc03..7b1524b52 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/report_entry.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/report_entry.go @@ -6,8 +6,8 @@ import ( "time" ) -//ReportEntryValue wraps a report entry's value ensuring it can be encoded and decoded safely into reports -//and across the network connection when running in parallel +// ReportEntryValue wraps a report entry's value ensuring it can be encoded and decoded safely into reports +// and across the network connection when running in parallel type ReportEntryValue struct { raw interface{} //unexported to prevent gob from freaking out about unregistered structs AsJSON string @@ -85,10 +85,12 @@ func (rev *ReportEntryValue) GobDecode(data []byte) error { type ReportEntry struct { // Visibility captures the visibility policy for this ReportEntry Visibility ReportEntryVisibility - // Time captures the time the AddReportEntry was called - Time time.Time // Location captures the location of the AddReportEntry call Location CodeLocation + + Time time.Time //need this for backwards compatibility + TimelineLocation TimelineLocation + // Name captures the name of this report Name string // Value captures the (optional) object passed into AddReportEntry - this can be @@ -120,7 +122,9 @@ func (entry ReportEntry) GetRawValue() interface{} { return entry.Value.GetRawValue() } - +func (entry ReportEntry) GetTimelineLocation() TimelineLocation { + return entry.TimelineLocation +} type ReportEntries []ReportEntry diff --git a/vendor/github.com/onsi/ginkgo/v2/types/types.go b/vendor/github.com/onsi/ginkgo/v2/types/types.go index aec9062ed..d048a8ada 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/types.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/types.go @@ -2,6 +2,8 @@ package types import ( "encoding/json" + "fmt" + "sort" "strings" "time" ) @@ -56,19 +58,20 @@ type Report struct { SuiteConfig SuiteConfig //SpecReports is a list of all SpecReports generated by this test run + //It is empty when the SuiteReport is provided to ReportBeforeSuite SpecReports SpecReports } -//PreRunStats contains a set of stats captured before the test run begins. This is primarily used -//by Ginkgo's reporter to tell the user how many specs are in the current suite (PreRunStats.TotalSpecs) -//and how many it intends to run (PreRunStats.SpecsThatWillRun) after applying any relevant focus or skip filters. +// PreRunStats contains a set of stats captured before the test run begins. This is primarily used +// by Ginkgo's reporter to tell the user how many specs are in the current suite (PreRunStats.TotalSpecs) +// and how many it intends to run (PreRunStats.SpecsThatWillRun) after applying any relevant focus or skip filters. type PreRunStats struct { TotalSpecs int SpecsThatWillRun int } -//Add is ued by Ginkgo's parallel aggregation mechanisms to combine test run reports form individual parallel processes -//to form a complete final report. +// Add is used by Ginkgo's parallel aggregation mechanisms to combine test run reports form individual parallel processes +// to form a complete final report. func (report Report) Add(other Report) Report { report.SuiteSucceeded = report.SuiteSucceeded && other.SuiteSucceeded @@ -147,14 +150,24 @@ type SpecReport struct { // ParallelProcess captures the parallel process that this spec ran on ParallelProcess int + // RunningInParallel captures whether this spec is part of a suite that ran in parallel + RunningInParallel bool + //Failure is populated if a spec has failed, panicked, been interrupted, or skipped by the user (e.g. calling Skip()) //It includes detailed information about the Failure Failure Failure - // NumAttempts captures the number of times this Spec was run. Flakey specs can be retried with - // ginkgo --flake-attempts=N + // NumAttempts captures the number of times this Spec was run. + // Flakey specs can be retried with ginkgo --flake-attempts=N or the use of the FlakeAttempts decorator. + // Repeated specs can be retried with the use of the MustPassRepeatedly decorator NumAttempts int + // MaxFlakeAttempts captures whether the spec has been retried with ginkgo --flake-attempts=N or the use of the FlakeAttempts decorator. + MaxFlakeAttempts int + + // MaxMustPassRepeatedly captures whether the spec has the MustPassRepeatedly decorator + MaxMustPassRepeatedly int + // CapturedGinkgoWriterOutput contains text printed to the GinkgoWriter CapturedGinkgoWriterOutput string @@ -168,6 +181,12 @@ type SpecReport struct { // ProgressReports contains any progress reports generated during this spec. These can either be manually triggered, or automatically generated by Ginkgo via the PollProgressAfter() decorator ProgressReports []ProgressReport + + // AdditionalFailures contains any failures that occurred after the initial spec failure. These typically occur in cleanup nodes after the initial failure and are only emitted when running in verbose mode. + AdditionalFailures []AdditionalFailure + + // SpecEvents capture additional events that occur during the spec run + SpecEvents SpecEvents } func (report SpecReport) MarshalJSON() ([]byte, error) { @@ -187,10 +206,14 @@ func (report SpecReport) MarshalJSON() ([]byte, error) { ParallelProcess int Failure *Failure `json:",omitempty"` NumAttempts int - CapturedGinkgoWriterOutput string `json:",omitempty"` - CapturedStdOutErr string `json:",omitempty"` - ReportEntries ReportEntries `json:",omitempty"` - ProgressReports []ProgressReport `json:",omitempty"` + MaxFlakeAttempts int + MaxMustPassRepeatedly int + CapturedGinkgoWriterOutput string `json:",omitempty"` + CapturedStdOutErr string `json:",omitempty"` + ReportEntries ReportEntries `json:",omitempty"` + ProgressReports []ProgressReport `json:",omitempty"` + AdditionalFailures []AdditionalFailure `json:",omitempty"` + SpecEvents SpecEvents `json:",omitempty"` }{ ContainerHierarchyTexts: report.ContainerHierarchyTexts, ContainerHierarchyLocations: report.ContainerHierarchyLocations, @@ -207,6 +230,8 @@ func (report SpecReport) MarshalJSON() ([]byte, error) { Failure: nil, ReportEntries: nil, NumAttempts: report.NumAttempts, + MaxFlakeAttempts: report.MaxFlakeAttempts, + MaxMustPassRepeatedly: report.MaxMustPassRepeatedly, CapturedGinkgoWriterOutput: report.CapturedGinkgoWriterOutput, CapturedStdOutErr: report.CapturedStdOutErr, } @@ -220,6 +245,12 @@ func (report SpecReport) MarshalJSON() ([]byte, error) { if len(report.ProgressReports) > 0 { out.ProgressReports = report.ProgressReports } + if len(report.AdditionalFailures) > 0 { + out.AdditionalFailures = report.AdditionalFailures + } + if len(report.SpecEvents) > 0 { + out.SpecEvents = report.SpecEvents + } return json.Marshal(out) } @@ -237,13 +268,13 @@ func (report SpecReport) CombinedOutput() string { return report.CapturedStdOutErr + "\n" + report.CapturedGinkgoWriterOutput } -//Failed returns true if report.State is one of the SpecStateFailureStates +// Failed returns true if report.State is one of the SpecStateFailureStates // (SpecStateFailed, SpecStatePanicked, SpecStateinterrupted, SpecStateAborted) func (report SpecReport) Failed() bool { return report.State.Is(SpecStateFailureStates) } -//FullText returns a concatenation of all the report.ContainerHierarchyTexts and report.LeafNodeText +// FullText returns a concatenation of all the report.ContainerHierarchyTexts and report.LeafNodeText func (report SpecReport) FullText() string { texts := []string{} texts = append(texts, report.ContainerHierarchyTexts...) @@ -253,7 +284,7 @@ func (report SpecReport) FullText() string { return strings.Join(texts, " ") } -//Labels returns a deduped set of all the spec's Labels. +// Labels returns a deduped set of all the spec's Labels. func (report SpecReport) Labels() []string { out := []string{} seen := map[string]bool{} @@ -275,7 +306,7 @@ func (report SpecReport) Labels() []string { return out } -//MatchesLabelFilter returns true if the spec satisfies the passed in label filter query +// MatchesLabelFilter returns true if the spec satisfies the passed in label filter query func (report SpecReport) MatchesLabelFilter(query string) (bool, error) { filter, err := ParseLabelFilter(query) if err != nil { @@ -284,29 +315,54 @@ func (report SpecReport) MatchesLabelFilter(query string) (bool, error) { return filter(report.Labels()), nil } -//FileName() returns the name of the file containing the spec +// FileName() returns the name of the file containing the spec func (report SpecReport) FileName() string { return report.LeafNodeLocation.FileName } -//LineNumber() returns the line number of the leaf node +// LineNumber() returns the line number of the leaf node func (report SpecReport) LineNumber() int { return report.LeafNodeLocation.LineNumber } -//FailureMessage() returns the failure message (or empty string if the test hasn't failed) +// FailureMessage() returns the failure message (or empty string if the test hasn't failed) func (report SpecReport) FailureMessage() string { return report.Failure.Message } -//FailureLocation() returns the location of the failure (or an empty CodeLocation if the test hasn't failed) +// FailureLocation() returns the location of the failure (or an empty CodeLocation if the test hasn't failed) func (report SpecReport) FailureLocation() CodeLocation { return report.Failure.Location } +// Timeline() returns a timeline view of the report +func (report SpecReport) Timeline() Timeline { + timeline := Timeline{} + if !report.Failure.IsZero() { + timeline = append(timeline, report.Failure) + if report.Failure.AdditionalFailure != nil { + timeline = append(timeline, *(report.Failure.AdditionalFailure)) + } + } + for _, additionalFailure := range report.AdditionalFailures { + timeline = append(timeline, additionalFailure) + } + for _, reportEntry := range report.ReportEntries { + timeline = append(timeline, reportEntry) + } + for _, progressReport := range report.ProgressReports { + timeline = append(timeline, progressReport) + } + for _, specEvent := range report.SpecEvents { + timeline = append(timeline, specEvent) + } + sort.Sort(timeline) + return timeline +} + type SpecReports []SpecReport -//WithLeafNodeType returns the subset of SpecReports with LeafNodeType matching one of the requested NodeTypes +// WithLeafNodeType returns the subset of SpecReports with LeafNodeType matching one of the requested NodeTypes func (reports SpecReports) WithLeafNodeType(nodeTypes NodeType) SpecReports { count := 0 for i := range reports { @@ -326,7 +382,7 @@ func (reports SpecReports) WithLeafNodeType(nodeTypes NodeType) SpecReports { return out } -//WithState returns the subset of SpecReports with State matching one of the requested SpecStates +// WithState returns the subset of SpecReports with State matching one of the requested SpecStates func (reports SpecReports) WithState(states SpecState) SpecReports { count := 0 for i := range reports { @@ -345,7 +401,7 @@ func (reports SpecReports) WithState(states SpecState) SpecReports { return out } -//CountWithState returns the number of SpecReports with State matching one of the requested SpecStates +// CountWithState returns the number of SpecReports with State matching one of the requested SpecStates func (reports SpecReports) CountWithState(states SpecState) int { n := 0 for i := range reports { @@ -356,17 +412,75 @@ func (reports SpecReports) CountWithState(states SpecState) int { return n } -//CountWithState returns the number of SpecReports that passed after multiple attempts +// If the Spec passes, CountOfFlakedSpecs returns the number of SpecReports that failed after multiple attempts. func (reports SpecReports) CountOfFlakedSpecs() int { n := 0 for i := range reports { - if reports[i].State.Is(SpecStatePassed) && reports[i].NumAttempts > 1 { + if reports[i].MaxFlakeAttempts > 1 && reports[i].State.Is(SpecStatePassed) && reports[i].NumAttempts > 1 { + n += 1 + } + } + return n +} + +// If the Spec fails, CountOfRepeatedSpecs returns the number of SpecReports that passed after multiple attempts +func (reports SpecReports) CountOfRepeatedSpecs() int { + n := 0 + for i := range reports { + if reports[i].MaxMustPassRepeatedly > 1 && reports[i].State.Is(SpecStateFailureStates) && reports[i].NumAttempts > 1 { n += 1 } } return n } +// TimelineLocation captures the location of an event in the spec's timeline +type TimelineLocation struct { + //Offset is the offset (in bytes) of the event relative to the GinkgoWriter stream + Offset int `json:",omitempty"` + + //Order is the order of the event with respect to other events. The absolute value of Order + //is irrelevant. All that matters is that an event with a lower Order occurs before ane vent with a higher Order + Order int `json:",omitempty"` + + Time time.Time +} + +// TimelineEvent represent an event on the timeline +// consumers of Timeline will need to check the concrete type of each entry to determine how to handle it +type TimelineEvent interface { + GetTimelineLocation() TimelineLocation +} + +type Timeline []TimelineEvent + +func (t Timeline) Len() int { return len(t) } +func (t Timeline) Less(i, j int) bool { + return t[i].GetTimelineLocation().Order < t[j].GetTimelineLocation().Order +} +func (t Timeline) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t Timeline) WithoutHiddenReportEntries() Timeline { + out := Timeline{} + for _, event := range t { + if reportEntry, isReportEntry := event.(ReportEntry); isReportEntry && reportEntry.Visibility == ReportEntryVisibilityNever { + continue + } + out = append(out, event) + } + return out +} + +func (t Timeline) WithoutVeryVerboseSpecEvents() Timeline { + out := Timeline{} + for _, event := range t { + if specEvent, isSpecEvent := event.(SpecEvent); isSpecEvent && specEvent.IsOnlyVisibleAtVeryVerbose() { + continue + } + out = append(out, event) + } + return out +} + // Failure captures failure information for an individual test type Failure struct { // Message - the failure message passed into Fail(...). When using a matcher library @@ -379,6 +493,8 @@ type Failure struct { // This CodeLocation will include a fully-populated StackTrace Location CodeLocation + TimelineLocation TimelineLocation + // ForwardedPanic - if the failure represents a captured panic (i.e. Summary.State == SpecStatePanicked) // then ForwardedPanic will be populated with a string representation of the captured panic. ForwardedPanic string `json:",omitempty"` @@ -391,19 +507,29 @@ type Failure struct { // FailureNodeType will contain the NodeType of the node in which the failure occurred. // FailureNodeLocation will contain the CodeLocation of the node in which the failure occurred. // If populated, FailureNodeContainerIndex will be the index into SpecReport.ContainerHierarchyTexts and SpecReport.ContainerHierarchyLocations that represents the parent container of the node in which the failure occurred. - FailureNodeContext FailureNodeContext - FailureNodeType NodeType - FailureNodeLocation CodeLocation - FailureNodeContainerIndex int + FailureNodeContext FailureNodeContext `json:",omitempty"` + + FailureNodeType NodeType `json:",omitempty"` + + FailureNodeLocation CodeLocation `json:",omitempty"` + + FailureNodeContainerIndex int `json:",omitempty"` //ProgressReport is populated if the spec was interrupted or timed out - ProgressReport ProgressReport + ProgressReport ProgressReport `json:",omitempty"` + + //AdditionalFailure is non-nil if a follow-on failure occurred within the same node after the primary failure. This only happens when a node has timed out or been interrupted. In such cases the AdditionalFailure can include information about where/why the spec was stuck. + AdditionalFailure *AdditionalFailure `json:",omitempty"` } func (f Failure) IsZero() bool { return f.Message == "" && (f.Location == CodeLocation{}) } +func (f Failure) GetTimelineLocation() TimelineLocation { + return f.TimelineLocation +} + // FailureNodeContext captures the location context for the node containing the failing line of code type FailureNodeContext uint @@ -434,6 +560,18 @@ func (fnc FailureNodeContext) MarshalJSON() ([]byte, error) { return fncEnumSupport.MarshJSON(uint(fnc)) } +// AdditionalFailure capturs any additional failures that occur after the initial failure of a psec +// these typically occur in clean up nodes after the spec has failed. +// We can't simply use Failure as we want to track the SpecState to know what kind of failure this is +type AdditionalFailure struct { + State SpecState + Failure Failure +} + +func (f AdditionalFailure) GetTimelineLocation() TimelineLocation { + return f.Failure.TimelineLocation +} + // SpecState captures the state of a spec // To determine if a given `state` represents a failure state, use `state.Is(SpecStateFailureStates)` type SpecState uint @@ -448,6 +586,7 @@ const ( SpecStateAborted SpecStatePanicked SpecStateInterrupted + SpecStateTimedout ) var ssEnumSupport = NewEnumSupport(map[uint]string{ @@ -459,11 +598,15 @@ var ssEnumSupport = NewEnumSupport(map[uint]string{ uint(SpecStateAborted): "aborted", uint(SpecStatePanicked): "panicked", uint(SpecStateInterrupted): "interrupted", + uint(SpecStateTimedout): "timedout", }) func (ss SpecState) String() string { return ssEnumSupport.String(uint(ss)) } +func (ss SpecState) GomegaString() string { + return ssEnumSupport.String(uint(ss)) +} func (ss *SpecState) UnmarshalJSON(b []byte) error { out, err := ssEnumSupport.UnmarshJSON(b) *ss = SpecState(out) @@ -473,7 +616,7 @@ func (ss SpecState) MarshalJSON() ([]byte, error) { return ssEnumSupport.MarshJSON(uint(ss)) } -var SpecStateFailureStates = SpecStateFailed | SpecStateAborted | SpecStatePanicked | SpecStateInterrupted +var SpecStateFailureStates = SpecStateFailed | SpecStateTimedout | SpecStateAborted | SpecStatePanicked | SpecStateInterrupted func (ss SpecState) Is(states SpecState) bool { return ss&states != 0 @@ -481,35 +624,40 @@ func (ss SpecState) Is(states SpecState) bool { // ProgressReport captures the progress of the current spec. It is, effectively, a structured Ginkgo-aware stack trace type ProgressReport struct { - ParallelProcess int - RunningInParallel bool + Message string `json:",omitempty"` + ParallelProcess int `json:",omitempty"` + RunningInParallel bool `json:",omitempty"` - Time time.Time + ContainerHierarchyTexts []string `json:",omitempty"` + LeafNodeText string `json:",omitempty"` + LeafNodeLocation CodeLocation `json:",omitempty"` + SpecStartTime time.Time `json:",omitempty"` - ContainerHierarchyTexts []string - LeafNodeText string - LeafNodeLocation CodeLocation - SpecStartTime time.Time + CurrentNodeType NodeType `json:",omitempty"` + CurrentNodeText string `json:",omitempty"` + CurrentNodeLocation CodeLocation `json:",omitempty"` + CurrentNodeStartTime time.Time `json:",omitempty"` - CurrentNodeType NodeType - CurrentNodeText string - CurrentNodeLocation CodeLocation - CurrentNodeStartTime time.Time + CurrentStepText string `json:",omitempty"` + CurrentStepLocation CodeLocation `json:",omitempty"` + CurrentStepStartTime time.Time `json:",omitempty"` - CurrentStepText string - CurrentStepLocation CodeLocation - CurrentStepStartTime time.Time + AdditionalReports []string `json:",omitempty"` - CapturedGinkgoWriterOutput string `json:",omitempty"` - GinkgoWriterOffset int + CapturedGinkgoWriterOutput string `json:",omitempty"` + TimelineLocation TimelineLocation `json:",omitempty"` - Goroutines []Goroutine + Goroutines []Goroutine `json:",omitempty"` } func (pr ProgressReport) IsZero() bool { return pr.CurrentNodeType == NodeTypeInvalid } +func (pr ProgressReport) Time() time.Time { + return pr.TimelineLocation.Time +} + func (pr ProgressReport) SpecGoroutine() Goroutine { for _, goroutine := range pr.Goroutines { if goroutine.IsSpecGoroutine { @@ -547,6 +695,22 @@ func (pr ProgressReport) WithoutCapturedGinkgoWriterOutput() ProgressReport { return out } +func (pr ProgressReport) WithoutOtherGoroutines() ProgressReport { + out := pr + filteredGoroutines := []Goroutine{} + for _, goroutine := range pr.Goroutines { + if goroutine.IsSpecGoroutine || goroutine.HasHighlights() { + filteredGoroutines = append(filteredGoroutines, goroutine) + } + } + out.Goroutines = filteredGoroutines + return out +} + +func (pr ProgressReport) GetTimelineLocation() TimelineLocation { + return pr.TimelineLocation +} + type Goroutine struct { ID uint64 State string @@ -601,6 +765,7 @@ const ( NodeTypeReportBeforeEach NodeTypeReportAfterEach + NodeTypeReportBeforeSuite NodeTypeReportAfterSuite NodeTypeCleanupInvalid @@ -610,7 +775,9 @@ const ( ) var NodeTypesForContainerAndIt = NodeTypeContainer | NodeTypeIt -var NodeTypesForSuiteLevelNodes = NodeTypeBeforeSuite | NodeTypeSynchronizedBeforeSuite | NodeTypeAfterSuite | NodeTypeSynchronizedAfterSuite | NodeTypeReportAfterSuite | NodeTypeCleanupAfterSuite +var NodeTypesForSuiteLevelNodes = NodeTypeBeforeSuite | NodeTypeSynchronizedBeforeSuite | NodeTypeAfterSuite | NodeTypeSynchronizedAfterSuite | NodeTypeReportBeforeSuite | NodeTypeReportAfterSuite | NodeTypeCleanupAfterSuite +var NodeTypesAllowedDuringCleanupInterrupt = NodeTypeAfterEach | NodeTypeJustAfterEach | NodeTypeAfterAll | NodeTypeAfterSuite | NodeTypeSynchronizedAfterSuite | NodeTypeCleanupAfterEach | NodeTypeCleanupAfterAll | NodeTypeCleanupAfterSuite +var NodeTypesAllowedDuringReportInterrupt = NodeTypeReportBeforeEach | NodeTypeReportAfterEach | NodeTypeReportBeforeSuite | NodeTypeReportAfterSuite var ntEnumSupport = NewEnumSupport(map[uint]string{ uint(NodeTypeInvalid): "INVALID NODE TYPE", @@ -628,9 +795,10 @@ var ntEnumSupport = NewEnumSupport(map[uint]string{ uint(NodeTypeSynchronizedAfterSuite): "SynchronizedAfterSuite", uint(NodeTypeReportBeforeEach): "ReportBeforeEach", uint(NodeTypeReportAfterEach): "ReportAfterEach", + uint(NodeTypeReportBeforeSuite): "ReportBeforeSuite", uint(NodeTypeReportAfterSuite): "ReportAfterSuite", - uint(NodeTypeCleanupInvalid): "INVALID CLEANUP NODE", - uint(NodeTypeCleanupAfterEach): "DeferCleanup", + uint(NodeTypeCleanupInvalid): "DeferCleanup", + uint(NodeTypeCleanupAfterEach): "DeferCleanup (Each)", uint(NodeTypeCleanupAfterAll): "DeferCleanup (All)", uint(NodeTypeCleanupAfterSuite): "DeferCleanup (Suite)", }) @@ -650,3 +818,99 @@ func (nt NodeType) MarshalJSON() ([]byte, error) { func (nt NodeType) Is(nodeTypes NodeType) bool { return nt&nodeTypes != 0 } + +/* +SpecEvent captures a vareity of events that can occur when specs run. See SpecEventType for the list of available events. +*/ +type SpecEvent struct { + SpecEventType SpecEventType + + CodeLocation CodeLocation + TimelineLocation TimelineLocation + + Message string `json:",omitempty"` + Duration time.Duration `json:",omitempty"` + NodeType NodeType `json:",omitempty"` + Attempt int `json:",omitempty"` +} + +func (se SpecEvent) GetTimelineLocation() TimelineLocation { + return se.TimelineLocation +} + +func (se SpecEvent) IsOnlyVisibleAtVeryVerbose() bool { + return se.SpecEventType.Is(SpecEventByEnd | SpecEventNodeStart | SpecEventNodeEnd) +} + +func (se SpecEvent) GomegaString() string { + out := &strings.Builder{} + out.WriteString("[" + se.SpecEventType.String() + " SpecEvent] ") + if se.Message != "" { + out.WriteString("Message=") + out.WriteString(`"` + se.Message + `",`) + } + if se.Duration != 0 { + out.WriteString("Duration=" + se.Duration.String() + ",") + } + if se.NodeType != NodeTypeInvalid { + out.WriteString("NodeType=" + se.NodeType.String() + ",") + } + if se.Attempt != 0 { + out.WriteString(fmt.Sprintf("Attempt=%d", se.Attempt) + ",") + } + out.WriteString("CL=" + se.CodeLocation.String() + ",") + out.WriteString(fmt.Sprintf("TL.Offset=%d", se.TimelineLocation.Offset)) + + return out.String() +} + +type SpecEvents []SpecEvent + +func (se SpecEvents) WithType(seType SpecEventType) SpecEvents { + out := SpecEvents{} + for _, event := range se { + if event.SpecEventType.Is(seType) { + out = append(out, event) + } + } + return out +} + +type SpecEventType uint + +const ( + SpecEventInvalid SpecEventType = 0 + + SpecEventByStart SpecEventType = 1 << iota + SpecEventByEnd + SpecEventNodeStart + SpecEventNodeEnd + SpecEventSpecRepeat + SpecEventSpecRetry +) + +var seEnumSupport = NewEnumSupport(map[uint]string{ + uint(SpecEventInvalid): "INVALID SPEC EVENT", + uint(SpecEventByStart): "By", + uint(SpecEventByEnd): "By (End)", + uint(SpecEventNodeStart): "Node", + uint(SpecEventNodeEnd): "Node (End)", + uint(SpecEventSpecRepeat): "Repeat", + uint(SpecEventSpecRetry): "Retry", +}) + +func (se SpecEventType) String() string { + return seEnumSupport.String(uint(se)) +} +func (se *SpecEventType) UnmarshalJSON(b []byte) error { + out, err := seEnumSupport.UnmarshJSON(b) + *se = SpecEventType(out) + return err +} +func (se SpecEventType) MarshalJSON() ([]byte, error) { + return seEnumSupport.MarshJSON(uint(se)) +} + +func (se SpecEventType) Is(specEventTypes SpecEventType) bool { + return se&specEventTypes != 0 +} diff --git a/vendor/github.com/onsi/ginkgo/v2/types/version.go b/vendor/github.com/onsi/ginkgo/v2/types/version.go index 3974fdc26..d11d08446 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/version.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/version.go @@ -1,3 +1,3 @@ package types -const VERSION = "2.2.0" +const VERSION = "2.8.4" diff --git a/vendor/github.com/onsi/gomega/.gitignore b/vendor/github.com/onsi/gomega/.gitignore index 720c13cba..52266eae1 100644 --- a/vendor/github.com/onsi/gomega/.gitignore +++ b/vendor/github.com/onsi/gomega/.gitignore @@ -3,3 +3,5 @@ . .idea gomega.iml +TODO.md +.vscode \ No newline at end of file diff --git a/vendor/github.com/onsi/gomega/CHANGELOG.md b/vendor/github.com/onsi/gomega/CHANGELOG.md index 06065866b..99ea94078 100644 --- a/vendor/github.com/onsi/gomega/CHANGELOG.md +++ b/vendor/github.com/onsi/gomega/CHANGELOG.md @@ -1,3 +1,154 @@ +## 1.27.1 + +### Maintenance + +- Bump golang.org/x/net from 0.6.0 to 0.7.0 (#640) [bc686cd] + +## 1.27.0 + +### Features +- Add HaveExactElements matcher (#634) [9d50783] +- update Gomega docs to discuss GinkgoHelper() [be32774] + +### Maintenance +- Bump github.com/onsi/ginkgo/v2 from 2.8.0 to 2.8.1 (#639) [296a68b] +- Bump golang.org/x/net from 0.5.0 to 0.6.0 (#638) [c2b098b] +- Bump github-pages from 227 to 228 in /docs (#636) [a9069ab] +- test: update matrix for Go 1.20 (#635) [6bd25c8] +- Bump github.com/onsi/ginkgo/v2 from 2.7.0 to 2.8.0 (#631) [5445f8b] +- Bump webrick from 1.7.0 to 1.8.1 in /docs (#630) [03e93bb] +- codeql: add ruby language (#626) [63c7d21] +- dependabot: add bundler package-ecosystem for docs (#625) [d92f963] + +## 1.26.0 + +### Features +- When a polled function returns an error, keep track of the actual and report on the matcher state of the last non-errored actual [21f3090] +- improve eventually failure message output [c530fb3] + +### Fixes +- fix several documentation spelling issues [e2eff1f] + + +## 1.25.0 + +### Features +- add `MustPassRepeatedly(int)` to asyncAssertion (#619) [4509f72] +- compare unwrapped errors using DeepEqual (#617) [aaeaa5d] + +### Maintenance +- Bump golang.org/x/net from 0.4.0 to 0.5.0 (#614) [c7cfea4] +- Bump github.com/onsi/ginkgo/v2 from 2.6.1 to 2.7.0 (#615) [71b8adb] +- Docs: Fix typo "MUltiple" -> "Multiple" (#616) [9351dda] +- clean up go.sum [cd1dc1d] + +## 1.24.2 + +### Fixes +- Correctly handle assertion failure panics for eventually/consistnetly "g Gomega"s in a goroutine [78f1660] +- docs:Fix typo "you an" -> "you can" (#607) [3187c1f] +- fixes issue #600 (#606) [808d192] + +### Maintenance +- Bump golang.org/x/net from 0.2.0 to 0.4.0 (#611) [6ebc0bf] +- Bump nokogiri from 1.13.9 to 1.13.10 in /docs (#612) [258cfc8] +- Bump github.com/onsi/ginkgo/v2 from 2.5.0 to 2.5.1 (#609) [e6c3eb9] + +## 1.24.1 + +### Fixes +- maintain backward compatibility for Eventually and Consisntetly's signatures [4c7df5e] +- fix small typo (#601) [ea0ebe6] + +### Maintenance +- Bump golang.org/x/net from 0.1.0 to 0.2.0 (#603) [1ba8372] +- Bump github.com/onsi/ginkgo/v2 from 2.4.0 to 2.5.0 (#602) [f9426cb] +- fix label-filter in test.yml [d795db6] +- stop running flakey tests and rely on external network dependencies in CI [7133290] + +## 1.24.0 + +### Features + +Introducting [gcustom](https://onsi.github.io/gomega/#gcustom-a-convenient-mechanism-for-buildling-custom-matchers) - a convenient mechanism for building custom matchers. + +This is an RC release for `gcustom`. The external API may be tweaked in response to feedback however it is expected to remain mostly stable. + +### Maintenance + +- Update BeComparableTo documentation [756eaa0] + +## 1.23.0 + +### Features +- Custom formatting on a per-type basis can be provided using `format.RegisterCustomFormatter()` -- see the docs [here](https://onsi.github.io/gomega/#adjusting-output) + +- Substantial improvement have been made to `StopTrying()`: + - Users can now use `StopTrying().Wrap(err)` to wrap errors and `StopTrying().Attach(description, object)` to attach arbitrary objects to the `StopTrying()` error + - `StopTrying()` is now always interpreted as a failure. If you are an early adopter of `StopTrying()` you may need to change your code as the prior version would match against the returned value even if `StopTrying()` was returned. Going forward the `StopTrying()` api should remain stable. + - `StopTrying()` and `StopTrying().Now()` can both be used in matchers - not just polled functions. + +- `TryAgainAfter(duration)` is used like `StopTrying()` but instructs `Eventually` and `Consistently` that the poll should be tried again after the specified duration. This allows you to dynamically adjust the polling duration. + +- `ctx` can now be passed-in as the first argument to `Eventually` and `Consistently`. + +## Maintenance + +- Bump github.com/onsi/ginkgo/v2 from 2.3.0 to 2.3.1 (#597) [afed901] +- Bump nokogiri from 1.13.8 to 1.13.9 in /docs (#599) [7c691b3] +- Bump github.com/google/go-cmp from 0.5.8 to 0.5.9 (#587) [ff22665] + +## 1.22.1 + +## Fixes +- When passed a context and no explicit timeout, Eventually will only timeout when the context is cancelled [e5105cf] +- Allow StopTrying() to be wrapped [bf3cba9] + +## Maintenance +- bump to ginkgo v2.3.0 [c5d5c39] + +## 1.22.0 + +### Features + +Several improvements have been made to `Eventually` and `Consistently` in this and the most recent releases: + +- Eventually and Consistently can take a context.Context [65c01bc] + This enables integration with Ginkgo 2.3.0's interruptible nodes and node timeouts. +- Eventually and Consistently that are passed a SpecContext can provide reports when an interrupt occurs [0d063c9] +- Eventually/Consistently will forward an attached context to functions that ask for one [e2091c5] +- Eventually/Consistently supports passing arguments to functions via WithArguments() [a2dc7c3] +- Eventually and Consistently can now be stopped early with StopTrying(message) and StopTrying(message).Now() [52976bb] + +These improvements are all documented in [Gomega's docs](https://onsi.github.io/gomega/#making-asynchronous-assertions) + +## Fixes + +## Maintenance + +## 1.21.1 + +### Features +- Eventually and Consistently that are passed a SpecContext can provide reports when an interrupt occurs [0d063c9] + +## 1.21.0 + +### Features +- Eventually and Consistently can take a context.Context [65c01bc] + This enables integration with Ginkgo 2.3.0's interruptible nodes and node timeouts. +- Introduces Eventually.Within.ProbeEvery with tests and documentation (#591) [f633800] +- New BeKeyOf matcher with documentation and unit tests (#590) [fb586b3] + +## Fixes +- Cover the entire gmeasure suite with leak detection [8c54344] +- Fix gmeasure leak [119d4ce] +- Ignore new Ginkgo ProgressSignal goroutine in gleak [ba548e2] + +## Maintenance + +- Fixes crashes on newer Ruby 3 installations by upgrading github-pages gem dependency (#596) [12469a0] + + ## 1.20.2 ## Fixes diff --git a/vendor/github.com/onsi/gomega/RELEASING.md b/vendor/github.com/onsi/gomega/RELEASING.md index 2d30d9992..9973fff49 100644 --- a/vendor/github.com/onsi/gomega/RELEASING.md +++ b/vendor/github.com/onsi/gomega/RELEASING.md @@ -1,7 +1,13 @@ A Gomega release is a tagged sha and a GitHub release. To cut a release: 1. Ensure CHANGELOG.md is up to date. - - Use `git log --pretty=format:'- %s [%h]' HEAD...vX.X.X` to list all the commits since the last release + - Use + ```bash + LAST_VERSION=$(git tag --sort=version:refname | tail -n1) + CHANGES=$(git log --pretty=format:'- %s [%h]' HEAD...$LAST_VERSION) + echo -e "## NEXT\n\n$CHANGES\n\n### Features\n\n### Fixes\n\n### Maintenance\n\n$(cat CHANGELOG.md)" > CHANGELOG.md + ``` + to update the changelog - Categorize the changes into - Breaking Changes (requires a major version) - New Features (minor version) diff --git a/vendor/github.com/onsi/gomega/format/format.go b/vendor/github.com/onsi/gomega/format/format.go index 6e78c391d..1a2ed877a 100644 --- a/vendor/github.com/onsi/gomega/format/format.go +++ b/vendor/github.com/onsi/gomega/format/format.go @@ -65,6 +65,52 @@ type GomegaStringer interface { GomegaString() string } +/* +CustomFormatters can be registered with Gomega via RegisterCustomFormatter() +Any value to be rendered by Gomega is passed to each registered CustomFormatters. +The CustomFormatter signals that it will handle formatting the value by returning (formatted-string, true) +If the CustomFormatter does not want to handle the object it should return ("", false) + +Strings returned by CustomFormatters are not truncated +*/ +type CustomFormatter func(value interface{}) (string, bool) +type CustomFormatterKey uint + +var customFormatterKey CustomFormatterKey = 1 + +type customFormatterKeyPair struct { + CustomFormatter + CustomFormatterKey +} + +/* +RegisterCustomFormatter registers a CustomFormatter and returns a CustomFormatterKey + +You can call UnregisterCustomFormatter with the returned key to unregister the associated CustomFormatter +*/ +func RegisterCustomFormatter(customFormatter CustomFormatter) CustomFormatterKey { + key := customFormatterKey + customFormatterKey += 1 + customFormatters = append(customFormatters, customFormatterKeyPair{customFormatter, key}) + return key +} + +/* +UnregisterCustomFormatter unregisters a previously registered CustomFormatter. You should pass in the key returned by RegisterCustomFormatter +*/ +func UnregisterCustomFormatter(key CustomFormatterKey) { + formatters := []customFormatterKeyPair{} + for _, f := range customFormatters { + if f.CustomFormatterKey == key { + continue + } + formatters = append(formatters, f) + } + customFormatters = formatters +} + +var customFormatters = []customFormatterKeyPair{} + /* Generates a formatted matcher success/failure message of the form: @@ -219,17 +265,24 @@ func Object(object interface{}, indentation uint) string { IndentString takes a string and indents each line by the specified amount. */ func IndentString(s string, indentation uint) string { + return indentString(s, indentation, true) +} + +func indentString(s string, indentation uint, indentFirstLine bool) string { + result := &strings.Builder{} components := strings.Split(s, "\n") - result := "" indent := strings.Repeat(Indent, int(indentation)) for i, component := range components { - result += indent + component + if i > 0 || indentFirstLine { + result.WriteString(indent) + } + result.WriteString(component) if i < len(components)-1 { - result += "\n" + result.WriteString("\n") } } - return result + return result.String() } func formatType(v reflect.Value) string { @@ -261,18 +314,27 @@ func formatValue(value reflect.Value, indentation uint) string { if value.CanInterface() { obj := value.Interface() + // if a CustomFormatter handles this values, we'll go with that + for _, customFormatter := range customFormatters { + formatted, handled := customFormatter.CustomFormatter(obj) + // do not truncate a user-provided CustomFormatter() + if handled { + return indentString(formatted, indentation+1, false) + } + } + // GomegaStringer will take precedence to other representations and disregards UseStringerRepresentation if x, ok := obj.(GomegaStringer); ok { - // do not truncate a user-defined GoMegaString() value - return x.GomegaString() + // do not truncate a user-defined GomegaString() value + return indentString(x.GomegaString(), indentation+1, false) } if UseStringerRepresentation { switch x := obj.(type) { case fmt.GoStringer: - return truncateLongStrings(x.GoString()) + return indentString(truncateLongStrings(x.GoString()), indentation+1, false) case fmt.Stringer: - return truncateLongStrings(x.String()) + return indentString(truncateLongStrings(x.String()), indentation+1, false) } } } diff --git a/vendor/github.com/onsi/gomega/gomega_dsl.go b/vendor/github.com/onsi/gomega/gomega_dsl.go index 60b1687ed..4405c0682 100644 --- a/vendor/github.com/onsi/gomega/gomega_dsl.go +++ b/vendor/github.com/onsi/gomega/gomega_dsl.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega/types" ) -const GOMEGA_VERSION = "1.20.2" +const GOMEGA_VERSION = "1.27.1" const nilGomegaPanic = `You are trying to make an assertion, but haven't registered Gomega's fail handler. If you're using Ginkgo then you probably forgot to put your assertion in an It(). @@ -86,12 +86,12 @@ func internalGomega(g Gomega) *internal.Gomega { // NewWithT takes a *testing.T and returns a `gomega.WithT` allowing you to use `Expect`, `Eventually`, and `Consistently` along with // Gomega's rich ecosystem of matchers in standard `testing` test suits. // -// func TestFarmHasCow(t *testing.T) { -// g := gomega.NewWithT(t) +// func TestFarmHasCow(t *testing.T) { +// g := gomega.NewWithT(t) // -// f := farm.New([]string{"Cow", "Horse"}) -// g.Expect(f.HasCow()).To(BeTrue(), "Farm should have cow") -// } +// f := farm.New([]string{"Cow", "Horse"}) +// g.Expect(f.HasCow()).To(BeTrue(), "Farm should have cow") +// } func NewWithT(t types.GomegaTestingT) *WithT { return internal.NewGomega(internalGomega(Default).DurationBundle).ConfigureWithT(t) } @@ -171,7 +171,8 @@ func ensureDefaultGomegaIsConfigured() { } // Ω wraps an actual value allowing assertions to be made on it: -// Ω("foo").Should(Equal("foo")) +// +// Ω("foo").Should(Equal("foo")) // // If Ω is passed more than one argument it will pass the *first* argument to the matcher. // All subsequent arguments will be required to be nil/zero. @@ -180,10 +181,13 @@ func ensureDefaultGomegaIsConfigured() { // a value and an error - a common patter in Go. // // For example, given a function with signature: -// func MyAmazingThing() (int, error) +// +// func MyAmazingThing() (int, error) // // Then: -// Ω(MyAmazingThing()).Should(Equal(3)) +// +// Ω(MyAmazingThing()).Should(Equal(3)) +// // Will succeed only if `MyAmazingThing()` returns `(3, nil)` // // Ω and Expect are identical @@ -193,7 +197,8 @@ func Ω(actual interface{}, extra ...interface{}) Assertion { } // Expect wraps an actual value allowing assertions to be made on it: -// Expect("foo").To(Equal("foo")) +// +// Expect("foo").To(Equal("foo")) // // If Expect is passed more than one argument it will pass the *first* argument to the matcher. // All subsequent arguments will be required to be nil/zero. @@ -202,10 +207,13 @@ func Ω(actual interface{}, extra ...interface{}) Assertion { // a value and an error - a common patter in Go. // // For example, given a function with signature: -// func MyAmazingThing() (int, error) +// +// func MyAmazingThing() (int, error) // // Then: -// Expect(MyAmazingThing()).Should(Equal(3)) +// +// Expect(MyAmazingThing()).Should(Equal(3)) +// // Will succeed only if `MyAmazingThing()` returns `(3, nil)` // // Expect and Ω are identical @@ -215,7 +223,8 @@ func Expect(actual interface{}, extra ...interface{}) Assertion { } // ExpectWithOffset wraps an actual value allowing assertions to be made on it: -// ExpectWithOffset(1, "foo").To(Equal("foo")) +// +// ExpectWithOffset(1, "foo").To(Equal("foo")) // // Unlike `Expect` and `Ω`, `ExpectWithOffset` takes an additional integer argument // that is used to modify the call-stack offset when computing line numbers. It is @@ -233,7 +242,7 @@ func ExpectWithOffset(offset int, actual interface{}, extra ...interface{}) Asse Eventually enables making assertions on asynchronous behavior. Eventually checks that an assertion *eventually* passes. Eventually blocks when called and attempts an assertion periodically until it passes or a timeout occurs. Both the timeout and polling interval are configurable as optional arguments. -The first optional argument is the timeout (which defaults to 1s), the second is the polling interval (which defaults to 10ms). Both intervals can be specified as time.Duration, parsable duration strings or floats/integers (in which case they are interpreted as seconds). +The first optional argument is the timeout (which defaults to 1s), the second is the polling interval (which defaults to 10ms). Both intervals can be specified as time.Duration, parsable duration strings or floats/integers (in which case they are interpreted as seconds). In addition an optional context.Context can be passed in - Eventually will keep trying until either the timeout epxires or the context is cancelled, whichever comes first. Eventually works with any Gomega compatible matcher and supports making assertions against three categories of actual value: @@ -241,15 +250,15 @@ Eventually works with any Gomega compatible matcher and supports making assertio There are several examples of values that can change over time. These can be passed in to Eventually and will be passed to the matcher repeatedly until a match occurs. For example: - c := make(chan bool) - go DoStuff(c) - Eventually(c, "50ms").Should(BeClosed()) + c := make(chan bool) + go DoStuff(c) + Eventually(c, "50ms").Should(BeClosed()) will poll the channel repeatedly until it is closed. In this example `Eventually` will block until either the specified timeout of 50ms has elapsed or the channel is closed, whichever comes first. Several Gomega libraries allow you to use Eventually in this way. For example, the gomega/gexec package allows you to block until a *gexec.Session exits successfully via: - Eventually(session).Should(gexec.Exit(0)) + Eventually(session).Should(gexec.Exit(0)) And the gomega/gbytes package allows you to monitor a streaming *gbytes.Buffer until a given string is seen: @@ -266,27 +275,51 @@ this will trigger Go's race detector as the goroutine polling via Eventually wil **Category 2: Make Eventually assertions on functions** -Eventually can be passed functions that **take no arguments** and **return at least one value**. When configured this way, Eventually will poll the function repeatedly and pass the first returned value to the matcher. +Eventually can be passed functions that **return at least one value**. When configured this way, Eventually will poll the function repeatedly and pass the first returned value to the matcher. For example: - Eventually(func() int { - return client.FetchCount() - }).Should(BeNumerically(">=", 17)) + Eventually(func() int { + return client.FetchCount() + }).Should(BeNumerically(">=", 17)) - will repeatedly poll client.FetchCount until the BeNumerically matcher is satisfied. (Note that this example could have been written as Eventually(client.FetchCount).Should(BeNumerically(">=", 17))) + will repeatedly poll client.FetchCount until the BeNumerically matcher is satisfied. (Note that this example could have been written as Eventually(client.FetchCount).Should(BeNumerically(">=", 17))) If multiple values are returned by the function, Eventually will pass the first value to the matcher and require that all others are zero-valued. This allows you to pass Eventually a function that returns a value and an error - a common pattern in Go. For example, consider a method that returns a value and an error: - func FetchFromDB() (string, error) + + func FetchFromDB() (string, error) Then - Eventually(FetchFromDB).Should(Equal("got it")) + + Eventually(FetchFromDB).Should(Equal("got it")) will pass only if and when the returned error is nil *and* the returned string satisfies the matcher. -It is important to note that the function passed into Eventually is invoked *synchronously* when polled. Eventually does not (in fact, it cannot) kill the function if it takes longer to return than Eventually's configured timeout. You should design your functions with this in mind. +Eventually can also accept functions that take arguments, however you must provide those arguments using .WithArguments(). For example, consider a function that takes a user-id and makes a network request to fetch a full name: + + func FetchFullName(userId int) (string, error) + +You can poll this function like so: + + Eventually(FetchFullName).WithArguments(1138).Should(Equal("Wookie")) + +It is important to note that the function passed into Eventually is invoked *synchronously* when polled. Eventually does not (in fact, it cannot) kill the function if it takes longer to return than Eventually's configured timeout. A common practice here is to use a context. Here's an example that combines Ginkgo's spec timeout support with Eventually: + + It("fetches the correct count", func(ctx SpecContext) { + Eventually(ctx, func() int { + return client.FetchCount(ctx, "/users") + }).Should(BeNumerically(">=", 17)) + }, SpecTimeout(time.Second)) + +you an also use Eventually().WithContext(ctx) to pass in the context. Passed-in contexts play nicely with paseed-in arguments as long as the context appears first. You can rewrite the above example as: + + It("fetches the correct count", func(ctx SpecContext) { + Eventually(client.FetchCount).WithContext(ctx).WithArguments("/users").Should(BeNumerically(">=", 17)) + }, SpecTimeout(time.Second)) + +Either way the context passd to Eventually is also passed to the underlying funciton. Now, when Ginkgo cancels the context both the FetchCount client and Gomega will be informed and can exit. **Category 3: Making assertions _in_ the function passed into Eventually** @@ -306,22 +339,48 @@ will pass only if all the assertions in the polled function pass and the return Eventually also supports a special case polling function that takes a single Gomega argument and returns no values. Eventually assumes such a function is making assertions and is designed to work with the Succeed matcher to validate that all assertions have passed. For example: - Eventually(func(g Gomega) { - model, err := client.Find(1138) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(model.Reticulate()).To(Succeed()) - g.Expect(model.IsReticulated()).To(BeTrue()) - g.Expect(model.Save()).To(Succeed()) - }).Should(Succeed()) + Eventually(func(g Gomega) { + model, err := client.Find(1138) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(model.Reticulate()).To(Succeed()) + g.Expect(model.IsReticulated()).To(BeTrue()) + g.Expect(model.Save()).To(Succeed()) + }).Should(Succeed()) will rerun the function until all assertions pass. -`Eventually` specifying a timeout interval (and an optional polling interval) are -the same as `Eventually(...).WithTimeout` or `Eventually(...).WithTimeout(...).WithPolling`. +You can also pass additional arugments to functions that take a Gomega. The only rule is that the Gomega argument must be first. If you also want to pass the context attached to Eventually you must ensure that is the second argument. For example: + + Eventually(func(g Gomega, ctx context.Context, path string, expected ...string){ + tok, err := client.GetToken(ctx) + g.Expect(err).NotTo(HaveOccurred()) + + elements, err := client.Fetch(ctx, tok, path) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(elements).To(ConsistOf(expected)) + }).WithContext(ctx).WithArguments("/names", "Joe", "Jane", "Sam").Should(Succeed()) + +You can ensure that you get a number of consecutive successful tries before succeeding using `MustPassRepeatedly(int)`. For Example: + + int count := 0 + Eventually(func() bool { + count++ + return count > 2 + }).MustPassRepeatedly(2).Should(BeTrue()) + // Because we had to wait for 2 calls that returned true + Expect(count).To(Equal(3)) + +Finally, in addition to passing timeouts and a context to Eventually you can be more explicit with Eventually's chaining configuration methods: + + Eventually(..., "1s", "2s", ctx).Should(...) + +is equivalent to + + Eventually(...).WithTimeout(time.Second).WithPolling(2*time.Second).WithContext(ctx).Should(...) */ -func Eventually(actual interface{}, intervals ...interface{}) AsyncAssertion { +func Eventually(actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.Eventually(actual, intervals...) + return Default.Eventually(actualOrCtx, args...) } // EventuallyWithOffset operates like Eventually but takes an additional @@ -333,9 +392,9 @@ func Eventually(actual interface{}, intervals ...interface{}) AsyncAssertion { // `EventuallyWithOffset` specifying a timeout interval (and an optional polling interval) are // the same as `Eventually(...).WithOffset(...).WithTimeout` or // `Eventually(...).WithOffset(...).WithTimeout(...).WithPolling`. -func EventuallyWithOffset(offset int, actual interface{}, intervals ...interface{}) AsyncAssertion { +func EventuallyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.EventuallyWithOffset(offset, actual, intervals...) + return Default.EventuallyWithOffset(offset, actualOrCtx, args...) } /* @@ -343,19 +402,19 @@ Consistently, like Eventually, enables making assertions on asynchronous behavio Consistently blocks when called for a specified duration. During that duration Consistently repeatedly polls its matcher and ensures that it is satisfied. If the matcher is consistently satisfied, then Consistently will pass. Otherwise Consistently will fail. -Both the total waiting duration and the polling interval are configurable as optional arguments. The first optional argument is the duration that Consistently will run for (defaults to 100ms), and the second argument is the polling interval (defaults to 10ms). As with Eventually, these intervals can be passed in as time.Duration, parsable duration strings or an integer or float number of seconds. +Both the total waiting duration and the polling interval are configurable as optional arguments. The first optional argument is the duration that Consistently will run for (defaults to 100ms), and the second argument is the polling interval (defaults to 10ms). As with Eventually, these intervals can be passed in as time.Duration, parsable duration strings or an integer or float number of seconds. You can also pass in an optional context.Context - Consistently will exit early (with a failure) if the context is cancelled before the waiting duration expires. Consistently accepts the same three categories of actual as Eventually, check the Eventually docs to learn more. Consistently is useful in cases where you want to assert that something *does not happen* for a period of time. For example, you may want to assert that a goroutine does *not* send data down a channel. In this case you could write: - Consistently(channel, "200ms").ShouldNot(Receive()) + Consistently(channel, "200ms").ShouldNot(Receive()) This will block for 200 milliseconds and repeatedly check the channel and ensure nothing has been received. */ -func Consistently(actual interface{}, intervals ...interface{}) AsyncAssertion { +func Consistently(actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.Consistently(actual, intervals...) + return Default.Consistently(actualOrCtx, args...) } // ConsistentlyWithOffset operates like Consistently but takes an additional @@ -364,11 +423,54 @@ func Consistently(actual interface{}, intervals ...interface{}) AsyncAssertion { // // `ConsistentlyWithOffset` is the same as `Consistently(...).WithOffset` and // optional `WithTimeout` and `WithPolling`. -func ConsistentlyWithOffset(offset int, actual interface{}, intervals ...interface{}) AsyncAssertion { +func ConsistentlyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.ConsistentlyWithOffset(offset, actual, intervals...) + return Default.ConsistentlyWithOffset(offset, actualOrCtx, args...) } +/* +StopTrying can be used to signal to Eventually and Consistentlythat they should abort and stop trying. This always results in a failure of the assertion - and the failure message is the content of the StopTrying signal. + +You can send the StopTrying signal by either returning StopTrying("message") as an error from your passed-in function _or_ by calling StopTrying("message").Now() to trigger a panic and end execution. + +You can also wrap StopTrying around an error with `StopTrying("message").Wrap(err)` and can attach additional objects via `StopTrying("message").Attach("description", object). When rendered, the signal will include the wrapped error and any attached objects rendered using Gomega's default formatting. + +Here are a couple of examples. This is how you might use StopTrying() as an error to signal that Eventually should stop: + + playerIndex, numPlayers := 0, 11 + Eventually(func() (string, error) { + if playerIndex == numPlayers { + return "", StopTrying("no more players left") + } + name := client.FetchPlayer(playerIndex) + playerIndex += 1 + return name, nil + }).Should(Equal("Patrick Mahomes")) + +And here's an example where `StopTrying().Now()` is called to halt execution immediately: + + Eventually(func() []string { + names, err := client.FetchAllPlayers() + if err == client.IRRECOVERABLE_ERROR { + StopTrying("Irrecoverable error occurred").Wrap(err).Now() + } + return names + }).Should(ContainElement("Patrick Mahomes")) +*/ +var StopTrying = internal.StopTrying + +/* +TryAgainAfter() allows you to adjust the polling interval for the _next_ iteration of `Eventually` or `Consistently`. Like `StopTrying` you can either return `TryAgainAfter` as an error or trigger it immedieately with `.Now()` + +When `TryAgainAfter(` is triggered `Eventually` and `Consistently` will wait for that duration. If a timeout occurs before the next poll is triggered both `Eventually` and `Consistently` will always fail with the content of the TryAgainAfter message. As with StopTrying you can `.Wrap()` and error and `.Attach()` additional objects to `TryAgainAfter`. +*/ +var TryAgainAfter = internal.TryAgainAfter + +/* +PollingSignalError is the error returned by StopTrying() and TryAgainAfter() +*/ +type PollingSignalError = internal.PollingSignalError + // SetDefaultEventuallyTimeout sets the default timeout duration for Eventually. Eventually will repeatedly poll your condition until it succeeds, or until this timeout elapses. func SetDefaultEventuallyTimeout(t time.Duration) { Default.SetDefaultEventuallyTimeout(t) @@ -402,8 +504,8 @@ func SetDefaultConsistentlyPollingInterval(t time.Duration) { // // Example: // -// Eventually(myChannel).Should(Receive(), "Something should have come down the pipe.") -// Consistently(myChannel).ShouldNot(Receive(), func() string { return "Nothing should have come down the pipe." }) +// Eventually(myChannel).Should(Receive(), "Something should have come down the pipe.") +// Consistently(myChannel).ShouldNot(Receive(), func() string { return "Nothing should have come down the pipe." }) type AsyncAssertion = types.AsyncAssertion // GomegaAsyncAssertion is deprecated in favor of AsyncAssertion, which does not stutter. @@ -425,7 +527,7 @@ type GomegaAsyncAssertion = types.AsyncAssertion // // Example: // -// Ω(farm.HasCow()).Should(BeTrue(), "Farm %v should have a cow", farm) +// Ω(farm.HasCow()).Should(BeTrue(), "Farm %v should have a cow", farm) type Assertion = types.Assertion // GomegaAssertion is deprecated in favor of Assertion, which does not stutter. diff --git a/vendor/github.com/onsi/gomega/internal/assertion.go b/vendor/github.com/onsi/gomega/internal/assertion.go index 7b7bdd149..08356a610 100644 --- a/vendor/github.com/onsi/gomega/internal/assertion.go +++ b/vendor/github.com/onsi/gomega/internal/assertion.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" + "github.com/onsi/gomega/format" "github.com/onsi/gomega/types" ) @@ -146,7 +147,12 @@ func vetActuals(actuals []interface{}, skipIndex int) (bool, string) { if actual != nil { zeroValue := reflect.Zero(reflect.TypeOf(actual)).Interface() if !reflect.DeepEqual(zeroValue, actual) { - message := fmt.Sprintf("Unexpected non-nil/non-zero argument at index %d:\n\t<%T>: %#v", i, actual, actual) + var message string + if err, ok := actual.(error); ok { + message = fmt.Sprintf("Unexpected error: %s\n%s", err, format.Object(err, 1)) + } else { + message = fmt.Sprintf("Unexpected non-nil/non-zero argument at index %d:\n\t<%T>: %#v", i, actual, actual) + } return false, message } } diff --git a/vendor/github.com/onsi/gomega/internal/async_assertion.go b/vendor/github.com/onsi/gomega/internal/async_assertion.go index 126bbcb3f..7f6226968 100644 --- a/vendor/github.com/onsi/gomega/internal/async_assertion.go +++ b/vendor/github.com/onsi/gomega/internal/async_assertion.go @@ -1,15 +1,53 @@ package internal import ( + "context" "errors" "fmt" "reflect" "runtime" + "sync" "time" + "github.com/onsi/gomega/format" "github.com/onsi/gomega/types" ) +var errInterface = reflect.TypeOf((*error)(nil)).Elem() +var gomegaType = reflect.TypeOf((*types.Gomega)(nil)).Elem() +var contextType = reflect.TypeOf(new(context.Context)).Elem() + +type formattedGomegaError interface { + FormattedGomegaError() string +} + +type asyncPolledActualError struct { + message string +} + +func (err *asyncPolledActualError) Error() string { + return err.message +} + +func (err *asyncPolledActualError) FormattedGomegaError() string { + return err.message +} + +type contextWithAttachProgressReporter interface { + AttachProgressReporter(func() string) func() +} + +type asyncGomegaHaltExecutionError struct{} + +func (a asyncGomegaHaltExecutionError) GinkgoRecoverShouldIgnoreThisPanic() {} +func (a asyncGomegaHaltExecutionError) Error() string { + return `An assertion has failed in a goroutine. You should call + + defer GinkgoRecover() + +at the top of the goroutine that caused this panic. This will allow Ginkgo and Gomega to correctly capture and manage this panic.` +} + type AsyncAssertionType uint const ( @@ -17,71 +55,45 @@ const ( AsyncAssertionTypeConsistently ) +func (at AsyncAssertionType) String() string { + switch at { + case AsyncAssertionTypeEventually: + return "Eventually" + case AsyncAssertionTypeConsistently: + return "Consistently" + } + return "INVALID ASYNC ASSERTION TYPE" +} + type AsyncAssertion struct { asyncType AsyncAssertionType - actualIsFunc bool - actualValue interface{} - actualFunc func() ([]reflect.Value, error) + actualIsFunc bool + actual interface{} + argsToForward []interface{} - timeoutInterval time.Duration - pollingInterval time.Duration - offset int - g *Gomega + timeoutInterval time.Duration + pollingInterval time.Duration + mustPassRepeatedly int + ctx context.Context + offset int + g *Gomega } -func NewAsyncAssertion(asyncType AsyncAssertionType, actualInput interface{}, g *Gomega, timeoutInterval time.Duration, pollingInterval time.Duration, offset int) *AsyncAssertion { +func NewAsyncAssertion(asyncType AsyncAssertionType, actualInput interface{}, g *Gomega, timeoutInterval time.Duration, pollingInterval time.Duration, mustPassRepeatedly int, ctx context.Context, offset int) *AsyncAssertion { out := &AsyncAssertion{ - asyncType: asyncType, - timeoutInterval: timeoutInterval, - pollingInterval: pollingInterval, - offset: offset, - g: g, + asyncType: asyncType, + timeoutInterval: timeoutInterval, + pollingInterval: pollingInterval, + mustPassRepeatedly: mustPassRepeatedly, + offset: offset, + ctx: ctx, + g: g, } - switch actualType := reflect.TypeOf(actualInput); { - case actualInput == nil || actualType.Kind() != reflect.Func: - out.actualValue = actualInput - case actualType.NumIn() == 0 && actualType.NumOut() > 0: + out.actual = actualInput + if actualInput != nil && reflect.TypeOf(actualInput).Kind() == reflect.Func { out.actualIsFunc = true - out.actualFunc = func() ([]reflect.Value, error) { - return reflect.ValueOf(actualInput).Call([]reflect.Value{}), nil - } - case actualType.NumIn() == 1 && actualType.In(0).Implements(reflect.TypeOf((*types.Gomega)(nil)).Elem()): - out.actualIsFunc = true - out.actualFunc = func() (values []reflect.Value, err error) { - var assertionFailure error - assertionCapturingGomega := NewGomega(g.DurationBundle).ConfigureWithFailHandler(func(message string, callerSkip ...int) { - skip := 0 - if len(callerSkip) > 0 { - skip = callerSkip[0] - } - _, file, line, _ := runtime.Caller(skip + 1) - assertionFailure = fmt.Errorf("Assertion in callback at %s:%d failed:\n%s", file, line, message) - panic("stop execution") - }) - - defer func() { - if actualType.NumOut() == 0 { - if assertionFailure == nil { - values = []reflect.Value{reflect.Zero(reflect.TypeOf((*error)(nil)).Elem())} - } else { - values = []reflect.Value{reflect.ValueOf(assertionFailure)} - } - } else { - err = assertionFailure - } - if e := recover(); e != nil && assertionFailure == nil { - panic(e) - } - }() - - values = reflect.ValueOf(actualInput).Call([]reflect.Value{reflect.ValueOf(assertionCapturingGomega)}) - return - } - default: - msg := fmt.Sprintf("The function passed to Gomega's async assertions should either take no arguments and return values, or take a single Gomega interface that it can use to make assertions within the body of the function. When taking a Gomega interface the function can optionally return values or return nothing. The function you passed takes %d arguments and returns %d values.", actualType.NumIn(), actualType.NumOut()) - g.Fail(msg, offset+4) } return out @@ -102,6 +114,31 @@ func (assertion *AsyncAssertion) WithPolling(interval time.Duration) types.Async return assertion } +func (assertion *AsyncAssertion) Within(timeout time.Duration) types.AsyncAssertion { + assertion.timeoutInterval = timeout + return assertion +} + +func (assertion *AsyncAssertion) ProbeEvery(interval time.Duration) types.AsyncAssertion { + assertion.pollingInterval = interval + return assertion +} + +func (assertion *AsyncAssertion) WithContext(ctx context.Context) types.AsyncAssertion { + assertion.ctx = ctx + return assertion +} + +func (assertion *AsyncAssertion) WithArguments(argsToForward ...interface{}) types.AsyncAssertion { + assertion.argsToForward = argsToForward + return assertion +} + +func (assertion *AsyncAssertion) MustPassRepeatedly(count int) types.AsyncAssertion { + assertion.mustPassRepeatedly = count + return assertion +} + func (assertion *AsyncAssertion) Should(matcher types.GomegaMatcher, optionalDescription ...interface{}) bool { assertion.g.THelper() vetOptionalDescription("Asynchronous assertion", optionalDescription...) @@ -126,112 +163,401 @@ func (assertion *AsyncAssertion) buildDescription(optionalDescription ...interfa return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...) + "\n" } -func (assertion *AsyncAssertion) pollActual() (interface{}, error) { +func (assertion *AsyncAssertion) processReturnValues(values []reflect.Value) (interface{}, error) { + if len(values) == 0 { + return nil, &asyncPolledActualError{ + message: fmt.Sprintf("The function passed to %s did not return any values", assertion.asyncType), + } + } + + actual := values[0].Interface() + if _, ok := AsPollingSignalError(actual); ok { + return actual, actual.(error) + } + + var err error + for i, extraValue := range values[1:] { + extra := extraValue.Interface() + if extra == nil { + continue + } + if _, ok := AsPollingSignalError(extra); ok { + return actual, extra.(error) + } + extraType := reflect.TypeOf(extra) + zero := reflect.Zero(extraType).Interface() + if reflect.DeepEqual(extra, zero) { + continue + } + if i == len(values)-2 && extraType.Implements(errInterface) { + err = extra.(error) + } + if err == nil { + err = &asyncPolledActualError{ + message: fmt.Sprintf("The function passed to %s had an unexpected non-nil/non-zero return value at index %d:\n%s", assertion.asyncType, i+1, format.Object(extra, 1)), + } + } + } + + return actual, err +} + +func (assertion *AsyncAssertion) invalidFunctionError(t reflect.Type) error { + return fmt.Errorf(`The function passed to %s had an invalid signature of %s. Functions passed to %s must either: + + (a) have return values or + (b) take a Gomega interface as their first argument and use that Gomega instance to make assertions. + +You can learn more at https://onsi.github.io/gomega/#eventually +`, assertion.asyncType, t, assertion.asyncType) +} + +func (assertion *AsyncAssertion) noConfiguredContextForFunctionError() error { + return fmt.Errorf(`The function passed to %s requested a context.Context, but no context has been provided. Please pass one in using %s().WithContext(). + +You can learn more at https://onsi.github.io/gomega/#eventually +`, assertion.asyncType, assertion.asyncType) +} + +func (assertion *AsyncAssertion) argumentMismatchError(t reflect.Type, numProvided int) error { + have := "have" + if numProvided == 1 { + have = "has" + } + return fmt.Errorf(`The function passed to %s has signature %s takes %d arguments but %d %s been provided. Please use %s().WithArguments() to pass the corect set of arguments. + +You can learn more at https://onsi.github.io/gomega/#eventually +`, assertion.asyncType, t, t.NumIn(), numProvided, have, assertion.asyncType) +} + +func (assertion *AsyncAssertion) invalidMustPassRepeatedlyError(reason string) error { + return fmt.Errorf(`Invalid use of MustPassRepeatedly with %s %s + +You can learn more at https://onsi.github.io/gomega/#eventually +`, assertion.asyncType, reason) +} + +func (assertion *AsyncAssertion) buildActualPoller() (func() (interface{}, error), error) { if !assertion.actualIsFunc { - return assertion.actualValue, nil + return func() (interface{}, error) { return assertion.actual, nil }, nil + } + actualValue := reflect.ValueOf(assertion.actual) + actualType := reflect.TypeOf(assertion.actual) + numIn, numOut, isVariadic := actualType.NumIn(), actualType.NumOut(), actualType.IsVariadic() + + if numIn == 0 && numOut == 0 { + return nil, assertion.invalidFunctionError(actualType) + } + takesGomega, takesContext := false, false + if numIn > 0 { + takesGomega, takesContext = actualType.In(0).Implements(gomegaType), actualType.In(0).Implements(contextType) + } + if takesGomega && numIn > 1 && actualType.In(1).Implements(contextType) { + takesContext = true + } + if takesContext && len(assertion.argsToForward) > 0 && reflect.TypeOf(assertion.argsToForward[0]).Implements(contextType) { + takesContext = false + } + if !takesGomega && numOut == 0 { + return nil, assertion.invalidFunctionError(actualType) + } + if takesContext && assertion.ctx == nil { + return nil, assertion.noConfiguredContextForFunctionError() } - values, err := assertion.actualFunc() - if err != nil { - return nil, err + var assertionFailure error + inValues := []reflect.Value{} + if takesGomega { + inValues = append(inValues, reflect.ValueOf(NewGomega(assertion.g.DurationBundle).ConfigureWithFailHandler(func(message string, callerSkip ...int) { + skip := 0 + if len(callerSkip) > 0 { + skip = callerSkip[0] + } + _, file, line, _ := runtime.Caller(skip + 1) + assertionFailure = &asyncPolledActualError{ + message: fmt.Sprintf("The function passed to %s failed at %s:%d with:\n%s", assertion.asyncType, file, line, message), + } + // we throw an asyncGomegaHaltExecutionError so that defer GinkgoRecover() can catch this error if the user makes an assertion in a goroutine + panic(asyncGomegaHaltExecutionError{}) + }))) } - extras := []interface{}{nil} - for _, value := range values[1:] { - extras = append(extras, value.Interface()) + if takesContext { + inValues = append(inValues, reflect.ValueOf(assertion.ctx)) } - success, message := vetActuals(extras, 0) - if !success { - return nil, errors.New(message) + for _, arg := range assertion.argsToForward { + inValues = append(inValues, reflect.ValueOf(arg)) } - return values[0].Interface(), nil + if !isVariadic && numIn != len(inValues) { + return nil, assertion.argumentMismatchError(actualType, len(inValues)) + } else if isVariadic && len(inValues) < numIn-1 { + return nil, assertion.argumentMismatchError(actualType, len(inValues)) + } + + if assertion.mustPassRepeatedly != 1 && assertion.asyncType != AsyncAssertionTypeEventually { + return nil, assertion.invalidMustPassRepeatedlyError("it can only be used with Eventually") + } + if assertion.mustPassRepeatedly < 1 { + return nil, assertion.invalidMustPassRepeatedlyError("parameter can't be < 1") + } + + return func() (actual interface{}, err error) { + var values []reflect.Value + assertionFailure = nil + defer func() { + if numOut == 0 && takesGomega { + actual = assertionFailure + } else { + actual, err = assertion.processReturnValues(values) + _, isAsyncError := AsPollingSignalError(err) + if assertionFailure != nil && !isAsyncError { + err = assertionFailure + } + } + if e := recover(); e != nil { + if _, isAsyncError := AsPollingSignalError(e); isAsyncError { + err = e.(error) + } else if assertionFailure == nil { + panic(e) + } + } + }() + values = actualValue.Call(inValues) + return + }, nil } -func (assertion *AsyncAssertion) matcherMayChange(matcher types.GomegaMatcher, value interface{}) bool { - if assertion.actualIsFunc { - return true +func (assertion *AsyncAssertion) afterTimeout() <-chan time.Time { + if assertion.timeoutInterval >= 0 { + return time.After(assertion.timeoutInterval) + } + + if assertion.asyncType == AsyncAssertionTypeConsistently { + return time.After(assertion.g.DurationBundle.ConsistentlyDuration) + } else { + if assertion.ctx == nil { + return time.After(assertion.g.DurationBundle.EventuallyTimeout) + } else { + return nil + } } - return types.MatchMayChangeInTheFuture(matcher, value) +} + +func (assertion *AsyncAssertion) afterPolling() <-chan time.Time { + if assertion.pollingInterval >= 0 { + return time.After(assertion.pollingInterval) + } + if assertion.asyncType == AsyncAssertionTypeConsistently { + return time.After(assertion.g.DurationBundle.ConsistentlyPollingInterval) + } else { + return time.After(assertion.g.DurationBundle.EventuallyPollingInterval) + } +} + +func (assertion *AsyncAssertion) matcherSaysStopTrying(matcher types.GomegaMatcher, value interface{}) bool { + if assertion.actualIsFunc || types.MatchMayChangeInTheFuture(matcher, value) { + return false + } + return true +} + +func (assertion *AsyncAssertion) pollMatcher(matcher types.GomegaMatcher, value interface{}) (matches bool, err error) { + defer func() { + if e := recover(); e != nil { + if _, isAsyncError := AsPollingSignalError(e); isAsyncError { + err = e.(error) + } else { + panic(e) + } + } + }() + + matches, err = matcher.Match(value) + + return } func (assertion *AsyncAssertion) match(matcher types.GomegaMatcher, desiredMatch bool, optionalDescription ...interface{}) bool { timer := time.Now() - timeout := time.After(assertion.timeoutInterval) + timeout := assertion.afterTimeout() + lock := sync.Mutex{} - var matches bool - var err error - mayChange := true - value, err := assertion.pollActual() - if err == nil { - mayChange = assertion.matcherMayChange(matcher, value) - matches, err = matcher.Match(value) - } + var matches, hasLastValidActual bool + var actual, lastValidActual interface{} + var actualErr, matcherErr error + var oracleMatcherSaysStop bool assertion.g.THelper() - fail := func(preamble string) { - errMsg := "" + pollActual, buildActualPollerErr := assertion.buildActualPoller() + if buildActualPollerErr != nil { + assertion.g.Fail(buildActualPollerErr.Error(), 2+assertion.offset) + return false + } + + actual, actualErr = pollActual() + if actualErr == nil { + lastValidActual = actual + hasLastValidActual = true + oracleMatcherSaysStop = assertion.matcherSaysStopTrying(matcher, actual) + matches, matcherErr = assertion.pollMatcher(matcher, actual) + } + + renderError := func(preamble string, err error) string { message := "" - if err != nil { - errMsg = "Error: " + err.Error() + if pollingSignalErr, ok := AsPollingSignalError(err); ok { + message = err.Error() + for _, attachment := range pollingSignalErr.Attachments { + message += fmt.Sprintf("\n%s:\n", attachment.Description) + message += format.Object(attachment.Object, 1) + } } else { - if desiredMatch { - message = matcher.FailureMessage(value) + message = preamble + "\n" + err.Error() + "\n" + format.Object(err, 1) + } + return message + } + + messageGenerator := func() string { + // can be called out of band by Ginkgo if the user requests a progress report + lock.Lock() + defer lock.Unlock() + message := "" + + if actualErr == nil { + if matcherErr == nil { + if desiredMatch { + message += matcher.FailureMessage(actual) + } else { + message += matcher.NegatedFailureMessage(actual) + } } else { - message = matcher.NegatedFailureMessage(value) + var fgErr formattedGomegaError + if errors.As(actualErr, &fgErr) { + message += fgErr.FormattedGomegaError() + "\n" + } else { + message += renderError(fmt.Sprintf("The matcher passed to %s returned the following error:", assertion.asyncType), matcherErr) + } + } + } else { + var fgErr formattedGomegaError + if errors.As(actualErr, &fgErr) { + message += fgErr.FormattedGomegaError() + "\n" + } else { + message += renderError(fmt.Sprintf("The function passed to %s returned the following error:", assertion.asyncType), actualErr) + } + if hasLastValidActual { + message += fmt.Sprintf("\nAt one point, however, the function did return successfully.\nYet, %s failed because", assertion.asyncType) + _, e := matcher.Match(lastValidActual) + if e != nil { + message += renderError(" the matcher returned the following error:", e) + } else { + message += " the matcher was not satisfied:\n" + if desiredMatch { + message += matcher.FailureMessage(lastValidActual) + } else { + message += matcher.NegatedFailureMessage(lastValidActual) + } + } } } - assertion.g.THelper() + description := assertion.buildDescription(optionalDescription...) - assertion.g.Fail(fmt.Sprintf("%s after %.3fs.\n%s%s%s", preamble, time.Since(timer).Seconds(), description, message, errMsg), 3+assertion.offset) + return fmt.Sprintf("%s%s", description, message) } - if assertion.asyncType == AsyncAssertionTypeEventually { - for { - if err == nil && matches == desiredMatch { - return true - } + fail := func(preamble string) { + assertion.g.THelper() + assertion.g.Fail(fmt.Sprintf("%s after %.3fs.\n%s", preamble, time.Since(timer).Seconds(), messageGenerator()), 3+assertion.offset) + } - if !mayChange { - fail("No future change is possible. Bailing out early") - return false - } + var contextDone <-chan struct{} + if assertion.ctx != nil { + contextDone = assertion.ctx.Done() + if v, ok := assertion.ctx.Value("GINKGO_SPEC_CONTEXT").(contextWithAttachProgressReporter); ok { + detach := v.AttachProgressReporter(messageGenerator) + defer detach() + } + } - select { - case <-time.After(assertion.pollingInterval): - value, err = assertion.pollActual() - if err == nil { - mayChange = assertion.matcherMayChange(matcher, value) - matches, err = matcher.Match(value) + // Used to count the number of times in a row a step passed + passedRepeatedlyCount := 0 + for { + var nextPoll <-chan time.Time = nil + var isTryAgainAfterError = false + + for _, err := range []error{actualErr, matcherErr} { + if pollingSignalErr, ok := AsPollingSignalError(err); ok { + if pollingSignalErr.IsStopTrying() { + fail("Told to stop trying") + return false + } + if pollingSignalErr.IsTryAgainAfter() { + nextPoll = time.After(pollingSignalErr.TryAgainDuration()) + isTryAgainAfterError = true } - case <-timeout: - fail("Timed out") - return false } } - } else if assertion.asyncType == AsyncAssertionTypeConsistently { - for { - if !(err == nil && matches == desiredMatch) { + + if actualErr == nil && matcherErr == nil && matches == desiredMatch { + if assertion.asyncType == AsyncAssertionTypeEventually { + passedRepeatedlyCount += 1 + if passedRepeatedlyCount == assertion.mustPassRepeatedly { + return true + } + } + } else if !isTryAgainAfterError { + if assertion.asyncType == AsyncAssertionTypeConsistently { fail("Failed") return false } + // Reset the consecutive pass count + passedRepeatedlyCount = 0 + } - if !mayChange { + if oracleMatcherSaysStop { + if assertion.asyncType == AsyncAssertionTypeEventually { + fail("No future change is possible. Bailing out early") + return false + } else { return true } + } + + if nextPoll == nil { + nextPoll = assertion.afterPolling() + } - select { - case <-time.After(assertion.pollingInterval): - value, err = assertion.pollActual() - if err == nil { - mayChange = assertion.matcherMayChange(matcher, value) - matches, err = matcher.Match(value) + select { + case <-nextPoll: + a, e := pollActual() + lock.Lock() + actual, actualErr = a, e + lock.Unlock() + if actualErr == nil { + lock.Lock() + lastValidActual = actual + hasLastValidActual = true + lock.Unlock() + oracleMatcherSaysStop = assertion.matcherSaysStopTrying(matcher, actual) + m, e := assertion.pollMatcher(matcher, actual) + lock.Lock() + matches, matcherErr = m, e + lock.Unlock() + } + case <-contextDone: + fail("Context was cancelled") + return false + case <-timeout: + if assertion.asyncType == AsyncAssertionTypeEventually { + fail("Timed out") + return false + } else { + if isTryAgainAfterError { + fail("Timed out while waiting on TryAgainAfter") + return false } - case <-timeout: return true } } } - - return false } diff --git a/vendor/github.com/onsi/gomega/internal/duration_bundle.go b/vendor/github.com/onsi/gomega/internal/duration_bundle.go index af8d989fa..6e0d90d3a 100644 --- a/vendor/github.com/onsi/gomega/internal/duration_bundle.go +++ b/vendor/github.com/onsi/gomega/internal/duration_bundle.go @@ -44,28 +44,28 @@ func durationFromEnv(key string, defaultDuration time.Duration) time.Duration { return duration } -func toDuration(input interface{}) time.Duration { +func toDuration(input interface{}) (time.Duration, error) { duration, ok := input.(time.Duration) if ok { - return duration + return duration, nil } value := reflect.ValueOf(input) kind := reflect.TypeOf(input).Kind() if reflect.Int <= kind && kind <= reflect.Int64 { - return time.Duration(value.Int()) * time.Second + return time.Duration(value.Int()) * time.Second, nil } else if reflect.Uint <= kind && kind <= reflect.Uint64 { - return time.Duration(value.Uint()) * time.Second + return time.Duration(value.Uint()) * time.Second, nil } else if reflect.Float32 <= kind && kind <= reflect.Float64 { - return time.Duration(value.Float() * float64(time.Second)) + return time.Duration(value.Float() * float64(time.Second)), nil } else if reflect.String == kind { duration, err := time.ParseDuration(value.String()) if err != nil { - panic(fmt.Sprintf("%#v is not a valid parsable duration string.", input)) + return 0, fmt.Errorf("%#v is not a valid parsable duration string: %w", input, err) } - return duration + return duration, nil } - panic(fmt.Sprintf("%v is not a valid interval. Must be time.Duration, parsable duration string or a number.", input)) + return 0, fmt.Errorf("%#v is not a valid interval. Must be a time.Duration, a parsable duration string, or a number.", input) } diff --git a/vendor/github.com/onsi/gomega/internal/gomega.go b/vendor/github.com/onsi/gomega/internal/gomega.go index d26a67485..de1f4f336 100644 --- a/vendor/github.com/onsi/gomega/internal/gomega.go +++ b/vendor/github.com/onsi/gomega/internal/gomega.go @@ -1,6 +1,7 @@ package internal import ( + "context" "time" "github.com/onsi/gomega/types" @@ -51,38 +52,64 @@ func (g *Gomega) ExpectWithOffset(offset int, actual interface{}, extra ...inter return NewAssertion(actual, g, offset, extra...) } -func (g *Gomega) Eventually(actual interface{}, intervals ...interface{}) types.AsyncAssertion { - return g.EventuallyWithOffset(0, actual, intervals...) +func (g *Gomega) Eventually(actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeEventually, 0, actualOrCtx, args...) } -func (g *Gomega) EventuallyWithOffset(offset int, actual interface{}, intervals ...interface{}) types.AsyncAssertion { - timeoutInterval := g.DurationBundle.EventuallyTimeout - pollingInterval := g.DurationBundle.EventuallyPollingInterval - if len(intervals) > 0 { - timeoutInterval = toDuration(intervals[0]) - } - if len(intervals) > 1 { - pollingInterval = toDuration(intervals[1]) - } +func (g *Gomega) EventuallyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeEventually, offset, actualOrCtx, args...) +} - return NewAsyncAssertion(AsyncAssertionTypeEventually, actual, g, timeoutInterval, pollingInterval, offset) +func (g *Gomega) Consistently(actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeConsistently, 0, actualOrCtx, args...) } -func (g *Gomega) Consistently(actual interface{}, intervals ...interface{}) types.AsyncAssertion { - return g.ConsistentlyWithOffset(0, actual, intervals...) +func (g *Gomega) ConsistentlyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeConsistently, offset, actualOrCtx, args...) } -func (g *Gomega) ConsistentlyWithOffset(offset int, actual interface{}, intervals ...interface{}) types.AsyncAssertion { - timeoutInterval := g.DurationBundle.ConsistentlyDuration - pollingInterval := g.DurationBundle.ConsistentlyPollingInterval +func (g *Gomega) makeAsyncAssertion(asyncAssertionType AsyncAssertionType, offset int, actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + baseOffset := 3 + timeoutInterval := -time.Duration(1) + pollingInterval := -time.Duration(1) + intervals := []interface{}{} + var ctx context.Context + + actual := actualOrCtx + startingIndex := 0 + if _, isCtx := actualOrCtx.(context.Context); isCtx && len(args) > 0 { + // the first argument is a context, we should accept it as the context _only if_ it is **not** the only argumnent **and** the second argument is not a parseable duration + // this is due to an unfortunate ambiguity in early version of Gomega in which multi-type durations are allowed after the actual + if _, err := toDuration(args[0]); err != nil { + ctx = actualOrCtx.(context.Context) + actual = args[0] + startingIndex = 1 + } + } + + for _, arg := range args[startingIndex:] { + switch v := arg.(type) { + case context.Context: + ctx = v + default: + intervals = append(intervals, arg) + } + } + var err error if len(intervals) > 0 { - timeoutInterval = toDuration(intervals[0]) + timeoutInterval, err = toDuration(intervals[0]) + if err != nil { + g.Fail(err.Error(), offset+baseOffset) + } } if len(intervals) > 1 { - pollingInterval = toDuration(intervals[1]) + pollingInterval, err = toDuration(intervals[1]) + if err != nil { + g.Fail(err.Error(), offset+baseOffset) + } } - return NewAsyncAssertion(AsyncAssertionTypeConsistently, actual, g, timeoutInterval, pollingInterval, offset) + return NewAsyncAssertion(asyncAssertionType, actual, g, timeoutInterval, pollingInterval, 1, ctx, offset) } func (g *Gomega) SetDefaultEventuallyTimeout(t time.Duration) { diff --git a/vendor/github.com/onsi/gomega/internal/polling_signal_error.go b/vendor/github.com/onsi/gomega/internal/polling_signal_error.go new file mode 100644 index 000000000..83b04b1a4 --- /dev/null +++ b/vendor/github.com/onsi/gomega/internal/polling_signal_error.go @@ -0,0 +1,106 @@ +package internal + +import ( + "errors" + "fmt" + "time" +) + +type PollingSignalErrorType int + +const ( + PollingSignalErrorTypeStopTrying PollingSignalErrorType = iota + PollingSignalErrorTypeTryAgainAfter +) + +type PollingSignalError interface { + error + Wrap(err error) PollingSignalError + Attach(description string, obj any) PollingSignalError + Now() +} + +var StopTrying = func(message string) PollingSignalError { + return &PollingSignalErrorImpl{ + message: message, + pollingSignalErrorType: PollingSignalErrorTypeStopTrying, + } +} + +var TryAgainAfter = func(duration time.Duration) PollingSignalError { + return &PollingSignalErrorImpl{ + message: fmt.Sprintf("told to try again after %s", duration), + duration: duration, + pollingSignalErrorType: PollingSignalErrorTypeTryAgainAfter, + } +} + +type PollingSignalErrorAttachment struct { + Description string + Object any +} + +type PollingSignalErrorImpl struct { + message string + wrappedErr error + pollingSignalErrorType PollingSignalErrorType + duration time.Duration + Attachments []PollingSignalErrorAttachment +} + +func (s *PollingSignalErrorImpl) Wrap(err error) PollingSignalError { + s.wrappedErr = err + return s +} + +func (s *PollingSignalErrorImpl) Attach(description string, obj any) PollingSignalError { + s.Attachments = append(s.Attachments, PollingSignalErrorAttachment{description, obj}) + return s +} + +func (s *PollingSignalErrorImpl) Error() string { + if s.wrappedErr == nil { + return s.message + } else { + return s.message + ": " + s.wrappedErr.Error() + } +} + +func (s *PollingSignalErrorImpl) Unwrap() error { + if s == nil { + return nil + } + return s.wrappedErr +} + +func (s *PollingSignalErrorImpl) Now() { + panic(s) +} + +func (s *PollingSignalErrorImpl) IsStopTrying() bool { + return s.pollingSignalErrorType == PollingSignalErrorTypeStopTrying +} + +func (s *PollingSignalErrorImpl) IsTryAgainAfter() bool { + return s.pollingSignalErrorType == PollingSignalErrorTypeTryAgainAfter +} + +func (s *PollingSignalErrorImpl) TryAgainDuration() time.Duration { + return s.duration +} + +func AsPollingSignalError(actual interface{}) (*PollingSignalErrorImpl, bool) { + if actual == nil { + return nil, false + } + if actualErr, ok := actual.(error); ok { + var target *PollingSignalErrorImpl + if errors.As(actualErr, &target) { + return target, true + } else { + return nil, false + } + } + + return nil, false +} diff --git a/vendor/github.com/onsi/gomega/matchers.go b/vendor/github.com/onsi/gomega/matchers.go index d6a099060..44056ad64 100644 --- a/vendor/github.com/onsi/gomega/matchers.go +++ b/vendor/github.com/onsi/gomega/matchers.go @@ -8,27 +8,28 @@ import ( "github.com/onsi/gomega/types" ) -//Equal uses reflect.DeepEqual to compare actual with expected. Equal is strict about -//types when performing comparisons. -//It is an error for both actual and expected to be nil. Use BeNil() instead. +// Equal uses reflect.DeepEqual to compare actual with expected. Equal is strict about +// types when performing comparisons. +// It is an error for both actual and expected to be nil. Use BeNil() instead. func Equal(expected interface{}) types.GomegaMatcher { return &matchers.EqualMatcher{ Expected: expected, } } -//BeEquivalentTo is more lax than Equal, allowing equality between different types. -//This is done by converting actual to have the type of expected before -//attempting equality with reflect.DeepEqual. -//It is an error for actual and expected to be nil. Use BeNil() instead. +// BeEquivalentTo is more lax than Equal, allowing equality between different types. +// This is done by converting actual to have the type of expected before +// attempting equality with reflect.DeepEqual. +// It is an error for actual and expected to be nil. Use BeNil() instead. func BeEquivalentTo(expected interface{}) types.GomegaMatcher { return &matchers.BeEquivalentToMatcher{ Expected: expected, } } -//BeComparableTo uses gocmp.Equal to compare. You can pass cmp.Option as options. -//It is an error for actual and expected to be nil. Use BeNil() instead. +// BeComparableTo uses gocmp.Equal from github.com/google/go-cmp (instead of reflect.DeepEqual) to perform a deep comparison. +// You can pass cmp.Option as options. +// It is an error for actual and expected to be nil. Use BeNil() instead. func BeComparableTo(expected interface{}, opts ...cmp.Option) types.GomegaMatcher { return &matchers.BeComparableToMatcher{ Expected: expected, @@ -36,116 +37,124 @@ func BeComparableTo(expected interface{}, opts ...cmp.Option) types.GomegaMatche } } -//BeIdenticalTo uses the == operator to compare actual with expected. -//BeIdenticalTo is strict about types when performing comparisons. -//It is an error for both actual and expected to be nil. Use BeNil() instead. +// BeIdenticalTo uses the == operator to compare actual with expected. +// BeIdenticalTo is strict about types when performing comparisons. +// It is an error for both actual and expected to be nil. Use BeNil() instead. func BeIdenticalTo(expected interface{}) types.GomegaMatcher { return &matchers.BeIdenticalToMatcher{ Expected: expected, } } -//BeNil succeeds if actual is nil +// BeNil succeeds if actual is nil func BeNil() types.GomegaMatcher { return &matchers.BeNilMatcher{} } -//BeTrue succeeds if actual is true +// BeTrue succeeds if actual is true func BeTrue() types.GomegaMatcher { return &matchers.BeTrueMatcher{} } -//BeFalse succeeds if actual is false +// BeFalse succeeds if actual is false func BeFalse() types.GomegaMatcher { return &matchers.BeFalseMatcher{} } -//HaveOccurred succeeds if actual is a non-nil error -//The typical Go error checking pattern looks like: -// err := SomethingThatMightFail() -// Expect(err).ShouldNot(HaveOccurred()) +// HaveOccurred succeeds if actual is a non-nil error +// The typical Go error checking pattern looks like: +// +// err := SomethingThatMightFail() +// Expect(err).ShouldNot(HaveOccurred()) func HaveOccurred() types.GomegaMatcher { return &matchers.HaveOccurredMatcher{} } -//Succeed passes if actual is a nil error -//Succeed is intended to be used with functions that return a single error value. Instead of -// err := SomethingThatMightFail() -// Expect(err).ShouldNot(HaveOccurred()) +// Succeed passes if actual is a nil error +// Succeed is intended to be used with functions that return a single error value. Instead of +// +// err := SomethingThatMightFail() +// Expect(err).ShouldNot(HaveOccurred()) +// +// You can write: // -//You can write: -// Expect(SomethingThatMightFail()).Should(Succeed()) +// Expect(SomethingThatMightFail()).Should(Succeed()) // -//It is a mistake to use Succeed with a function that has multiple return values. Gomega's Ω and Expect -//functions automatically trigger failure if any return values after the first return value are non-zero/non-nil. -//This means that Ω(MultiReturnFunc()).ShouldNot(Succeed()) can never pass. +// It is a mistake to use Succeed with a function that has multiple return values. Gomega's Ω and Expect +// functions automatically trigger failure if any return values after the first return value are non-zero/non-nil. +// This means that Ω(MultiReturnFunc()).ShouldNot(Succeed()) can never pass. func Succeed() types.GomegaMatcher { return &matchers.SucceedMatcher{} } -//MatchError succeeds if actual is a non-nil error that matches the passed in string/error. +// MatchError succeeds if actual is a non-nil error that matches the passed in string/error. // -//These are valid use-cases: -// Expect(err).Should(MatchError("an error")) //asserts that err.Error() == "an error" -// Expect(err).Should(MatchError(SomeError)) //asserts that err == SomeError (via reflect.DeepEqual) +// These are valid use-cases: // -//It is an error for err to be nil or an object that does not implement the Error interface +// Expect(err).Should(MatchError("an error")) //asserts that err.Error() == "an error" +// Expect(err).Should(MatchError(SomeError)) //asserts that err == SomeError (via reflect.DeepEqual) +// +// It is an error for err to be nil or an object that does not implement the Error interface func MatchError(expected interface{}) types.GomegaMatcher { return &matchers.MatchErrorMatcher{ Expected: expected, } } -//BeClosed succeeds if actual is a closed channel. -//It is an error to pass a non-channel to BeClosed, it is also an error to pass nil +// BeClosed succeeds if actual is a closed channel. +// It is an error to pass a non-channel to BeClosed, it is also an error to pass nil // -//In order to check whether or not the channel is closed, Gomega must try to read from the channel -//(even in the `ShouldNot(BeClosed())` case). You should keep this in mind if you wish to make subsequent assertions about -//values coming down the channel. +// In order to check whether or not the channel is closed, Gomega must try to read from the channel +// (even in the `ShouldNot(BeClosed())` case). You should keep this in mind if you wish to make subsequent assertions about +// values coming down the channel. // -//Also, if you are testing that a *buffered* channel is closed you must first read all values out of the channel before -//asserting that it is closed (it is not possible to detect that a buffered-channel has been closed until all its buffered values are read). +// Also, if you are testing that a *buffered* channel is closed you must first read all values out of the channel before +// asserting that it is closed (it is not possible to detect that a buffered-channel has been closed until all its buffered values are read). // -//Finally, as a corollary: it is an error to check whether or not a send-only channel is closed. +// Finally, as a corollary: it is an error to check whether or not a send-only channel is closed. func BeClosed() types.GomegaMatcher { return &matchers.BeClosedMatcher{} } -//Receive succeeds if there is a value to be received on actual. -//Actual must be a channel (and cannot be a send-only channel) -- anything else is an error. +// Receive succeeds if there is a value to be received on actual. +// Actual must be a channel (and cannot be a send-only channel) -- anything else is an error. +// +// Receive returns immediately and never blocks: // -//Receive returns immediately and never blocks: +// - If there is nothing on the channel `c` then Expect(c).Should(Receive()) will fail and Ω(c).ShouldNot(Receive()) will pass. // -//- If there is nothing on the channel `c` then Expect(c).Should(Receive()) will fail and Ω(c).ShouldNot(Receive()) will pass. +// - If the channel `c` is closed then Expect(c).Should(Receive()) will fail and Ω(c).ShouldNot(Receive()) will pass. // -//- If the channel `c` is closed then Expect(c).Should(Receive()) will fail and Ω(c).ShouldNot(Receive()) will pass. +// - If there is something on the channel `c` ready to be read, then Expect(c).Should(Receive()) will pass and Ω(c).ShouldNot(Receive()) will fail. // -//- If there is something on the channel `c` ready to be read, then Expect(c).Should(Receive()) will pass and Ω(c).ShouldNot(Receive()) will fail. +// If you have a go-routine running in the background that will write to channel `c` you can: // -//If you have a go-routine running in the background that will write to channel `c` you can: -// Eventually(c).Should(Receive()) +// Eventually(c).Should(Receive()) // -//This will timeout if nothing gets sent to `c` (you can modify the timeout interval as you normally do with `Eventually`) +// This will timeout if nothing gets sent to `c` (you can modify the timeout interval as you normally do with `Eventually`) // -//A similar use-case is to assert that no go-routine writes to a channel (for a period of time). You can do this with `Consistently`: -// Consistently(c).ShouldNot(Receive()) +// A similar use-case is to assert that no go-routine writes to a channel (for a period of time). You can do this with `Consistently`: // -//You can pass `Receive` a matcher. If you do so, it will match the received object against the matcher. For example: -// Expect(c).Should(Receive(Equal("foo"))) +// Consistently(c).ShouldNot(Receive()) // -//When given a matcher, `Receive` will always fail if there is nothing to be received on the channel. +// You can pass `Receive` a matcher. If you do so, it will match the received object against the matcher. For example: // -//Passing Receive a matcher is especially useful when paired with Eventually: +// Expect(c).Should(Receive(Equal("foo"))) // -// Eventually(c).Should(Receive(ContainSubstring("bar"))) +// When given a matcher, `Receive` will always fail if there is nothing to be received on the channel. // -//will repeatedly attempt to pull values out of `c` until a value matching "bar" is received. +// Passing Receive a matcher is especially useful when paired with Eventually: // -//Finally, if you want to have a reference to the value *sent* to the channel you can pass the `Receive` matcher a pointer to a variable of the appropriate type: -// var myThing thing -// Eventually(thingChan).Should(Receive(&myThing)) -// Expect(myThing.Sprocket).Should(Equal("foo")) -// Expect(myThing.IsValid()).Should(BeTrue()) +// Eventually(c).Should(Receive(ContainSubstring("bar"))) +// +// will repeatedly attempt to pull values out of `c` until a value matching "bar" is received. +// +// Finally, if you want to have a reference to the value *sent* to the channel you can pass the `Receive` matcher a pointer to a variable of the appropriate type: +// +// var myThing thing +// Eventually(thingChan).Should(Receive(&myThing)) +// Expect(myThing.Sprocket).Should(Equal("foo")) +// Expect(myThing.IsValid()).Should(BeTrue()) func Receive(args ...interface{}) types.GomegaMatcher { var arg interface{} if len(args) > 0 { @@ -157,27 +166,27 @@ func Receive(args ...interface{}) types.GomegaMatcher { } } -//BeSent succeeds if a value can be sent to actual. -//Actual must be a channel (and cannot be a receive-only channel) that can sent the type of the value passed into BeSent -- anything else is an error. -//In addition, actual must not be closed. +// BeSent succeeds if a value can be sent to actual. +// Actual must be a channel (and cannot be a receive-only channel) that can sent the type of the value passed into BeSent -- anything else is an error. +// In addition, actual must not be closed. // -//BeSent never blocks: +// BeSent never blocks: // -//- If the channel `c` is not ready to receive then Expect(c).Should(BeSent("foo")) will fail immediately -//- If the channel `c` is eventually ready to receive then Eventually(c).Should(BeSent("foo")) will succeed.. presuming the channel becomes ready to receive before Eventually's timeout -//- If the channel `c` is closed then Expect(c).Should(BeSent("foo")) and Ω(c).ShouldNot(BeSent("foo")) will both fail immediately +// - If the channel `c` is not ready to receive then Expect(c).Should(BeSent("foo")) will fail immediately +// - If the channel `c` is eventually ready to receive then Eventually(c).Should(BeSent("foo")) will succeed.. presuming the channel becomes ready to receive before Eventually's timeout +// - If the channel `c` is closed then Expect(c).Should(BeSent("foo")) and Ω(c).ShouldNot(BeSent("foo")) will both fail immediately // -//Of course, the value is actually sent to the channel. The point of `BeSent` is less to make an assertion about the availability of the channel (which is typically an implementation detail that your test should not be concerned with). -//Rather, the point of `BeSent` is to make it possible to easily and expressively write tests that can timeout on blocked channel sends. +// Of course, the value is actually sent to the channel. The point of `BeSent` is less to make an assertion about the availability of the channel (which is typically an implementation detail that your test should not be concerned with). +// Rather, the point of `BeSent` is to make it possible to easily and expressively write tests that can timeout on blocked channel sends. func BeSent(arg interface{}) types.GomegaMatcher { return &matchers.BeSentMatcher{ Arg: arg, } } -//MatchRegexp succeeds if actual is a string or stringer that matches the -//passed-in regexp. Optional arguments can be provided to construct a regexp -//via fmt.Sprintf(). +// MatchRegexp succeeds if actual is a string or stringer that matches the +// passed-in regexp. Optional arguments can be provided to construct a regexp +// via fmt.Sprintf(). func MatchRegexp(regexp string, args ...interface{}) types.GomegaMatcher { return &matchers.MatchRegexpMatcher{ Regexp: regexp, @@ -185,9 +194,9 @@ func MatchRegexp(regexp string, args ...interface{}) types.GomegaMatcher { } } -//ContainSubstring succeeds if actual is a string or stringer that contains the -//passed-in substring. Optional arguments can be provided to construct the substring -//via fmt.Sprintf(). +// ContainSubstring succeeds if actual is a string or stringer that contains the +// passed-in substring. Optional arguments can be provided to construct the substring +// via fmt.Sprintf(). func ContainSubstring(substr string, args ...interface{}) types.GomegaMatcher { return &matchers.ContainSubstringMatcher{ Substr: substr, @@ -195,9 +204,9 @@ func ContainSubstring(substr string, args ...interface{}) types.GomegaMatcher { } } -//HavePrefix succeeds if actual is a string or stringer that contains the -//passed-in string as a prefix. Optional arguments can be provided to construct -//via fmt.Sprintf(). +// HavePrefix succeeds if actual is a string or stringer that contains the +// passed-in string as a prefix. Optional arguments can be provided to construct +// via fmt.Sprintf(). func HavePrefix(prefix string, args ...interface{}) types.GomegaMatcher { return &matchers.HavePrefixMatcher{ Prefix: prefix, @@ -205,9 +214,9 @@ func HavePrefix(prefix string, args ...interface{}) types.GomegaMatcher { } } -//HaveSuffix succeeds if actual is a string or stringer that contains the -//passed-in string as a suffix. Optional arguments can be provided to construct -//via fmt.Sprintf(). +// HaveSuffix succeeds if actual is a string or stringer that contains the +// passed-in string as a suffix. Optional arguments can be provided to construct +// via fmt.Sprintf(). func HaveSuffix(suffix string, args ...interface{}) types.GomegaMatcher { return &matchers.HaveSuffixMatcher{ Suffix: suffix, @@ -215,73 +224,74 @@ func HaveSuffix(suffix string, args ...interface{}) types.GomegaMatcher { } } -//MatchJSON succeeds if actual is a string or stringer of JSON that matches -//the expected JSON. The JSONs are decoded and the resulting objects are compared via -//reflect.DeepEqual so things like key-ordering and whitespace shouldn't matter. +// MatchJSON succeeds if actual is a string or stringer of JSON that matches +// the expected JSON. The JSONs are decoded and the resulting objects are compared via +// reflect.DeepEqual so things like key-ordering and whitespace shouldn't matter. func MatchJSON(json interface{}) types.GomegaMatcher { return &matchers.MatchJSONMatcher{ JSONToMatch: json, } } -//MatchXML succeeds if actual is a string or stringer of XML that matches -//the expected XML. The XMLs are decoded and the resulting objects are compared via -//reflect.DeepEqual so things like whitespaces shouldn't matter. +// MatchXML succeeds if actual is a string or stringer of XML that matches +// the expected XML. The XMLs are decoded and the resulting objects are compared via +// reflect.DeepEqual so things like whitespaces shouldn't matter. func MatchXML(xml interface{}) types.GomegaMatcher { return &matchers.MatchXMLMatcher{ XMLToMatch: xml, } } -//MatchYAML succeeds if actual is a string or stringer of YAML that matches -//the expected YAML. The YAML's are decoded and the resulting objects are compared via -//reflect.DeepEqual so things like key-ordering and whitespace shouldn't matter. +// MatchYAML succeeds if actual is a string or stringer of YAML that matches +// the expected YAML. The YAML's are decoded and the resulting objects are compared via +// reflect.DeepEqual so things like key-ordering and whitespace shouldn't matter. func MatchYAML(yaml interface{}) types.GomegaMatcher { return &matchers.MatchYAMLMatcher{ YAMLToMatch: yaml, } } -//BeEmpty succeeds if actual is empty. Actual must be of type string, array, map, chan, or slice. +// BeEmpty succeeds if actual is empty. Actual must be of type string, array, map, chan, or slice. func BeEmpty() types.GomegaMatcher { return &matchers.BeEmptyMatcher{} } -//HaveLen succeeds if actual has the passed-in length. Actual must be of type string, array, map, chan, or slice. +// HaveLen succeeds if actual has the passed-in length. Actual must be of type string, array, map, chan, or slice. func HaveLen(count int) types.GomegaMatcher { return &matchers.HaveLenMatcher{ Count: count, } } -//HaveCap succeeds if actual has the passed-in capacity. Actual must be of type array, chan, or slice. +// HaveCap succeeds if actual has the passed-in capacity. Actual must be of type array, chan, or slice. func HaveCap(count int) types.GomegaMatcher { return &matchers.HaveCapMatcher{ Count: count, } } -//BeZero succeeds if actual is the zero value for its type or if actual is nil. +// BeZero succeeds if actual is the zero value for its type or if actual is nil. func BeZero() types.GomegaMatcher { return &matchers.BeZeroMatcher{} } -//ContainElement succeeds if actual contains the passed in element. By default -//ContainElement() uses Equal() to perform the match, however a matcher can be -//passed in instead: -// Expect([]string{"Foo", "FooBar"}).Should(ContainElement(ContainSubstring("Bar"))) +// ContainElement succeeds if actual contains the passed in element. By default +// ContainElement() uses Equal() to perform the match, however a matcher can be +// passed in instead: // -//Actual must be an array, slice or map. For maps, ContainElement searches -//through the map's values. +// Expect([]string{"Foo", "FooBar"}).Should(ContainElement(ContainSubstring("Bar"))) // -//If you want to have a copy of the matching element(s) found you can pass a -//pointer to a variable of the appropriate type. If the variable isn't a slice -//or map, then exactly one match will be expected and returned. If the variable -//is a slice or map, then at least one match is expected and all matches will be -//stored in the variable. +// Actual must be an array, slice or map. For maps, ContainElement searches +// through the map's values. // -// var findings []string -// Expect([]string{"Foo", "FooBar"}).Should(ContainElement(ContainSubString("Bar", &findings))) +// If you want to have a copy of the matching element(s) found you can pass a +// pointer to a variable of the appropriate type. If the variable isn't a slice +// or map, then exactly one match will be expected and returned. If the variable +// is a slice or map, then at least one match is expected and all matches will be +// stored in the variable. +// +// var findings []string +// Expect([]string{"Foo", "FooBar"}).Should(ContainElement(ContainSubString("Bar", &findings))) func ContainElement(element interface{}, result ...interface{}) types.GomegaMatcher { return &matchers.ContainElementMatcher{ Element: element, @@ -289,86 +299,116 @@ func ContainElement(element interface{}, result ...interface{}) types.GomegaMatc } } -//BeElementOf succeeds if actual is contained in the passed in elements. -//BeElementOf() always uses Equal() to perform the match. -//When the passed in elements are comprised of a single element that is either an Array or Slice, BeElementOf() behaves -//as the reverse of ContainElement() that operates with Equal() to perform the match. -// Expect(2).Should(BeElementOf([]int{1, 2})) -// Expect(2).Should(BeElementOf([2]int{1, 2})) -//Otherwise, BeElementOf() provides a syntactic sugar for Or(Equal(_), Equal(_), ...): -// Expect(2).Should(BeElementOf(1, 2)) +// BeElementOf succeeds if actual is contained in the passed in elements. +// BeElementOf() always uses Equal() to perform the match. +// When the passed in elements are comprised of a single element that is either an Array or Slice, BeElementOf() behaves +// as the reverse of ContainElement() that operates with Equal() to perform the match. +// +// Expect(2).Should(BeElementOf([]int{1, 2})) +// Expect(2).Should(BeElementOf([2]int{1, 2})) // -//Actual must be typed. +// Otherwise, BeElementOf() provides a syntactic sugar for Or(Equal(_), Equal(_), ...): +// +// Expect(2).Should(BeElementOf(1, 2)) +// +// Actual must be typed. func BeElementOf(elements ...interface{}) types.GomegaMatcher { return &matchers.BeElementOfMatcher{ Elements: elements, } } -//ConsistOf succeeds if actual contains precisely the elements passed into the matcher. The ordering of the elements does not matter. -//By default ConsistOf() uses Equal() to match the elements, however custom matchers can be passed in instead. Here are some examples: +// BeKeyOf succeeds if actual is contained in the keys of the passed in map. +// BeKeyOf() always uses Equal() to perform the match between actual and the map keys. // -// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf("FooBar", "Foo")) -// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf(ContainSubstring("Bar"), "Foo")) -// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf(ContainSubstring("Foo"), ContainSubstring("Foo"))) +// Expect("foo").Should(BeKeyOf(map[string]bool{"foo": true, "bar": false})) +func BeKeyOf(element interface{}) types.GomegaMatcher { + return &matchers.BeKeyOfMatcher{ + Map: element, + } +} + +// ConsistOf succeeds if actual contains precisely the elements passed into the matcher. The ordering of the elements does not matter. +// By default ConsistOf() uses Equal() to match the elements, however custom matchers can be passed in instead. Here are some examples: // -//Actual must be an array, slice or map. For maps, ConsistOf matches against the map's values. +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf("FooBar", "Foo")) +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf(ContainSubstring("Bar"), "Foo")) +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf(ContainSubstring("Foo"), ContainSubstring("Foo"))) // -//You typically pass variadic arguments to ConsistOf (as in the examples above). However, if you need to pass in a slice you can provided that it -//is the only element passed in to ConsistOf: +// Actual must be an array, slice or map. For maps, ConsistOf matches against the map's values. // -// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf([]string{"FooBar", "Foo"})) +// You typically pass variadic arguments to ConsistOf (as in the examples above). However, if you need to pass in a slice you can provided that it +// is the only element passed in to ConsistOf: // -//Note that Go's type system does not allow you to write this as ConsistOf([]string{"FooBar", "Foo"}...) as []string and []interface{} are different types - hence the need for this special rule. +// Expect([]string{"Foo", "FooBar"}).Should(ConsistOf([]string{"FooBar", "Foo"})) +// +// Note that Go's type system does not allow you to write this as ConsistOf([]string{"FooBar", "Foo"}...) as []string and []interface{} are different types - hence the need for this special rule. func ConsistOf(elements ...interface{}) types.GomegaMatcher { return &matchers.ConsistOfMatcher{ Elements: elements, } } -//ContainElements succeeds if actual contains the passed in elements. The ordering of the elements does not matter. -//By default ContainElements() uses Equal() to match the elements, however custom matchers can be passed in instead. Here are some examples: +// HaveExactElemets succeeds if actual contains elements that precisely match the elemets passed into the matcher. The ordering of the elements does matter. +// By default HaveExactElements() uses Equal() to match the elements, however custom matchers can be passed in instead. Here are some examples: +// +// Expect([]string{"Foo", "FooBar"}).Should(HaveExactElements("Foo", "FooBar")) +// Expect([]string{"Foo", "FooBar"}).Should(HaveExactElements("Foo", ContainSubstring("Bar"))) +// Expect([]string{"Foo", "FooBar"}).Should(HaveExactElements(ContainSubstring("Foo"), ContainSubstring("Foo"))) +// +// Actual must be an array or slice. +func HaveExactElements(elements ...interface{}) types.GomegaMatcher { + return &matchers.HaveExactElementsMatcher{ + Elements: elements, + } +} + +// ContainElements succeeds if actual contains the passed in elements. The ordering of the elements does not matter. +// By default ContainElements() uses Equal() to match the elements, however custom matchers can be passed in instead. Here are some examples: // -// Expect([]string{"Foo", "FooBar"}).Should(ContainElements("FooBar")) -// Expect([]string{"Foo", "FooBar"}).Should(ContainElements(ContainSubstring("Bar"), "Foo")) +// Expect([]string{"Foo", "FooBar"}).Should(ContainElements("FooBar")) +// Expect([]string{"Foo", "FooBar"}).Should(ContainElements(ContainSubstring("Bar"), "Foo")) // -//Actual must be an array, slice or map. -//For maps, ContainElements searches through the map's values. +// Actual must be an array, slice or map. +// For maps, ContainElements searches through the map's values. func ContainElements(elements ...interface{}) types.GomegaMatcher { return &matchers.ContainElementsMatcher{ Elements: elements, } } -//HaveEach succeeds if actual solely contains elements that match the passed in element. -//Please note that if actual is empty, HaveEach always will succeed. -//By default HaveEach() uses Equal() to perform the match, however a -//matcher can be passed in instead: -// Expect([]string{"Foo", "FooBar"}).Should(HaveEach(ContainSubstring("Foo"))) +// HaveEach succeeds if actual solely contains elements that match the passed in element. +// Please note that if actual is empty, HaveEach always will succeed. +// By default HaveEach() uses Equal() to perform the match, however a +// matcher can be passed in instead: // -//Actual must be an array, slice or map. -//For maps, HaveEach searches through the map's values. +// Expect([]string{"Foo", "FooBar"}).Should(HaveEach(ContainSubstring("Foo"))) +// +// Actual must be an array, slice or map. +// For maps, HaveEach searches through the map's values. func HaveEach(element interface{}) types.GomegaMatcher { return &matchers.HaveEachMatcher{ Element: element, } } -//HaveKey succeeds if actual is a map with the passed in key. -//By default HaveKey uses Equal() to perform the match, however a -//matcher can be passed in instead: -// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKey(MatchRegexp(`.+Foo$`))) +// HaveKey succeeds if actual is a map with the passed in key. +// By default HaveKey uses Equal() to perform the match, however a +// matcher can be passed in instead: +// +// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKey(MatchRegexp(`.+Foo$`))) func HaveKey(key interface{}) types.GomegaMatcher { return &matchers.HaveKeyMatcher{ Key: key, } } -//HaveKeyWithValue succeeds if actual is a map with the passed in key and value. -//By default HaveKeyWithValue uses Equal() to perform the match, however a -//matcher can be passed in instead: -// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKeyWithValue("Foo", "Bar")) -// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKeyWithValue(MatchRegexp(`.+Foo$`), "Bar")) +// HaveKeyWithValue succeeds if actual is a map with the passed in key and value. +// By default HaveKeyWithValue uses Equal() to perform the match, however a +// matcher can be passed in instead: +// +// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKeyWithValue("Foo", "Bar")) +// Expect(map[string]string{"Foo": "Bar", "BazFoo": "Duck"}).Should(HaveKeyWithValue(MatchRegexp(`.+Foo$`), "Bar")) func HaveKeyWithValue(key interface{}, value interface{}) types.GomegaMatcher { return &matchers.HaveKeyWithValueMatcher{ Key: key, @@ -376,27 +416,27 @@ func HaveKeyWithValue(key interface{}, value interface{}) types.GomegaMatcher { } } -//HaveField succeeds if actual is a struct and the value at the passed in field -//matches the passed in matcher. By default HaveField used Equal() to perform the match, -//however a matcher can be passed in in stead. -// -//The field must be a string that resolves to the name of a field in the struct. Structs can be traversed -//using the '.' delimiter. If the field ends with '()' a method named field is assumed to exist on the struct and is invoked. -//Such methods must take no arguments and return a single value: -// -// type Book struct { -// Title string -// Author Person -// } -// type Person struct { -// FirstName string -// LastName string -// DOB time.Time -// } -// Expect(book).To(HaveField("Title", "Les Miserables")) -// Expect(book).To(HaveField("Title", ContainSubstring("Les")) -// Expect(book).To(HaveField("Author.FirstName", Equal("Victor")) -// Expect(book).To(HaveField("Author.DOB.Year()", BeNumerically("<", 1900)) +// HaveField succeeds if actual is a struct and the value at the passed in field +// matches the passed in matcher. By default HaveField used Equal() to perform the match, +// however a matcher can be passed in in stead. +// +// The field must be a string that resolves to the name of a field in the struct. Structs can be traversed +// using the '.' delimiter. If the field ends with '()' a method named field is assumed to exist on the struct and is invoked. +// Such methods must take no arguments and return a single value: +// +// type Book struct { +// Title string +// Author Person +// } +// type Person struct { +// FirstName string +// LastName string +// DOB time.Time +// } +// Expect(book).To(HaveField("Title", "Les Miserables")) +// Expect(book).To(HaveField("Title", ContainSubstring("Les")) +// Expect(book).To(HaveField("Author.FirstName", Equal("Victor")) +// Expect(book).To(HaveField("Author.DOB.Year()", BeNumerically("<", 1900)) func HaveField(field string, expected interface{}) types.GomegaMatcher { return &matchers.HaveFieldMatcher{ Field: field, @@ -410,7 +450,7 @@ func HaveField(field string, expected interface{}) types.GomegaMatcher { // HaveExistingField can be combined with HaveField in order to cover use cases // with optional fields. HaveField alone would trigger an error in such situations. // -// Expect(MrHarmless).NotTo(And(HaveExistingField("Title"), HaveField("Title", "Supervillain"))) +// Expect(MrHarmless).NotTo(And(HaveExistingField("Title"), HaveField("Title", "Supervillain"))) func HaveExistingField(field string) types.GomegaMatcher { return &matchers.HaveExistingFieldMatcher{ Field: field, @@ -428,26 +468,27 @@ func HaveExistingField(field string) types.GomegaMatcher { // be a pointer (as gstruct.PointTo does) but instead also accepts non-pointer // and even interface values. // -// actual := 42 -// Expect(actual).To(HaveValue(42)) -// Expect(&actual).To(HaveValue(42)) +// actual := 42 +// Expect(actual).To(HaveValue(42)) +// Expect(&actual).To(HaveValue(42)) func HaveValue(matcher types.GomegaMatcher) types.GomegaMatcher { return &matchers.HaveValueMatcher{ Matcher: matcher, } } -//BeNumerically performs numerical assertions in a type-agnostic way. -//Actual and expected should be numbers, though the specific type of -//number is irrelevant (float32, float64, uint8, etc...). +// BeNumerically performs numerical assertions in a type-agnostic way. +// Actual and expected should be numbers, though the specific type of +// number is irrelevant (float32, float64, uint8, etc...). +// +// There are six, self-explanatory, supported comparators: // -//There are six, self-explanatory, supported comparators: -// Expect(1.0).Should(BeNumerically("==", 1)) -// Expect(1.0).Should(BeNumerically("~", 0.999, 0.01)) -// Expect(1.0).Should(BeNumerically(">", 0.9)) -// Expect(1.0).Should(BeNumerically(">=", 1.0)) -// Expect(1.0).Should(BeNumerically("<", 3)) -// Expect(1.0).Should(BeNumerically("<=", 1.0)) +// Expect(1.0).Should(BeNumerically("==", 1)) +// Expect(1.0).Should(BeNumerically("~", 0.999, 0.01)) +// Expect(1.0).Should(BeNumerically(">", 0.9)) +// Expect(1.0).Should(BeNumerically(">=", 1.0)) +// Expect(1.0).Should(BeNumerically("<", 3)) +// Expect(1.0).Should(BeNumerically("<=", 1.0)) func BeNumerically(comparator string, compareTo ...interface{}) types.GomegaMatcher { return &matchers.BeNumericallyMatcher{ Comparator: comparator, @@ -455,10 +496,11 @@ func BeNumerically(comparator string, compareTo ...interface{}) types.GomegaMatc } } -//BeTemporally compares time.Time's like BeNumerically -//Actual and expected must be time.Time. The comparators are the same as for BeNumerically -// Expect(time.Now()).Should(BeTemporally(">", time.Time{})) -// Expect(time.Now()).Should(BeTemporally("~", time.Now(), time.Second)) +// BeTemporally compares time.Time's like BeNumerically +// Actual and expected must be time.Time. The comparators are the same as for BeNumerically +// +// Expect(time.Now()).Should(BeTemporally(">", time.Time{})) +// Expect(time.Now()).Should(BeTemporally("~", time.Now(), time.Second)) func BeTemporally(comparator string, compareTo time.Time, threshold ...time.Duration) types.GomegaMatcher { return &matchers.BeTemporallyMatcher{ Comparator: comparator, @@ -467,58 +509,61 @@ func BeTemporally(comparator string, compareTo time.Time, threshold ...time.Dura } } -//BeAssignableToTypeOf succeeds if actual is assignable to the type of expected. -//It will return an error when one of the values is nil. -// Expect(0).Should(BeAssignableToTypeOf(0)) // Same values -// Expect(5).Should(BeAssignableToTypeOf(-1)) // different values same type -// Expect("foo").Should(BeAssignableToTypeOf("bar")) // different values same type -// Expect(struct{ Foo string }{}).Should(BeAssignableToTypeOf(struct{ Foo string }{})) +// BeAssignableToTypeOf succeeds if actual is assignable to the type of expected. +// It will return an error when one of the values is nil. +// +// Expect(0).Should(BeAssignableToTypeOf(0)) // Same values +// Expect(5).Should(BeAssignableToTypeOf(-1)) // different values same type +// Expect("foo").Should(BeAssignableToTypeOf("bar")) // different values same type +// Expect(struct{ Foo string }{}).Should(BeAssignableToTypeOf(struct{ Foo string }{})) func BeAssignableToTypeOf(expected interface{}) types.GomegaMatcher { return &matchers.AssignableToTypeOfMatcher{ Expected: expected, } } -//Panic succeeds if actual is a function that, when invoked, panics. -//Actual must be a function that takes no arguments and returns no results. +// Panic succeeds if actual is a function that, when invoked, panics. +// Actual must be a function that takes no arguments and returns no results. func Panic() types.GomegaMatcher { return &matchers.PanicMatcher{} } -//PanicWith succeeds if actual is a function that, when invoked, panics with a specific value. -//Actual must be a function that takes no arguments and returns no results. +// PanicWith succeeds if actual is a function that, when invoked, panics with a specific value. +// Actual must be a function that takes no arguments and returns no results. +// +// By default PanicWith uses Equal() to perform the match, however a +// matcher can be passed in instead: // -//By default PanicWith uses Equal() to perform the match, however a -//matcher can be passed in instead: -// Expect(fn).Should(PanicWith(MatchRegexp(`.+Foo$`))) +// Expect(fn).Should(PanicWith(MatchRegexp(`.+Foo$`))) func PanicWith(expected interface{}) types.GomegaMatcher { return &matchers.PanicMatcher{Expected: expected} } -//BeAnExistingFile succeeds if a file exists. -//Actual must be a string representing the abs path to the file being checked. +// BeAnExistingFile succeeds if a file exists. +// Actual must be a string representing the abs path to the file being checked. func BeAnExistingFile() types.GomegaMatcher { return &matchers.BeAnExistingFileMatcher{} } -//BeARegularFile succeeds if a file exists and is a regular file. -//Actual must be a string representing the abs path to the file being checked. +// BeARegularFile succeeds if a file exists and is a regular file. +// Actual must be a string representing the abs path to the file being checked. func BeARegularFile() types.GomegaMatcher { return &matchers.BeARegularFileMatcher{} } -//BeADirectory succeeds if a file exists and is a directory. -//Actual must be a string representing the abs path to the file being checked. +// BeADirectory succeeds if a file exists and is a directory. +// Actual must be a string representing the abs path to the file being checked. func BeADirectory() types.GomegaMatcher { return &matchers.BeADirectoryMatcher{} } -//HaveHTTPStatus succeeds if the Status or StatusCode field of an HTTP response matches. -//Actual must be either a *http.Response or *httptest.ResponseRecorder. -//Expected must be either an int or a string. -// Expect(resp).Should(HaveHTTPStatus(http.StatusOK)) // asserts that resp.StatusCode == 200 -// Expect(resp).Should(HaveHTTPStatus("404 Not Found")) // asserts that resp.Status == "404 Not Found" -// Expect(resp).Should(HaveHTTPStatus(http.StatusOK, http.StatusNoContent)) // asserts that resp.StatusCode == 200 || resp.StatusCode == 204 +// HaveHTTPStatus succeeds if the Status or StatusCode field of an HTTP response matches. +// Actual must be either a *http.Response or *httptest.ResponseRecorder. +// Expected must be either an int or a string. +// +// Expect(resp).Should(HaveHTTPStatus(http.StatusOK)) // asserts that resp.StatusCode == 200 +// Expect(resp).Should(HaveHTTPStatus("404 Not Found")) // asserts that resp.Status == "404 Not Found" +// Expect(resp).Should(HaveHTTPStatus(http.StatusOK, http.StatusNoContent)) // asserts that resp.StatusCode == 200 || resp.StatusCode == 204 func HaveHTTPStatus(expected ...interface{}) types.GomegaMatcher { return &matchers.HaveHTTPStatusMatcher{Expected: expected} } @@ -541,63 +586,70 @@ func HaveHTTPBody(expected interface{}) types.GomegaMatcher { return &matchers.HaveHTTPBodyMatcher{Expected: expected} } -//And succeeds only if all of the given matchers succeed. -//The matchers are tried in order, and will fail-fast if one doesn't succeed. -// Expect("hi").To(And(HaveLen(2), Equal("hi")) +// And succeeds only if all of the given matchers succeed. +// The matchers are tried in order, and will fail-fast if one doesn't succeed. +// +// Expect("hi").To(And(HaveLen(2), Equal("hi")) // -//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +// And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. func And(ms ...types.GomegaMatcher) types.GomegaMatcher { return &matchers.AndMatcher{Matchers: ms} } -//SatisfyAll is an alias for And(). -// Expect("hi").Should(SatisfyAll(HaveLen(2), Equal("hi"))) +// SatisfyAll is an alias for And(). +// +// Expect("hi").Should(SatisfyAll(HaveLen(2), Equal("hi"))) func SatisfyAll(matchers ...types.GomegaMatcher) types.GomegaMatcher { return And(matchers...) } -//Or succeeds if any of the given matchers succeed. -//The matchers are tried in order and will return immediately upon the first successful match. -// Expect("hi").To(Or(HaveLen(3), HaveLen(2)) +// Or succeeds if any of the given matchers succeed. +// The matchers are tried in order and will return immediately upon the first successful match. +// +// Expect("hi").To(Or(HaveLen(3), HaveLen(2)) // -//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +// And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. func Or(ms ...types.GomegaMatcher) types.GomegaMatcher { return &matchers.OrMatcher{Matchers: ms} } -//SatisfyAny is an alias for Or(). -// Expect("hi").SatisfyAny(Or(HaveLen(3), HaveLen(2)) +// SatisfyAny is an alias for Or(). +// +// Expect("hi").SatisfyAny(Or(HaveLen(3), HaveLen(2)) func SatisfyAny(matchers ...types.GomegaMatcher) types.GomegaMatcher { return Or(matchers...) } -//Not negates the given matcher; it succeeds if the given matcher fails. -// Expect(1).To(Not(Equal(2)) +// Not negates the given matcher; it succeeds if the given matcher fails. +// +// Expect(1).To(Not(Equal(2)) // -//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +// And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. func Not(matcher types.GomegaMatcher) types.GomegaMatcher { return &matchers.NotMatcher{Matcher: matcher} } -//WithTransform applies the `transform` to the actual value and matches it against `matcher`. -//The given transform must be either a function of one parameter that returns one value or a +// WithTransform applies the `transform` to the actual value and matches it against `matcher`. +// The given transform must be either a function of one parameter that returns one value or a // function of one parameter that returns two values, where the second value must be of the // error type. -// var plus1 = func(i int) int { return i + 1 } -// Expect(1).To(WithTransform(plus1, Equal(2)) // -// var failingplus1 = func(i int) (int, error) { return 42, "this does not compute" } -// Expect(1).To(WithTransform(failingplus1, Equal(2))) +// var plus1 = func(i int) int { return i + 1 } +// Expect(1).To(WithTransform(plus1, Equal(2)) // -//And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. +// var failingplus1 = func(i int) (int, error) { return 42, "this does not compute" } +// Expect(1).To(WithTransform(failingplus1, Equal(2))) +// +// And(), Or(), Not() and WithTransform() allow matchers to be composed into complex expressions. func WithTransform(transform interface{}, matcher types.GomegaMatcher) types.GomegaMatcher { return matchers.NewWithTransformMatcher(transform, matcher) } -//Satisfy matches the actual value against the `predicate` function. -//The given predicate must be a function of one paramter that returns bool. -// var isEven = func(i int) bool { return i%2 == 0 } -// Expect(2).To(Satisfy(isEven)) +// Satisfy matches the actual value against the `predicate` function. +// The given predicate must be a function of one paramter that returns bool. +// +// var isEven = func(i int) bool { return i%2 == 0 } +// Expect(2).To(Satisfy(isEven)) func Satisfy(predicate interface{}) types.GomegaMatcher { return matchers.NewSatisfyMatcher(predicate) } diff --git a/vendor/github.com/onsi/gomega/matchers/be_key_of_matcher.go b/vendor/github.com/onsi/gomega/matchers/be_key_of_matcher.go new file mode 100644 index 000000000..449a291ef --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/be_key_of_matcher.go @@ -0,0 +1,45 @@ +package matchers + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" +) + +type BeKeyOfMatcher struct { + Map interface{} +} + +func (matcher *BeKeyOfMatcher) Match(actual interface{}) (success bool, err error) { + if !isMap(matcher.Map) { + return false, fmt.Errorf("BeKeyOf matcher needs expected to be a map type") + } + + if reflect.TypeOf(actual) == nil { + return false, fmt.Errorf("BeKeyOf matcher expects actual to be typed") + } + + var lastError error + for _, key := range reflect.ValueOf(matcher.Map).MapKeys() { + matcher := &EqualMatcher{Expected: key.Interface()} + success, err := matcher.Match(actual) + if err != nil { + lastError = err + continue + } + if success { + return true, nil + } + } + + return false, lastError +} + +func (matcher *BeKeyOfMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, "to be a key of", presentable(valuesOf(matcher.Map))) +} + +func (matcher *BeKeyOfMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be a key of", presentable(valuesOf(matcher.Map))) +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_exact_elements.go b/vendor/github.com/onsi/gomega/matchers/have_exact_elements.go new file mode 100644 index 000000000..19d8f3d1d --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_exact_elements.go @@ -0,0 +1,75 @@ +package matchers + +import ( + "fmt" + + "github.com/onsi/gomega/format" +) + +type mismatchFailure struct { + failure string + index int +} + +type HaveExactElementsMatcher struct { + Elements []interface{} + mismatchFailures []mismatchFailure + missingIndex int + extraIndex int +} + +func (matcher *HaveExactElementsMatcher) Match(actual interface{}) (success bool, err error) { + if isMap(actual) { + return false, fmt.Errorf("error") + } + + matchers := matchers(matcher.Elements) + values := valuesOf(actual) + + lenMatchers := len(matchers) + lenValues := len(values) + + for i := 0; i < lenMatchers || i < lenValues; i++ { + if i >= lenMatchers { + matcher.extraIndex = i + continue + } + + if i >= lenValues { + matcher.missingIndex = i + return + } + + elemMatcher := matchers[i].(omegaMatcher) + match, err := elemMatcher.Match(values[i]) + if err != nil || !match { + matcher.mismatchFailures = append(matcher.mismatchFailures, mismatchFailure{ + index: i, + failure: elemMatcher.FailureMessage(values[i]), + }) + } + } + + return matcher.missingIndex+matcher.extraIndex+len(matcher.mismatchFailures) == 0, nil +} + +func (matcher *HaveExactElementsMatcher) FailureMessage(actual interface{}) (message string) { + message = format.Message(actual, "to have exact elements with", presentable(matcher.Elements)) + if matcher.missingIndex > 0 { + message = fmt.Sprintf("%s\nthe missing elements start from index %d", message, matcher.missingIndex) + } + if matcher.extraIndex > 0 { + message = fmt.Sprintf("%s\nthe extra elements start from index %d", message, matcher.extraIndex) + } + if len(matcher.mismatchFailures) != 0 { + message = fmt.Sprintf("%s\nthe mismatch indexes were:", message) + } + for _, mismatch := range matcher.mismatchFailures { + message = fmt.Sprintf("%s\n%d: %s", message, mismatch.index, mismatch.failure) + } + return +} + +func (matcher *HaveExactElementsMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to contain elements", presentable(matcher.Elements)) +} diff --git a/vendor/github.com/onsi/gomega/matchers/match_error_matcher.go b/vendor/github.com/onsi/gomega/matchers/match_error_matcher.go index c8993a86d..827475ea5 100644 --- a/vendor/github.com/onsi/gomega/matchers/match_error_matcher.go +++ b/vendor/github.com/onsi/gomega/matchers/match_error_matcher.go @@ -25,7 +25,17 @@ func (matcher *MatchErrorMatcher) Match(actual interface{}) (success bool, err e expected := matcher.Expected if isError(expected) { - return reflect.DeepEqual(actualErr, expected) || errors.Is(actualErr, expected.(error)), nil + // first try the built-in errors.Is + if errors.Is(actualErr, expected.(error)) { + return true, nil + } + // if not, try DeepEqual along the error chain + for unwrapped := actualErr; unwrapped != nil; unwrapped = errors.Unwrap(unwrapped) { + if reflect.DeepEqual(unwrapped, expected) { + return true, nil + } + } + return false, nil } if isString(expected) { diff --git a/vendor/github.com/onsi/gomega/matchers/succeed_matcher.go b/vendor/github.com/onsi/gomega/matchers/succeed_matcher.go index 721ed5529..da5a39594 100644 --- a/vendor/github.com/onsi/gomega/matchers/succeed_matcher.go +++ b/vendor/github.com/onsi/gomega/matchers/succeed_matcher.go @@ -1,11 +1,16 @@ package matchers import ( + "errors" "fmt" "github.com/onsi/gomega/format" ) +type formattedGomegaError interface { + FormattedGomegaError() string +} + type SucceedMatcher struct { } @@ -25,6 +30,10 @@ func (matcher *SucceedMatcher) Match(actual interface{}) (success bool, err erro } func (matcher *SucceedMatcher) FailureMessage(actual interface{}) (message string) { + var fgErr formattedGomegaError + if errors.As(actual.(error), &fgErr) { + return fgErr.FormattedGomegaError() + } return fmt.Sprintf("Expected success, but got an error:\n%s\n%s", format.Object(actual, 1), format.IndentString(actual.(error).Error(), 1)) } diff --git a/vendor/github.com/onsi/gomega/types/types.go b/vendor/github.com/onsi/gomega/types/types.go index c315ef065..7c7adb941 100644 --- a/vendor/github.com/onsi/gomega/types/types.go +++ b/vendor/github.com/onsi/gomega/types/types.go @@ -1,12 +1,13 @@ package types import ( + "context" "time" ) type GomegaFailHandler func(message string, callerSkip ...int) -//A simple *testing.T interface wrapper +// A simple *testing.T interface wrapper type GomegaTestingT interface { Helper() Fatalf(format string, args ...interface{}) @@ -18,11 +19,11 @@ type Gomega interface { Expect(actual interface{}, extra ...interface{}) Assertion ExpectWithOffset(offset int, actual interface{}, extra ...interface{}) Assertion - Eventually(actual interface{}, intervals ...interface{}) AsyncAssertion - EventuallyWithOffset(offset int, actual interface{}, intervals ...interface{}) AsyncAssertion + Eventually(actualOrCtx interface{}, args ...interface{}) AsyncAssertion + EventuallyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion - Consistently(actual interface{}, intervals ...interface{}) AsyncAssertion - ConsistentlyWithOffset(offset int, actual interface{}, intervals ...interface{}) AsyncAssertion + Consistently(actualOrCtx interface{}, args ...interface{}) AsyncAssertion + ConsistentlyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion SetDefaultEventuallyTimeout(time.Duration) SetDefaultEventuallyPollingInterval(time.Duration) @@ -30,9 +31,9 @@ type Gomega interface { SetDefaultConsistentlyPollingInterval(time.Duration) } -//All Gomega matchers must implement the GomegaMatcher interface +// All Gomega matchers must implement the GomegaMatcher interface // -//For details on writing custom matchers, check out: http://onsi.github.io/gomega/#adding-your-own-matchers +// For details on writing custom matchers, check out: http://onsi.github.io/gomega/#adding-your-own-matchers type GomegaMatcher interface { Match(actual interface{}) (success bool, err error) FailureMessage(actual interface{}) (message string) @@ -70,6 +71,11 @@ type AsyncAssertion interface { WithOffset(offset int) AsyncAssertion WithTimeout(interval time.Duration) AsyncAssertion WithPolling(interval time.Duration) AsyncAssertion + Within(timeout time.Duration) AsyncAssertion + ProbeEvery(interval time.Duration) AsyncAssertion + WithContext(ctx context.Context) AsyncAssertion + WithArguments(argsToForward ...interface{}) AsyncAssertion + MustPassRepeatedly(count int) AsyncAssertion } // Assertions are returned by Ω and Expect and enable assertions against Gomega matchers diff --git a/vendor/github.com/rs/cors/.travis.yml b/vendor/github.com/rs/cors/.travis.yml deleted file mode 100644 index f59a345fd..000000000 --- a/vendor/github.com/rs/cors/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go -go: -- "1.13" -- tip -matrix: - allow_failures: - - go: tip diff --git a/vendor/github.com/rs/cors/README.md b/vendor/github.com/rs/cors/README.md index ecc83b295..0ad3e94e1 100644 --- a/vendor/github.com/rs/cors/README.md +++ b/vendor/github.com/rs/cors/README.md @@ -88,13 +88,14 @@ handler = c.Handler(handler) * **AllowedOrigins** `[]string`: A list of origins a cross-domain request can be executed from. If the special `*` value is present in the list, all origins will be allowed. An origin may contain a wildcard (`*`) to replace 0 or more characters (i.e.: `http://*.domain.com`). Usage of wildcards implies a small performance penality. Only one wildcard can be used per origin. The default value is `*`. * **AllowOriginFunc** `func (origin string) bool`: A custom function to validate the origin. It takes the origin as an argument and returns true if allowed, or false otherwise. If this option is set, the content of `AllowedOrigins` is ignored. -* **AllowOriginRequestFunc** `func (r *http.Request origin string) bool`: A custom function to validate the origin. It takes the HTTP Request object and the origin as argument and returns true if allowed or false otherwise. If this option is set, the content of `AllowedOrigins` and `AllowOriginFunc` is ignored +* **AllowOriginRequestFunc** `func (r *http.Request, origin string) bool`: A custom function to validate the origin. It takes the HTTP Request object and the origin as argument and returns true if allowed or false otherwise. If this option is set, the content of `AllowedOrigins` and `AllowOriginFunc` is ignored * **AllowedMethods** `[]string`: A list of methods the client is allowed to use with cross-domain requests. Default value is simple methods (`GET` and `POST`). * **AllowedHeaders** `[]string`: A list of non simple headers the client is allowed to use with cross-domain requests. * **ExposedHeaders** `[]string`: Indicates which headers are safe to expose to the API of a CORS API specification * **AllowCredentials** `bool`: Indicates whether the request can include user credentials like cookies, HTTP authentication or client side SSL certificates. The default is `false`. * **MaxAge** `int`: Indicates how long (in seconds) the results of a preflight request can be cached. The default is `0` which stands for no max age. * **OptionsPassthrough** `bool`: Instructs preflight to let other potential next handlers to process the `OPTIONS` method. Turn this on if your application handles `OPTIONS`. +* **OptionsSuccessStatus** `int`: Provides a status code to use for successful OPTIONS requests. Default value is `http.StatusNoContent` (`204`). * **Debug** `bool`: Debugging flag adds additional output to debug server side CORS issues. See [API documentation](http://godoc.org/github.com/rs/cors) for more info. diff --git a/vendor/github.com/rs/cors/cors.go b/vendor/github.com/rs/cors/cors.go index 41d5f7cb6..a47b7df87 100644 --- a/vendor/github.com/rs/cors/cors.go +++ b/vendor/github.com/rs/cors/cors.go @@ -62,9 +62,15 @@ type Options struct { // AllowCredentials indicates whether the request can include user credentials like // cookies, HTTP authentication or client side SSL certificates. AllowCredentials bool + // AllowPrivateNetwork indicates whether to accept cross-origin requests over a + // private network. + AllowPrivateNetwork bool // OptionsPassthrough instructs preflight to let other potential next handlers to // process the OPTIONS method. Turn this on if your application handles OPTIONS. OptionsPassthrough bool + // Provides a status code to use for successful OPTIONS requests. + // Default value is http.StatusNoContent (204). + OptionsSuccessStatus int // Debugging flag adds additional output to debug server side CORS issues Debug bool } @@ -97,8 +103,11 @@ type Cors struct { allowedOriginsAll bool // Set to true when allowed headers contains a "*" allowedHeadersAll bool - allowCredentials bool - optionPassthrough bool + // Status code to use for successful OPTIONS requests + optionsSuccessStatus int + allowCredentials bool + allowPrivateNetwork bool + optionPassthrough bool } // New creates a new Cors handler with the provided options. @@ -108,6 +117,7 @@ func New(options Options) *Cors { allowOriginFunc: options.AllowOriginFunc, allowOriginRequestFunc: options.AllowOriginRequestFunc, allowCredentials: options.AllowCredentials, + allowPrivateNetwork: options.AllowPrivateNetwork, maxAge: options.MaxAge, optionPassthrough: options.OptionsPassthrough, } @@ -171,6 +181,13 @@ func New(options Options) *Cors { c.allowedMethods = convert(options.AllowedMethods, strings.ToUpper) } + // Options Success Status Code + if options.OptionsSuccessStatus == 0 { + c.optionsSuccessStatus = http.StatusNoContent + } else { + c.optionsSuccessStatus = options.OptionsSuccessStatus + } + return c } @@ -211,7 +228,7 @@ func (c *Cors) Handler(h http.Handler) http.Handler { if c.optionPassthrough { h.ServeHTTP(w, r) } else { - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(c.optionsSuccessStatus) } } else { c.logf("Handler: Actual request") @@ -226,6 +243,8 @@ func (c *Cors) HandlerFunc(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" { c.logf("HandlerFunc: Preflight request") c.handlePreflight(w, r) + + w.WriteHeader(c.optionsSuccessStatus) } else { c.logf("HandlerFunc: Actual request") c.handleActualRequest(w, r) @@ -244,7 +263,7 @@ func (c *Cors) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.Handl if c.optionPassthrough { next(w, r) } else { - w.WriteHeader(http.StatusNoContent) + w.WriteHeader(c.optionsSuccessStatus) } } else { c.logf("ServeHTTP: Actual request") @@ -268,6 +287,9 @@ func (c *Cors) handlePreflight(w http.ResponseWriter, r *http.Request) { headers.Add("Vary", "Origin") headers.Add("Vary", "Access-Control-Request-Method") headers.Add("Vary", "Access-Control-Request-Headers") + if c.allowPrivateNetwork { + headers.Add("Vary", "Access-Control-Request-Private-Network") + } if origin == "" { c.logf(" Preflight aborted: empty origin") @@ -305,6 +327,9 @@ func (c *Cors) handlePreflight(w http.ResponseWriter, r *http.Request) { if c.allowCredentials { headers.Set("Access-Control-Allow-Credentials", "true") } + if c.allowPrivateNetwork && r.Header.Get("Access-Control-Request-Private-Network") == "true" { + headers.Set("Access-Control-Allow-Private-Network", "true") + } if c.maxAge > 0 { headers.Set("Access-Control-Max-Age", strconv.Itoa(c.maxAge)) } diff --git a/vendor/github.com/sigurn/crc8/crc8.go b/vendor/github.com/sigurn/crc8/crc8.go index e9976136d..b0444899a 100644 --- a/vendor/github.com/sigurn/crc8/crc8.go +++ b/vendor/github.com/sigurn/crc8/crc8.go @@ -3,7 +3,7 @@ // It provides parameters for the majority of well-known CRC-8 algorithms. package crc8 -import "github.com/sigurn/utils" +import "math/bits" // Params represents parameters of a CRC-8 algorithm including polynomial and initial value. // More information about algorithms parametrization and parameter descriptions @@ -35,7 +35,7 @@ var ( CRC8_WCDMA = Params{0x9B, 0x00, true, true, 0x00, 0x25, "CRC-8/WCDMA"} ) -// Table is a 256-byte table representing polinomial and algorithm settings for efficient processing. +// Table is a 256-byte table representing polynomial and algorithm settings for efficient processing. type Table struct { params Params data [256]uint8 @@ -66,26 +66,29 @@ func Init(table *Table) uint8 { // Update returns the result of adding the bytes in data to the crc. func Update(crc uint8, data []byte, table *Table) uint8 { - for _, d := range data { - if table.params.RefIn { - d = utils.ReverseByte(d) + if table.params.RefIn { + for _, d := range data { + d = bits.Reverse8(d) + crc = table.data[crc^d] + } + } else { + for _, d := range data { + crc = table.data[crc^d] } - crc = table.data[crc^d] } - return crc } // Complete returns the result of CRC calculation and post-calculation processing of the crc. func Complete(crc uint8, table *Table) uint8 { if table.params.RefOut { - crc = utils.ReverseUint8(crc) + crc = bits.Reverse8(crc) } return crc ^ table.params.XorOut } -// Checksum returns CRC checksum of data usign scpecified algorithm represented by the Table. +// Checksum returns CRC checksum of data using specified algorithm represented by the Table. func Checksum(data []byte, table *Table) uint8 { crc := Init(table) crc = Update(crc, data, table) diff --git a/vendor/github.com/sigurn/utils/utils.go b/vendor/github.com/sigurn/utils/utils.go deleted file mode 100644 index b837ee89c..000000000 --- a/vendor/github.com/sigurn/utils/utils.go +++ /dev/null @@ -1,25 +0,0 @@ -package utils - -func ReverseByte(val byte) byte { - var rval byte = 0 - for i := uint(0); i < 8; i++ { - if val&(1<> i - } - } - return rval -} - -func ReverseUint8(val uint8) uint8 { - return ReverseByte(val) -} - -func ReverseUint16(val uint16) uint16 { - var rval uint16 = 0 - for i := uint(0); i < 16; i++ { - if val&(uint16(1)<> i - } - } - return rval -} diff --git a/vendor/go.chromium.org/luci/common/runtime/goroutine/goroutine_id.go b/vendor/go.chromium.org/luci/common/runtime/goroutine/goroutine_id.go index c4a7fc81c..ff4b5d70d 100644 --- a/vendor/go.chromium.org/luci/common/runtime/goroutine/goroutine_id.go +++ b/vendor/go.chromium.org/luci/common/runtime/goroutine/goroutine_id.go @@ -58,7 +58,7 @@ func CurID() ID { } var littleBuf = sync.Pool{ - New: func() interface{} { + New: func() any { buf := make([]byte, 64) return &buf }, diff --git a/vendor/go.opencensus.io/Makefile b/vendor/go.opencensus.io/Makefile index b3ce3df30..d896edc99 100644 --- a/vendor/go.opencensus.io/Makefile +++ b/vendor/go.opencensus.io/Makefile @@ -91,7 +91,7 @@ embedmd: .PHONY: install-tools install-tools: - go get -u golang.org/x/lint/golint - go get -u golang.org/x/tools/cmd/cover - go get -u golang.org/x/tools/cmd/goimports - go get -u github.com/rakyll/embedmd + go install golang.org/x/lint/golint@latest + go install golang.org/x/tools/cmd/cover@latest + go install golang.org/x/tools/cmd/goimports@latest + go install github.com/rakyll/embedmd@latest diff --git a/vendor/go.opencensus.io/opencensus.go b/vendor/go.opencensus.io/opencensus.go index e5e4b4368..11e31f421 100644 --- a/vendor/go.opencensus.io/opencensus.go +++ b/vendor/go.opencensus.io/opencensus.go @@ -17,5 +17,5 @@ package opencensus // import "go.opencensus.io" // Version is the current release version of OpenCensus in use. func Version() string { - return "0.23.0" + return "0.24.0" } diff --git a/vendor/go.opencensus.io/trace/doc.go b/vendor/go.opencensus.io/trace/doc.go index 04b1ee4f3..7a1616a55 100644 --- a/vendor/go.opencensus.io/trace/doc.go +++ b/vendor/go.opencensus.io/trace/doc.go @@ -18,24 +18,23 @@ Package trace contains support for OpenCensus distributed tracing. The following assumes a basic familiarity with OpenCensus concepts. See http://opencensus.io - -Exporting Traces +# Exporting Traces To export collected tracing data, register at least one exporter. You can use one of the provided exporters or write your own. - trace.RegisterExporter(exporter) + trace.RegisterExporter(exporter) By default, traces will be sampled relatively rarely. To change the sampling frequency for your entire program, call ApplyConfig. Use a ProbabilitySampler to sample a subset of traces, or use AlwaysSample to collect a trace on every run: - trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) Be careful about using trace.AlwaysSample in a production application with significant traffic: a new trace will be started and exported for every request. -Adding Spans to a Trace +# Adding Spans to a Trace A trace consists of a tree of spans. In Go, the current span is carried in a context.Context. @@ -44,8 +43,8 @@ It is common to want to capture all the activity of a function call in a span. F this to work, the function must take a context.Context as a parameter. Add these two lines to the top of the function: - ctx, span := trace.StartSpan(ctx, "example.com/Run") - defer span.End() + ctx, span := trace.StartSpan(ctx, "example.com/Run") + defer span.End() StartSpan will create a new top-level span if the context doesn't contain another span, otherwise it will create a child span. diff --git a/vendor/go.opencensus.io/trace/lrumap.go b/vendor/go.opencensus.io/trace/lrumap.go index 908c2497e..80095a5f6 100644 --- a/vendor/go.opencensus.io/trace/lrumap.go +++ b/vendor/go.opencensus.io/trace/lrumap.go @@ -44,7 +44,7 @@ func (lm lruMap) len() int { } func (lm lruMap) keys() []interface{} { - keys := make([]interface{}, len(lm.cacheKeys)) + keys := make([]interface{}, 0, len(lm.cacheKeys)) for k := range lm.cacheKeys { keys = append(keys, k) } diff --git a/vendor/go.opencensus.io/trace/trace_go11.go b/vendor/go.opencensus.io/trace/trace_go11.go index b7d8aaf28..b8fc1e495 100644 --- a/vendor/go.opencensus.io/trace/trace_go11.go +++ b/vendor/go.opencensus.io/trace/trace_go11.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build go1.11 // +build go1.11 package trace diff --git a/vendor/go.opencensus.io/trace/trace_nongo11.go b/vendor/go.opencensus.io/trace/trace_nongo11.go index e25419859..da488fc87 100644 --- a/vendor/go.opencensus.io/trace/trace_nongo11.go +++ b/vendor/go.opencensus.io/trace/trace_nongo11.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !go1.11 // +build !go1.11 package trace diff --git a/vendor/go.uber.org/multierr/CHANGELOG.md b/vendor/go.uber.org/multierr/CHANGELOG.md index 3ba05276f..d2c8aadaf 100644 --- a/vendor/go.uber.org/multierr/CHANGELOG.md +++ b/vendor/go.uber.org/multierr/CHANGELOG.md @@ -1,6 +1,14 @@ Releases ======== +v1.9.0 (2022-12-12) +=================== + +- Add `AppendFunc` that allow passsing functions to similar to + `AppendInvoke`. + +- Bump up yaml.v3 dependency to 3.0.1. + v1.8.0 (2022-02-28) =================== diff --git a/vendor/go.uber.org/multierr/error.go b/vendor/go.uber.org/multierr/error.go index f45af149c..cdd91ae56 100644 --- a/vendor/go.uber.org/multierr/error.go +++ b/vendor/go.uber.org/multierr/error.go @@ -20,106 +20,109 @@ // Package multierr allows combining one or more errors together. // -// Overview +// # Overview // // Errors can be combined with the use of the Combine function. // -// multierr.Combine( -// reader.Close(), -// writer.Close(), -// conn.Close(), -// ) +// multierr.Combine( +// reader.Close(), +// writer.Close(), +// conn.Close(), +// ) // // If only two errors are being combined, the Append function may be used // instead. // -// err = multierr.Append(reader.Close(), writer.Close()) +// err = multierr.Append(reader.Close(), writer.Close()) // // The underlying list of errors for a returned error object may be retrieved // with the Errors function. // -// errors := multierr.Errors(err) -// if len(errors) > 0 { -// fmt.Println("The following errors occurred:", errors) -// } +// errors := multierr.Errors(err) +// if len(errors) > 0 { +// fmt.Println("The following errors occurred:", errors) +// } // -// Appending from a loop +// # Appending from a loop // // You sometimes need to append into an error from a loop. // -// var err error -// for _, item := range items { -// err = multierr.Append(err, process(item)) -// } +// var err error +// for _, item := range items { +// err = multierr.Append(err, process(item)) +// } // // Cases like this may require knowledge of whether an individual instance // failed. This usually requires introduction of a new variable. // -// var err error -// for _, item := range items { -// if perr := process(item); perr != nil { -// log.Warn("skipping item", item) -// err = multierr.Append(err, perr) -// } -// } +// var err error +// for _, item := range items { +// if perr := process(item); perr != nil { +// log.Warn("skipping item", item) +// err = multierr.Append(err, perr) +// } +// } // // multierr includes AppendInto to simplify cases like this. // -// var err error -// for _, item := range items { -// if multierr.AppendInto(&err, process(item)) { -// log.Warn("skipping item", item) -// } -// } +// var err error +// for _, item := range items { +// if multierr.AppendInto(&err, process(item)) { +// log.Warn("skipping item", item) +// } +// } // // This will append the error into the err variable, and return true if that // individual error was non-nil. // -// See AppendInto for more information. +// See [AppendInto] for more information. // -// Deferred Functions +// # Deferred Functions // // Go makes it possible to modify the return value of a function in a defer // block if the function was using named returns. This makes it possible to // record resource cleanup failures from deferred blocks. // -// func sendRequest(req Request) (err error) { -// conn, err := openConnection() -// if err != nil { -// return err -// } -// defer func() { -// err = multierr.Append(err, conn.Close()) -// }() -// // ... -// } +// func sendRequest(req Request) (err error) { +// conn, err := openConnection() +// if err != nil { +// return err +// } +// defer func() { +// err = multierr.Append(err, conn.Close()) +// }() +// // ... +// } // // multierr provides the Invoker type and AppendInvoke function to make cases // like the above simpler and obviate the need for a closure. The following is // roughly equivalent to the example above. // -// func sendRequest(req Request) (err error) { -// conn, err := openConnection() -// if err != nil { -// return err -// } -// defer multierr.AppendInvoke(&err, multierr.Close(conn)) -// // ... -// } +// func sendRequest(req Request) (err error) { +// conn, err := openConnection() +// if err != nil { +// return err +// } +// defer multierr.AppendInvoke(&err, multierr.Close(conn)) +// // ... +// } // -// See AppendInvoke and Invoker for more information. +// See [AppendInvoke] and [Invoker] for more information. // -// Advanced Usage +// NOTE: If you're modifying an error from inside a defer, you MUST use a named +// return value for that function. +// +// # Advanced Usage // // Errors returned by Combine and Append MAY implement the following // interface. // -// type errorGroup interface { -// // Returns a slice containing the underlying list of errors. -// // -// // This slice MUST NOT be modified by the caller. -// Errors() []error -// } +// type errorGroup interface { +// // Returns a slice containing the underlying list of errors. +// // +// // This slice MUST NOT be modified by the caller. +// Errors() []error +// } // // Note that if you need access to list of errors behind a multierr error, you // should prefer using the Errors function. That said, if you need cheap @@ -128,13 +131,13 @@ // because errors returned by Combine and Append are not guaranteed to // implement this interface. // -// var errors []error -// group, ok := err.(errorGroup) -// if ok { -// errors = group.Errors() -// } else { -// errors = []error{err} -// } +// var errors []error +// group, ok := err.(errorGroup) +// if ok { +// errors = group.Errors() +// } else { +// errors = []error{err} +// } package multierr // import "go.uber.org/multierr" import ( @@ -185,8 +188,8 @@ type errorGroup interface { // Errors returns a slice containing zero or more errors that the supplied // error is composed of. If the error is nil, a nil slice is returned. // -// err := multierr.Append(r.Close(), w.Close()) -// errors := multierr.Errors(err) +// err := multierr.Append(r.Close(), w.Close()) +// errors := multierr.Errors(err) // // If the error is not composed of other errors, the returned slice contains // just the error that was passed in. @@ -209,10 +212,7 @@ func Errors(err error) []error { return []error{err} } - errors := eg.Errors() - result := make([]error, len(errors)) - copy(result, errors) - return result + return append(([]error)(nil), eg.Errors()...) } // multiError is an error that holds one or more errors. @@ -393,8 +393,7 @@ func fromSlice(errors []error) error { // Otherwise "errors" escapes to the heap // unconditionally for all other cases. // This lets us optimize for the "no errors" case. - out := make([]error, len(errors)) - copy(out, errors) + out := append(([]error)(nil), errors...) return &multiError{errors: out} } } @@ -420,32 +419,32 @@ func fromSlice(errors []error) error { // If zero arguments were passed or if all items are nil, a nil error is // returned. // -// Combine(nil, nil) // == nil +// Combine(nil, nil) // == nil // // If only a single error was passed, it is returned as-is. // -// Combine(err) // == err +// Combine(err) // == err // // Combine skips over nil arguments so this function may be used to combine // together errors from operations that fail independently of each other. // -// multierr.Combine( -// reader.Close(), -// writer.Close(), -// pipe.Close(), -// ) +// multierr.Combine( +// reader.Close(), +// writer.Close(), +// pipe.Close(), +// ) // // If any of the passed errors is a multierr error, it will be flattened along // with the other errors. // -// multierr.Combine(multierr.Combine(err1, err2), err3) -// // is the same as -// multierr.Combine(err1, err2, err3) +// multierr.Combine(multierr.Combine(err1, err2), err3) +// // is the same as +// multierr.Combine(err1, err2, err3) // // The returned error formats into a readable multi-line error message if // formatted with %+v. // -// fmt.Sprintf("%+v", multierr.Combine(err1, err2)) +// fmt.Sprintf("%+v", multierr.Combine(err1, err2)) func Combine(errors ...error) error { return fromSlice(errors) } @@ -455,16 +454,19 @@ func Combine(errors ...error) error { // This function is a specialization of Combine for the common case where // there are only two errors. // -// err = multierr.Append(reader.Close(), writer.Close()) +// err = multierr.Append(reader.Close(), writer.Close()) // // The following pattern may also be used to record failure of deferred // operations without losing information about the original error. // -// func doSomething(..) (err error) { -// f := acquireResource() -// defer func() { -// err = multierr.Append(err, f.Close()) -// }() +// func doSomething(..) (err error) { +// f := acquireResource() +// defer func() { +// err = multierr.Append(err, f.Close()) +// }() +// +// Note that the variable MUST be a named return to append an error to it from +// the defer statement. See also [AppendInvoke]. func Append(left error, right error) error { switch { case left == nil: @@ -494,37 +496,37 @@ func Append(left error, right error) error { // AppendInto appends an error into the destination of an error pointer and // returns whether the error being appended was non-nil. // -// var err error -// multierr.AppendInto(&err, r.Close()) -// multierr.AppendInto(&err, w.Close()) +// var err error +// multierr.AppendInto(&err, r.Close()) +// multierr.AppendInto(&err, w.Close()) // // The above is equivalent to, // -// err := multierr.Append(r.Close(), w.Close()) +// err := multierr.Append(r.Close(), w.Close()) // // As AppendInto reports whether the provided error was non-nil, it may be // used to build a multierr error in a loop more ergonomically. For example: // -// var err error -// for line := range lines { -// var item Item -// if multierr.AppendInto(&err, parse(line, &item)) { -// continue -// } -// items = append(items, item) -// } +// var err error +// for line := range lines { +// var item Item +// if multierr.AppendInto(&err, parse(line, &item)) { +// continue +// } +// items = append(items, item) +// } // // Compare this with a version that relies solely on Append: // -// var err error -// for line := range lines { -// var item Item -// if parseErr := parse(line, &item); parseErr != nil { -// err = multierr.Append(err, parseErr) -// continue -// } -// items = append(items, item) -// } +// var err error +// for line := range lines { +// var item Item +// if parseErr := parse(line, &item); parseErr != nil { +// err = multierr.Append(err, parseErr) +// continue +// } +// items = append(items, item) +// } func AppendInto(into *error, err error) (errored bool) { if into == nil { // We panic if 'into' is nil. This is not documented above @@ -545,7 +547,7 @@ func AppendInto(into *error, err error) (errored bool) { // AppendInvoke to append the result of calling the function into an error. // This allows you to conveniently defer capture of failing operations. // -// See also, Close and Invoke. +// See also, [Close] and [Invoke]. type Invoker interface { Invoke() error } @@ -556,19 +558,22 @@ type Invoker interface { // // For example, // -// func processReader(r io.Reader) (err error) { -// scanner := bufio.NewScanner(r) -// defer multierr.AppendInvoke(&err, multierr.Invoke(scanner.Err)) -// for scanner.Scan() { -// // ... -// } -// // ... -// } +// func processReader(r io.Reader) (err error) { +// scanner := bufio.NewScanner(r) +// defer multierr.AppendInvoke(&err, multierr.Invoke(scanner.Err)) +// for scanner.Scan() { +// // ... +// } +// // ... +// } // // In this example, the following line will construct the Invoker right away, // but defer the invocation of scanner.Err() until the function returns. // -// defer multierr.AppendInvoke(&err, multierr.Invoke(scanner.Err)) +// defer multierr.AppendInvoke(&err, multierr.Invoke(scanner.Err)) +// +// Note that the error you're appending to from the defer statement MUST be a +// named return. type Invoke func() error // Invoke calls the supplied function and returns its result. @@ -579,19 +584,22 @@ func (i Invoke) Invoke() error { return i() } // // For example, // -// func processFile(path string) (err error) { -// f, err := os.Open(path) -// if err != nil { -// return err -// } -// defer multierr.AppendInvoke(&err, multierr.Close(f)) -// return processReader(f) -// } +// func processFile(path string) (err error) { +// f, err := os.Open(path) +// if err != nil { +// return err +// } +// defer multierr.AppendInvoke(&err, multierr.Close(f)) +// return processReader(f) +// } // // In this example, multierr.Close will construct the Invoker right away, but // defer the invocation of f.Close until the function returns. // -// defer multierr.AppendInvoke(&err, multierr.Close(f)) +// defer multierr.AppendInvoke(&err, multierr.Close(f)) +// +// Note that the error you're appending to from the defer statement MUST be a +// named return. func Close(closer io.Closer) Invoker { return Invoke(closer.Close) } @@ -601,52 +609,73 @@ func Close(closer io.Closer) Invoker { // invocation of fallible operations until a function returns, and capture the // resulting errors. // -// func doSomething(...) (err error) { -// // ... -// f, err := openFile(..) -// if err != nil { -// return err -// } +// func doSomething(...) (err error) { +// // ... +// f, err := openFile(..) +// if err != nil { +// return err +// } +// +// // multierr will call f.Close() when this function returns and +// // if the operation fails, its append its error into the +// // returned error. +// defer multierr.AppendInvoke(&err, multierr.Close(f)) // -// // multierr will call f.Close() when this function returns and -// // if the operation fails, its append its error into the -// // returned error. -// defer multierr.AppendInvoke(&err, multierr.Close(f)) +// scanner := bufio.NewScanner(f) +// // Similarly, this scheduled scanner.Err to be called and +// // inspected when the function returns and append its error +// // into the returned error. +// defer multierr.AppendInvoke(&err, multierr.Invoke(scanner.Err)) // -// scanner := bufio.NewScanner(f) -// // Similarly, this scheduled scanner.Err to be called and -// // inspected when the function returns and append its error -// // into the returned error. -// defer multierr.AppendInvoke(&err, multierr.Invoke(scanner.Err)) +// // ... +// } // -// // ... -// } +// NOTE: If used with a defer, the error variable MUST be a named return. // // Without defer, AppendInvoke behaves exactly like AppendInto. // -// err := // ... -// multierr.AppendInvoke(&err, mutltierr.Invoke(foo)) +// err := // ... +// multierr.AppendInvoke(&err, mutltierr.Invoke(foo)) // -// // ...is roughly equivalent to... +// // ...is roughly equivalent to... // -// err := // ... -// multierr.AppendInto(&err, foo()) +// err := // ... +// multierr.AppendInto(&err, foo()) // // The advantage of the indirection introduced by Invoker is to make it easy // to defer the invocation of a function. Without this indirection, the // invoked function will be evaluated at the time of the defer block rather // than when the function returns. // -// // BAD: This is likely not what the caller intended. This will evaluate -// // foo() right away and append its result into the error when the -// // function returns. -// defer multierr.AppendInto(&err, foo()) +// // BAD: This is likely not what the caller intended. This will evaluate +// // foo() right away and append its result into the error when the +// // function returns. +// defer multierr.AppendInto(&err, foo()) // -// // GOOD: This will defer invocation of foo unutil the function returns. -// defer multierr.AppendInvoke(&err, multierr.Invoke(foo)) +// // GOOD: This will defer invocation of foo unutil the function returns. +// defer multierr.AppendInvoke(&err, multierr.Invoke(foo)) // // multierr provides a few Invoker implementations out of the box for -// convenience. See Invoker for more information. +// convenience. See [Invoker] for more information. func AppendInvoke(into *error, invoker Invoker) { AppendInto(into, invoker.Invoke()) } + +// AppendFunc is a shorthand for [AppendInvoke]. +// It allows using function or method value directly +// without having to wrap it into an [Invoker] interface. +// +// func doSomething(...) (err error) { +// w, err := startWorker(...) +// if err != nil { +// return err +// } +// +// // multierr will call w.Stop() when this function returns and +// // if the operation fails, it appends its error into the +// // returned error. +// defer multierr.AppendFunc(&err, w.Stop) +// } +func AppendFunc(into *error, fn func() error) { + AppendInvoke(into, Invoke(fn)) +} diff --git a/vendor/go.uber.org/zap/CHANGELOG.md b/vendor/go.uber.org/zap/CHANGELOG.md index da36c3708..0db1f9f15 100644 --- a/vendor/go.uber.org/zap/CHANGELOG.md +++ b/vendor/go.uber.org/zap/CHANGELOG.md @@ -3,6 +3,19 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## 1.24.0 (30 Nov 2022) + +Enhancements: +* [#1148][]: Add `Level` to both `Logger` and `SugaredLogger` that reports the + current minimum enabled log level. +* [#1185][]: `SugaredLogger` turns errors to zap.Error automatically. + +Thanks to @Abirdcfly, @craigpastro, @nnnkkk7, and @sashamelentyev for their +contributions to this release. + +[#1148]: https://github.coml/uber-go/zap/pull/1148 +[#1185]: https://github.coml/uber-go/zap/pull/1185 + ## 1.23.0 (24 Aug 2022) Enhancements: diff --git a/vendor/go.uber.org/zap/array_go118.go b/vendor/go.uber.org/zap/array_go118.go index ea94f9d75..d0d2c49d6 100644 --- a/vendor/go.uber.org/zap/array_go118.go +++ b/vendor/go.uber.org/zap/array_go118.go @@ -76,9 +76,9 @@ func (os objects[T]) MarshalLogArray(arr zapcore.ArrayEncoder) error { return nil } -// objectMarshalerPtr is a constraint that specifies that the given type +// ObjectMarshalerPtr is a constraint that specifies that the given type // implements zapcore.ObjectMarshaler on a pointer receiver. -type objectMarshalerPtr[T any] interface { +type ObjectMarshalerPtr[T any] interface { *T zapcore.ObjectMarshaler } @@ -105,11 +105,11 @@ type objectMarshalerPtr[T any] interface { // // var requests []*Request = ... // logger.Info("sending requests", zap.Objects("requests", requests)) -func ObjectValues[T any, P objectMarshalerPtr[T]](key string, values []T) Field { +func ObjectValues[T any, P ObjectMarshalerPtr[T]](key string, values []T) Field { return Array(key, objectValues[T, P](values)) } -type objectValues[T any, P objectMarshalerPtr[T]] []T +type objectValues[T any, P ObjectMarshalerPtr[T]] []T func (os objectValues[T, P]) MarshalLogArray(arr zapcore.ArrayEncoder) error { for i := range os { diff --git a/vendor/go.uber.org/zap/logger.go b/vendor/go.uber.org/zap/logger.go index b5f9a99fd..cd44030d1 100644 --- a/vendor/go.uber.org/zap/logger.go +++ b/vendor/go.uber.org/zap/logger.go @@ -183,6 +183,13 @@ func (log *Logger) With(fields ...Field) *Logger { return l } +// Level reports the minimum enabled level for this logger. +// +// For NopLoggers, this is [zapcore.InvalidLevel]. +func (log *Logger) Level() zapcore.Level { + return zapcore.LevelOf(log.core) +} + // Check returns a CheckedEntry if logging a message at the specified level // is enabled. It's a completely optional optimization; in high-performance // applications, Check can help avoid allocating a slice to hold fields. diff --git a/vendor/go.uber.org/zap/options.go b/vendor/go.uber.org/zap/options.go index 1511166c0..c4f3bca3d 100644 --- a/vendor/go.uber.org/zap/options.go +++ b/vendor/go.uber.org/zap/options.go @@ -133,7 +133,8 @@ func IncreaseLevel(lvl zapcore.LevelEnabler) Option { } // OnFatal sets the action to take on fatal logs. -// Deprecated: Use WithFatalHook instead. +// +// Deprecated: Use [WithFatalHook] instead. func OnFatal(action zapcore.CheckWriteAction) Option { return WithFatalHook(action) } diff --git a/vendor/go.uber.org/zap/sink.go b/vendor/go.uber.org/zap/sink.go index df46fa87a..478c9a10f 100644 --- a/vendor/go.uber.org/zap/sink.go +++ b/vendor/go.uber.org/zap/sink.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016 Uber Technologies, Inc. +// Copyright (c) 2016-2022 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ import ( "io" "net/url" "os" + "path/filepath" "strings" "sync" @@ -34,23 +35,7 @@ import ( const schemeFile = "file" -var ( - _sinkMutex sync.RWMutex - _sinkFactories map[string]func(*url.URL) (Sink, error) // keyed by scheme -) - -func init() { - resetSinkRegistry() -} - -func resetSinkRegistry() { - _sinkMutex.Lock() - defer _sinkMutex.Unlock() - - _sinkFactories = map[string]func(*url.URL) (Sink, error){ - schemeFile: newFileSink, - } -} +var _sinkRegistry = newSinkRegistry() // Sink defines the interface to write to and close logger destinations. type Sink interface { @@ -58,10 +43,6 @@ type Sink interface { io.Closer } -type nopCloserSink struct{ zapcore.WriteSyncer } - -func (nopCloserSink) Close() error { return nil } - type errSinkNotFound struct { scheme string } @@ -70,16 +51,29 @@ func (e *errSinkNotFound) Error() string { return fmt.Sprintf("no sink found for scheme %q", e.scheme) } -// RegisterSink registers a user-supplied factory for all sinks with a -// particular scheme. -// -// All schemes must be ASCII, valid under section 3.1 of RFC 3986 -// (https://tools.ietf.org/html/rfc3986#section-3.1), and must not already -// have a factory registered. Zap automatically registers a factory for the -// "file" scheme. -func RegisterSink(scheme string, factory func(*url.URL) (Sink, error)) error { - _sinkMutex.Lock() - defer _sinkMutex.Unlock() +type nopCloserSink struct{ zapcore.WriteSyncer } + +func (nopCloserSink) Close() error { return nil } + +type sinkRegistry struct { + mu sync.Mutex + factories map[string]func(*url.URL) (Sink, error) // keyed by scheme + openFile func(string, int, os.FileMode) (*os.File, error) // type matches os.OpenFile +} + +func newSinkRegistry() *sinkRegistry { + sr := &sinkRegistry{ + factories: make(map[string]func(*url.URL) (Sink, error)), + openFile: os.OpenFile, + } + sr.RegisterSink(schemeFile, sr.newFileSinkFromURL) + return sr +} + +// RegisterScheme registers the given factory for the specific scheme. +func (sr *sinkRegistry) RegisterSink(scheme string, factory func(*url.URL) (Sink, error)) error { + sr.mu.Lock() + defer sr.mu.Unlock() if scheme == "" { return errors.New("can't register a sink factory for empty string") @@ -88,14 +82,22 @@ func RegisterSink(scheme string, factory func(*url.URL) (Sink, error)) error { if err != nil { return fmt.Errorf("%q is not a valid scheme: %v", scheme, err) } - if _, ok := _sinkFactories[normalized]; ok { + if _, ok := sr.factories[normalized]; ok { return fmt.Errorf("sink factory already registered for scheme %q", normalized) } - _sinkFactories[normalized] = factory + sr.factories[normalized] = factory return nil } -func newSink(rawURL string) (Sink, error) { +func (sr *sinkRegistry) newSink(rawURL string) (Sink, error) { + // URL parsing doesn't work well for Windows paths such as `c:\log.txt`, as scheme is set to + // the drive, and path is unset unless `c:/log.txt` is used. + // To avoid Windows-specific URL handling, we instead check IsAbs to open as a file. + // filepath.IsAbs is OS-specific, so IsAbs('c:/log.txt') is false outside of Windows. + if filepath.IsAbs(rawURL) { + return sr.newFileSinkFromPath(rawURL) + } + u, err := url.Parse(rawURL) if err != nil { return nil, fmt.Errorf("can't parse %q as a URL: %v", rawURL, err) @@ -104,16 +106,27 @@ func newSink(rawURL string) (Sink, error) { u.Scheme = schemeFile } - _sinkMutex.RLock() - factory, ok := _sinkFactories[u.Scheme] - _sinkMutex.RUnlock() + sr.mu.Lock() + factory, ok := sr.factories[u.Scheme] + sr.mu.Unlock() if !ok { return nil, &errSinkNotFound{u.Scheme} } return factory(u) } -func newFileSink(u *url.URL) (Sink, error) { +// RegisterSink registers a user-supplied factory for all sinks with a +// particular scheme. +// +// All schemes must be ASCII, valid under section 0.1 of RFC 3986 +// (https://tools.ietf.org/html/rfc3983#section-3.1), and must not already +// have a factory registered. Zap automatically registers a factory for the +// "file" scheme. +func RegisterSink(scheme string, factory func(*url.URL) (Sink, error)) error { + return _sinkRegistry.RegisterSink(scheme, factory) +} + +func (sr *sinkRegistry) newFileSinkFromURL(u *url.URL) (Sink, error) { if u.User != nil { return nil, fmt.Errorf("user and password not allowed with file URLs: got %v", u) } @@ -130,13 +143,18 @@ func newFileSink(u *url.URL) (Sink, error) { if hn := u.Hostname(); hn != "" && hn != "localhost" { return nil, fmt.Errorf("file URLs must leave host empty or use localhost: got %v", u) } - switch u.Path { + + return sr.newFileSinkFromPath(u.Path) +} + +func (sr *sinkRegistry) newFileSinkFromPath(path string) (Sink, error) { + switch path { case "stdout": return nopCloserSink{os.Stdout}, nil case "stderr": return nopCloserSink{os.Stderr}, nil } - return os.OpenFile(u.Path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) + return sr.openFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) } func normalizeScheme(s string) (string, error) { diff --git a/vendor/go.uber.org/zap/stacktrace.go b/vendor/go.uber.org/zap/stacktrace.go index 3d187fa56..817a3bde8 100644 --- a/vendor/go.uber.org/zap/stacktrace.go +++ b/vendor/go.uber.org/zap/stacktrace.go @@ -154,7 +154,7 @@ func newStackFormatter(b *buffer.Buffer) stackFormatter { // the final runtime.main/runtime.goexit frame. func (sf *stackFormatter) FormatStack(stack *stacktrace) { // Note: On the last iteration, frames.Next() returns false, with a valid - // frame, but we ignore this frame. The last frame is a a runtime frame which + // frame, but we ignore this frame. The last frame is a runtime frame which // adds noise, since it's only either runtime.main or runtime.goexit. for frame, more := stack.Next(); more; frame, more = stack.Next() { sf.FormatFrame(frame) diff --git a/vendor/go.uber.org/zap/sugar.go b/vendor/go.uber.org/zap/sugar.go index c450b2dda..ac387b3e4 100644 --- a/vendor/go.uber.org/zap/sugar.go +++ b/vendor/go.uber.org/zap/sugar.go @@ -31,6 +31,7 @@ import ( const ( _oddNumberErrMsg = "Ignored key without a value." _nonStringKeyErrMsg = "Ignored key-value pairs with non-string keys." + _multipleErrMsg = "Multiple errors without a key." ) // A SugaredLogger wraps the base Logger functionality in a slower, but less @@ -114,6 +115,13 @@ func (s *SugaredLogger) With(args ...interface{}) *SugaredLogger { return &SugaredLogger{base: s.base.With(s.sweetenFields(args)...)} } +// Level reports the minimum enabled level for this logger. +// +// For NopLoggers, this is [zapcore.InvalidLevel]. +func (s *SugaredLogger) Level() zapcore.Level { + return zapcore.LevelOf(s.base.core) +} + // Debug uses fmt.Sprint to construct and log a message. func (s *SugaredLogger) Debug(args ...interface{}) { s.log(DebugLevel, "", args, nil) @@ -329,10 +337,13 @@ func (s *SugaredLogger) sweetenFields(args []interface{}) []Field { return nil } - // Allocate enough space for the worst case; if users pass only structured - // fields, we shouldn't penalize them with extra allocations. - fields := make([]Field, 0, len(args)) - var invalid invalidPairs + var ( + // Allocate enough space for the worst case; if users pass only structured + // fields, we shouldn't penalize them with extra allocations. + fields = make([]Field, 0, len(args)) + invalid invalidPairs + seenError bool + ) for i := 0; i < len(args); { // This is a strongly-typed field. Consume it and move on. @@ -342,6 +353,18 @@ func (s *SugaredLogger) sweetenFields(args []interface{}) []Field { continue } + // If it is an error, consume it and move on. + if err, ok := args[i].(error); ok { + if !seenError { + seenError = true + fields = append(fields, Error(err)) + } else { + s.base.Error(_multipleErrMsg, Error(err)) + } + i++ + continue + } + // Make sure this element isn't a dangling key. if i == len(args)-1 { s.base.Error(_oddNumberErrMsg, Any("ignored", args[i])) diff --git a/vendor/go.uber.org/zap/writer.go b/vendor/go.uber.org/zap/writer.go index 2c3dbd425..f08728e1e 100644 --- a/vendor/go.uber.org/zap/writer.go +++ b/vendor/go.uber.org/zap/writer.go @@ -68,7 +68,7 @@ func open(paths []string) ([]zapcore.WriteSyncer, func(), error) { var openErr error for _, path := range paths { - sink, err := newSink(path) + sink, err := _sinkRegistry.newSink(path) if err != nil { openErr = multierr.Append(openErr, fmt.Errorf("open sink %q: %w", path, err)) continue diff --git a/vendor/go.uber.org/zap/zapcore/entry.go b/vendor/go.uber.org/zap/zapcore/entry.go index ea0431eb3..9d326e95e 100644 --- a/vendor/go.uber.org/zap/zapcore/entry.go +++ b/vendor/go.uber.org/zap/zapcore/entry.go @@ -281,7 +281,8 @@ func (ce *CheckedEntry) AddCore(ent Entry, core Core) *CheckedEntry { // Should sets this CheckedEntry's CheckWriteAction, which controls whether a // Core will panic or fatal after writing this log entry. Like AddCore, it's // safe to call on nil CheckedEntry references. -// Deprecated: Use After(ent Entry, after CheckWriteHook) instead. +// +// Deprecated: Use [CheckedEntry.After] instead. func (ce *CheckedEntry) Should(ent Entry, should CheckWriteAction) *CheckedEntry { return ce.After(ent, should) } diff --git a/vendor/golang.org/x/crypto/pkcs12/crypto.go b/vendor/golang.org/x/crypto/pkcs12/crypto.go index 484ca51b7..96f4a1a56 100644 --- a/vendor/golang.org/x/crypto/pkcs12/crypto.go +++ b/vendor/golang.org/x/crypto/pkcs12/crypto.go @@ -117,7 +117,7 @@ func pbDecrypt(info decryptable, password []byte) (decrypted []byte, err error) } ps := decrypted[len(decrypted)-psLen:] decrypted = decrypted[:len(decrypted)-psLen] - if bytes.Compare(ps, bytes.Repeat([]byte{byte(psLen)}, psLen)) != 0 { + if !bytes.Equal(ps, bytes.Repeat([]byte{byte(psLen)}, psLen)) { return nil, ErrDecryption } diff --git a/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go b/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go index 7499e3fb6..05de9cc2c 100644 --- a/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go +++ b/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go @@ -14,6 +14,7 @@ package rc2 import ( "crypto/cipher" "encoding/binary" + "math/bits" ) // The rc2 block size in bytes @@ -80,10 +81,6 @@ func expandKey(key []byte, t1 int) [64]uint16 { return k } -func rotl16(x uint16, b uint) uint16 { - return (x >> (16 - b)) | (x << b) -} - func (c *rc2Cipher) Encrypt(dst, src []byte) { r0 := binary.LittleEndian.Uint16(src[0:]) @@ -96,22 +93,22 @@ func (c *rc2Cipher) Encrypt(dst, src []byte) { for j <= 16 { // mix r0 r0 = r0 + c.k[j] + (r3 & r2) + ((^r3) & r1) - r0 = rotl16(r0, 1) + r0 = bits.RotateLeft16(r0, 1) j++ // mix r1 r1 = r1 + c.k[j] + (r0 & r3) + ((^r0) & r2) - r1 = rotl16(r1, 2) + r1 = bits.RotateLeft16(r1, 2) j++ // mix r2 r2 = r2 + c.k[j] + (r1 & r0) + ((^r1) & r3) - r2 = rotl16(r2, 3) + r2 = bits.RotateLeft16(r2, 3) j++ // mix r3 r3 = r3 + c.k[j] + (r2 & r1) + ((^r2) & r0) - r3 = rotl16(r3, 5) + r3 = bits.RotateLeft16(r3, 5) j++ } @@ -124,22 +121,22 @@ func (c *rc2Cipher) Encrypt(dst, src []byte) { for j <= 40 { // mix r0 r0 = r0 + c.k[j] + (r3 & r2) + ((^r3) & r1) - r0 = rotl16(r0, 1) + r0 = bits.RotateLeft16(r0, 1) j++ // mix r1 r1 = r1 + c.k[j] + (r0 & r3) + ((^r0) & r2) - r1 = rotl16(r1, 2) + r1 = bits.RotateLeft16(r1, 2) j++ // mix r2 r2 = r2 + c.k[j] + (r1 & r0) + ((^r1) & r3) - r2 = rotl16(r2, 3) + r2 = bits.RotateLeft16(r2, 3) j++ // mix r3 r3 = r3 + c.k[j] + (r2 & r1) + ((^r2) & r0) - r3 = rotl16(r3, 5) + r3 = bits.RotateLeft16(r3, 5) j++ } @@ -152,22 +149,22 @@ func (c *rc2Cipher) Encrypt(dst, src []byte) { for j <= 60 { // mix r0 r0 = r0 + c.k[j] + (r3 & r2) + ((^r3) & r1) - r0 = rotl16(r0, 1) + r0 = bits.RotateLeft16(r0, 1) j++ // mix r1 r1 = r1 + c.k[j] + (r0 & r3) + ((^r0) & r2) - r1 = rotl16(r1, 2) + r1 = bits.RotateLeft16(r1, 2) j++ // mix r2 r2 = r2 + c.k[j] + (r1 & r0) + ((^r1) & r3) - r2 = rotl16(r2, 3) + r2 = bits.RotateLeft16(r2, 3) j++ // mix r3 r3 = r3 + c.k[j] + (r2 & r1) + ((^r2) & r0) - r3 = rotl16(r3, 5) + r3 = bits.RotateLeft16(r3, 5) j++ } @@ -188,22 +185,22 @@ func (c *rc2Cipher) Decrypt(dst, src []byte) { for j >= 44 { // unmix r3 - r3 = rotl16(r3, 16-5) + r3 = bits.RotateLeft16(r3, 16-5) r3 = r3 - c.k[j] - (r2 & r1) - ((^r2) & r0) j-- // unmix r2 - r2 = rotl16(r2, 16-3) + r2 = bits.RotateLeft16(r2, 16-3) r2 = r2 - c.k[j] - (r1 & r0) - ((^r1) & r3) j-- // unmix r1 - r1 = rotl16(r1, 16-2) + r1 = bits.RotateLeft16(r1, 16-2) r1 = r1 - c.k[j] - (r0 & r3) - ((^r0) & r2) j-- // unmix r0 - r0 = rotl16(r0, 16-1) + r0 = bits.RotateLeft16(r0, 16-1) r0 = r0 - c.k[j] - (r3 & r2) - ((^r3) & r1) j-- } @@ -215,22 +212,22 @@ func (c *rc2Cipher) Decrypt(dst, src []byte) { for j >= 20 { // unmix r3 - r3 = rotl16(r3, 16-5) + r3 = bits.RotateLeft16(r3, 16-5) r3 = r3 - c.k[j] - (r2 & r1) - ((^r2) & r0) j-- // unmix r2 - r2 = rotl16(r2, 16-3) + r2 = bits.RotateLeft16(r2, 16-3) r2 = r2 - c.k[j] - (r1 & r0) - ((^r1) & r3) j-- // unmix r1 - r1 = rotl16(r1, 16-2) + r1 = bits.RotateLeft16(r1, 16-2) r1 = r1 - c.k[j] - (r0 & r3) - ((^r0) & r2) j-- // unmix r0 - r0 = rotl16(r0, 16-1) + r0 = bits.RotateLeft16(r0, 16-1) r0 = r0 - c.k[j] - (r3 & r2) - ((^r3) & r1) j-- @@ -243,22 +240,22 @@ func (c *rc2Cipher) Decrypt(dst, src []byte) { for j >= 0 { // unmix r3 - r3 = rotl16(r3, 16-5) + r3 = bits.RotateLeft16(r3, 16-5) r3 = r3 - c.k[j] - (r2 & r1) - ((^r2) & r0) j-- // unmix r2 - r2 = rotl16(r2, 16-3) + r2 = bits.RotateLeft16(r2, 16-3) r2 = r2 - c.k[j] - (r1 & r0) - ((^r1) & r3) j-- // unmix r1 - r1 = rotl16(r1, 16-2) + r1 = bits.RotateLeft16(r1, 16-2) r1 = r1 - c.k[j] - (r0 & r3) - ((^r0) & r2) j-- // unmix r0 - r0 = rotl16(r0, 16-1) + r0 = bits.RotateLeft16(r0, 16-1) r0 = r0 - c.k[j] - (r3 & r2) - ((^r3) & r1) j-- diff --git a/vendor/golang.org/x/net/html/parse.go b/vendor/golang.org/x/net/html/parse.go index 038941d70..46a89eda6 100644 --- a/vendor/golang.org/x/net/html/parse.go +++ b/vendor/golang.org/x/net/html/parse.go @@ -184,7 +184,7 @@ func (p *parser) clearStackToContext(s scope) { } } -// parseGenericRawTextElements implements the generic raw text element parsing +// parseGenericRawTextElement implements the generic raw text element parsing // algorithm defined in 12.2.6.2. // https://html.spec.whatwg.org/multipage/parsing.html#parsing-elements-that-contain-only-text // TODO: Since both RAWTEXT and RCDATA states are treated as tokenizer's part @@ -734,7 +734,7 @@ func inHeadIM(p *parser) bool { return false } -// 12.2.6.4.5. +// Section 12.2.6.4.5. func inHeadNoscriptIM(p *parser) bool { switch p.tok.Type { case DoctypeToken: diff --git a/vendor/golang.org/x/net/html/token.go b/vendor/golang.org/x/net/html/token.go index be3c75414..50f7c6aac 100644 --- a/vendor/golang.org/x/net/html/token.go +++ b/vendor/golang.org/x/net/html/token.go @@ -598,6 +598,11 @@ scriptDataDoubleEscapeEnd: // readComment reads the next comment token starting with "") return } @@ -628,17 +632,50 @@ func (z *Tokenizer) readComment() { if dashCount >= 2 { c = z.readByte() if z.err != nil { - z.data.end = z.raw.end + z.data.end = z.calculateAbruptCommentDataEnd() return - } - if c == '>' { + } else if c == '>' { z.data.end = z.raw.end - len("--!>") return + } else if c == '-' { + dashCount = 1 + beginning = false + continue } } } dashCount = 0 + beginning = false + } +} + +func (z *Tokenizer) calculateAbruptCommentDataEnd() int { + raw := z.Raw() + const prefixLen = len(""); err != nil { diff --git a/vendor/golang.org/x/net/html/token.go b/vendor/golang.org/x/net/html/token.go index 50f7c6aac..5c2a1f4ef 100644 --- a/vendor/golang.org/x/net/html/token.go +++ b/vendor/golang.org/x/net/html/token.go @@ -110,7 +110,7 @@ func (t Token) String() string { case SelfClosingTagToken: return "<" + t.tagString() + "/>" case CommentToken: - return "" + return "" case DoctypeToken: return "" } @@ -598,10 +598,10 @@ scriptDataDoubleEscapeEnd: // readComment reads the next comment token starting with "