diff --git a/doc/.wordlist.txt b/doc/.wordlist.txt index dda025890f5e..1e830304f4b2 100644 --- a/doc/.wordlist.txt +++ b/doc/.wordlist.txt @@ -14,6 +14,7 @@ EBS EKS enablement favicon +FlashArray Furo GDB Git @@ -21,6 +22,7 @@ GitHub Grafana IAM installable +iSCSI JSON Juju Kubeflow diff --git a/doc/explanation/storage.md b/doc/explanation/storage.md index ac3f6d98d380..a85ae48d1143 100644 --- a/doc/explanation/storage.md +++ b/doc/explanation/storage.md @@ -24,6 +24,7 @@ The following storage drivers are supported: - [CephFS - `cephfs`](storage-cephfs) - [Ceph Object - `cephobject`](storage-cephobject) - [Dell PowerFlex - `powerflex`](storage-powerflex) +- [Pure Storage - `pure`](storage-pure) See the following how-to guides for additional information: @@ -36,12 +37,12 @@ See the following how-to guides for additional information: Where the LXD data is stored depends on the configuration and the selected storage driver. Depending on the storage driver that is used, LXD can either share the file system with its host or keep its data separate. -Storage location | Directory | Btrfs | LVM | ZFS | Ceph (all) | Dell PowerFlex | -:--- | :-: | :-: | :-: | :-: | :-: | :-: | -Shared with the host | ✓ | ✓ | - | ✓ | - | - | -Dedicated disk/partition | - | ✓ | ✓ | ✓ | - | - | -Loop disk | - | ✓ | ✓ | ✓ | - | - | -Remote storage | - | - | - | - | ✓ | ✓ | +Storage location | Directory | Btrfs | LVM | ZFS | Ceph (all) | Dell PowerFlex | Pure Storage | +:--- | :-: | :-: | :-: | :-: | :-: | :-: | :-: | +Shared with the host | ✓ | ✓ | - | ✓ | - | - | - | +Dedicated disk/partition | - | ✓ | ✓ | ✓ | - | - | - | +Loop disk | - | ✓ | ✓ | ✓ | - | - | - | +Remote storage | - | - | - | - | ✓ | ✓ | ✓ | #### Shared with the host @@ -71,7 +72,7 @@ You can increase their size (quota) though; see {ref}`storage-resize-pool`. #### Remote storage The `ceph`, `cephfs` and `cephobject` drivers store the data in a completely independent Ceph storage cluster that must be set up separately. -The same applies to the `powerflex` driver. +The same applies to the `powerflex` and `pure` drivers. (storage-default-pool)= ### Default storage pool diff --git a/doc/howto/storage_pools.md b/doc/howto/storage_pools.md index 1c721a4ceb31..d16ea1fd7b5c 100644 --- a/doc/howto/storage_pools.md +++ b/doc/howto/storage_pools.md @@ -173,6 +173,20 @@ Create a storage pool named `pool5` that explicitly uses the PowerFlex SDC: lxc storage create pool5 powerflex powerflex.mode=sdc powerflex.pool= powerflex.gateway=https://powerflex powerflex.user.name=lxd powerflex.user.password=foo +#### Create a Pure Storage pool + +Create a storage pool named `pool1` that uses NVMe/TCP by default: + + lxc storage create pool1 pure pure.gateway=https:// pure.api.token= + +Create a storage pool named `pool2` that uses a Pure Storage gateway with a certificate that is not trusted: + + lxc storage create pool2 pure pure.gateway=https:// pure.gateway.verify=false pure.api.token= + +Create a storage pool named `pool3` that uses iSCSI to connect to Pure Storage array: + + lxc storage create pool3 pure pure.gateway=https:// pure.api.token= pure.mode=iscsi + (storage-pools-cluster)= ## Create a storage pool in a cluster @@ -240,6 +254,19 @@ Storage pool my-remote-pool2 pending on member vm03 Storage pool my-remote-pool2 created ``` +Create a third storage pool named `my-remote-pool3` using the Pure Storage driver: + +```{terminal} +:input: lxc storage create my-remote-pool3 pure --target=vm01 +Storage pool my-remote-pool3 pending on member vm01 +:input: lxc storage create my-remote-pool3 pure --target=vm02 +Storage pool my-remote-pool3 pending on member vm02 +:input: lxc storage create my-remote-pool3 pure --target=vm03 +Storage pool my-remote-pool3 pending on member vm03 +:input: lxc storage create my-remote-pool3 pure pure.gateway=https:// pure.api.token= +Storage pool my-remote-pool3 created +``` + ## Configure storage pool settings See the {ref}`storage-drivers` documentation for the available configuration options for each storage driver. diff --git a/doc/metadata.txt b/doc/metadata.txt index 26693a7a400d..ec60fc8f3893 100644 --- a/doc/metadata.txt +++ b/doc/metadata.txt @@ -5915,6 +5915,114 @@ Specify either a cron expression (` `), a comm ``` + +```{config:option} pure.api.token storage-pure-pool-conf +:shortdesc: "API token for Pure Storage gateway authentication" +:type: "string" + +``` + +```{config:option} pure.gateway storage-pure-pool-conf +:shortdesc: "Address of the Pure Storage gateway" +:type: "string" + +``` + +```{config:option} pure.gateway.verify storage-pure-pool-conf +:defaultdesc: "`true`" +:shortdesc: "Whether to verify the Pure Storage gateway's certificate" +:type: "bool" + +``` + +```{config:option} pure.mode storage-pure-pool-conf +:defaultdesc: "the discovered mode" +:shortdesc: "How volumes are mapped to the local server" +:type: "string" +The mode to use to map Pure Storage volumes to the local server. +Supported values are `iscsi` and `nvme`. +``` + +```{config:option} volume.size storage-pure-pool-conf +:defaultdesc: "`10GiB`" +:shortdesc: "Size/quota of the storage volume" +:type: "string" +Default Pure Storage volume size rounded to 512B. The minimum size is 1MiB. +``` + + + +```{config:option} block.filesystem storage-pure-volume-conf +:condition: "block-based volume with content type `filesystem`" +:defaultdesc: "same as `volume.block.filesystem`" +:shortdesc: "File system of the storage volume" +:type: "string" +Valid options are: `btrfs`, `ext4`, `xfs` +If not set, `ext4` is assumed. +``` + +```{config:option} block.mount_options storage-pure-volume-conf +:condition: "block-based volume with content type `filesystem`" +:defaultdesc: "same as `volume.block.mount_options`" +:shortdesc: "Mount options for block-backed file system volumes" +:type: "string" + +``` + +```{config:option} size storage-pure-volume-conf +:defaultdesc: "same as `volume.size`" +:shortdesc: "Size/quota of the storage volume" +:type: "string" +Default Pure Storage volume size rounded to 512B. The minimum size is 1MiB. +``` + +```{config:option} snapshots.expiry storage-pure-volume-conf +:condition: "custom volume" +:defaultdesc: "same as `volume.snapshots.expiry`" +:scope: "global" +:shortdesc: "When snapshots are to be deleted" +:type: "string" +Specify an expression like `1M 2H 3d 4w 5m 6y`. +``` + +```{config:option} snapshots.pattern storage-pure-volume-conf +:condition: "custom volume" +:defaultdesc: "same as `volume.snapshots.pattern` or `snap%d`" +:scope: "global" +:shortdesc: "Template for the snapshot name" +:type: "string" +You can specify a naming template that is used for scheduled snapshots and unnamed snapshots. + +The `snapshots.pattern` option takes a Pongo2 template string to format the snapshot name. + +To add a time stamp to the snapshot name, use the Pongo2 context variable `creation_date`. +Make sure to format the date in your template string to avoid forbidden characters in the snapshot name. +For example, set `snapshots.pattern` to `{{ creation_date|date:'2006-01-02_15-04-05' }}` to name the snapshots after their time of creation, down to the precision of a second. + +Another way to avoid name collisions is to use the placeholder `%d` in the pattern. +For the first snapshot, the placeholder is replaced with `0`. +For subsequent snapshots, the existing snapshot names are taken into account to find the highest number at the placeholder's position. +This number is then incremented by one for the new name. +``` + +```{config:option} snapshots.schedule storage-pure-volume-conf +:condition: "custom volume" +:defaultdesc: "same as `snapshots.schedule`" +:scope: "global" +:shortdesc: "Schedule for automatic volume snapshots" +:type: "string" +Specify either a cron expression (` `), a comma-separated list of schedule aliases (`@hourly`, `@daily`, `@midnight`, `@weekly`, `@monthly`, `@annually`, `@yearly`), or leave empty to disable automatic snapshots (the default). +``` + +```{config:option} volatile.uuid storage-pure-volume-conf +:defaultdesc: "random UUID" +:scope: "global" +:shortdesc: "The volume's UUID" +:type: "string" + +``` + + ```{config:option} size storage-zfs-bucket-conf :condition: "appropriate driver" diff --git a/doc/reference/storage_drivers.md b/doc/reference/storage_drivers.md index 195d60534d16..1ac8426af4be 100644 --- a/doc/reference/storage_drivers.md +++ b/doc/reference/storage_drivers.md @@ -15,6 +15,7 @@ storage_cephfs storage_cephobject storage_ceph storage_powerflex +storage_pure storage_dir storage_lvm storage_zfs @@ -27,23 +28,23 @@ See the corresponding pages for driver-specific information and configuration op Where possible, LXD uses the advanced features of each storage system to optimize operations. -Feature | Directory | Btrfs | LVM | ZFS | Ceph RBD | CephFS | Ceph Object | Dell PowerFlex -:--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- -{ref}`storage-optimized-image-storage` | ❌ | ✅ | ✅ | ✅ | ✅ | ➖ | ➖ | ❌ -Optimized instance creation | ❌ | ✅ | ✅ | ✅ | ✅ | ➖ | ➖ | ❌ -Optimized snapshot creation | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ➖ | ✅ -Optimized image transfer | ❌ | ✅ | ❌ | ✅ | ✅ | ➖ | ➖ | ❌ -Optimized backup (import/export) | ❌ | ✅ | ❌ | ✅ | ❌ | ➖ | ➖ | ❌ -{ref}`storage-optimized-volume-transfer` | ❌ | ✅ | ❌ | ✅ | ✅[^1] | ➖ | ➖ | ❌ -{ref}`storage-optimized-volume-refresh` | ❌ | ✅ | ✅[^2] | ✅ | ✅[^3] | ➖ | ➖ | ❌ -Copy on write | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ➖ | ✅ -Block based | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ➖ | ✅ -Instant cloning | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ➖ | ❌ -Storage driver usable inside a container | ✅ | ✅ | ❌ | ✅[^4] | ❌ | ➖ | ➖ | ❌ -Restore from older snapshots (not latest) | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ➖ | ✅ -Storage quotas | ✅[^5] | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ -Available on `lxd init` | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ -Object storage | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ +Feature | Directory | Btrfs | LVM | ZFS | Ceph RBD | CephFS | Ceph Object | Dell PowerFlex | Pure Storage +:--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- +{ref}`storage-optimized-image-storage` | ❌ | ✅ | ✅ | ✅ | ✅ | ➖ | ➖ | ❌ | ✅ +Optimized instance creation | ❌ | ✅ | ✅ | ✅ | ✅ | ➖ | ➖ | ❌ | ✅ +Optimized snapshot creation | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ➖ | ✅ | ✅ +Optimized image transfer | ❌ | ✅ | ❌ | ✅ | ✅ | ➖ | ➖ | ❌ | ✅ +Optimized backup (import/export) | ❌ | ✅ | ❌ | ✅ | ❌ | ➖ | ➖ | ❌ | ❌ +{ref}`storage-optimized-volume-transfer` | ❌ | ✅ | ❌ | ✅ | ✅[^1] | ➖ | ➖ | ❌ | ❌ +{ref}`storage-optimized-volume-refresh` | ❌ | ✅ | ✅[^2] | ✅ | ✅[^3] | ➖ | ➖ | ❌ | ❌ +Copy on write | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ➖ | ✅ | ✅ +Block based | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ➖ | ✅ | ✅ +Instant cloning | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ➖ | ❌ | ✅ +Storage driver usable inside a container | ✅ | ✅ | ❌ | ✅[^4] | ❌ | ➖ | ➖ | ❌ | ❌ +Restore from older snapshots (not latest) | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ➖ | ✅ | ✅ +Storage quotas | ✅[^5] | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ +Available on `lxd init` | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ +Object storage | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ [^1]: Volumes of type `block` will fall back to non-optimized transfer when migrating to an older LXD server that doesn't yet support the `RBD_AND_RSYNC` migration type. [^2]: Requires {config:option}`storage-lvm-pool-conf:lvm.use_thinpool` to be enabled. Only when refreshing local volumes. diff --git a/doc/reference/storage_pure.md b/doc/reference/storage_pure.md new file mode 100644 index 000000000000..c548429a4c58 --- /dev/null +++ b/doc/reference/storage_pure.md @@ -0,0 +1,104 @@ +(storage-pure)= +# Pure Storage - `pure` + +[Pure Storage](https://www.purestorage.com/) is a software-defined storage solution. It offers the consumption of redundant block storage across the network. + +LXD supports connecting to Pure Storage storage clusters through two protocols: either {abbr}`iSCSI (Internet Small Computer Systems Interface)` or {abbr}`NVMe/TCP (Non-Volatile Memory Express over Transmission Control Protocol)`. +In addition, Pure Storage offers copy-on-write snapshots, thin provisioning, and other features. + +To use Pure Storage, ensure that the required kernel modules for the selected protocol are installed on your host system. +For iSCSI, the iSCSI CLI named `iscsiadm` needs to be installed in addition to the required kernel modules. + +## Terminology + +Each storage pool created in LXD using a Pure Storage driver represents a Pure Storage *pod*, which is an abstraction that groups multiple volumes under a specific name. +One benefit of using Pure Storage pods is that they can be linked with multiple Pure Storage arrays to provide additional redundancy. + +LXD creates volumes within a pod that is identified by the storage pool name. +When the first volume needs to be mapped to a specific LXD host, a corresponding Pure Storage host is created with the name of the LXD host and a suffix of the used protocol. +For example, if the LXD host is `host01` and the mode is `nvme`, the resulting Pure Storage host would be `host01-nvme`. + +The Pure Storage host is then connected with the required volumes, to allow attaching and accessing volumes from the LXD host. +The created Pure Storage host is automatically removed once there are no volumes connected to it. + +## The `pure` driver in LXD + +The `pure` driver in LXD uses Pure Storage volumes for custom storage volumes, instances, and snapshots. +All created volumes are thin-provisioned block volumes. If required (for example, for containers and custom file system volumes), LXD formats the volume with a desired file system. + +LXD expects Pure Storage to be pre-configured with a specific service (e.g. iSCSI) on network interfaces whose address is provided during storage pool configuration. +Furthermore, LXD assumes that it has full control over the Pure Storage pods it manages. +Therefore, you should never maintain any volumes in Pure Storage pods that are not owned by LXD because LXD might disconnect or even delete them. + +This driver behaves differently than some of the other drivers in that it provides remote storage. +As a result, and depending on the internal network, storage access might be a bit slower compared to local storage. +On the other hand, using remote storage has significant advantages in a cluster setup: all cluster members have access to the same storage pools with the exact same contents, without the need to synchronize them. + +When creating a new storage pool using the `pure` driver in either `iscsi` or `nvme` mode, LXD automatically discovers the array's qualified name and target address (portal). +Upon successful discovery, LXD attaches all volumes that are connected to the Pure Storage host that is associated with a specific LXD server. +Pure Storage hosts and volume connections are fully managed by LXD. + +Volume snapshots are also supported by Pure Storage. However, each snapshot is associated with a parent volume and cannot be directly attached to the host. +Therefore, when a snapshot is being exported, LXD creates a temporary volume behind the scenes. This volume is attached to the LXD host and removed once the operation is completed. +Similarly, when a volume with at least one snapshot is being copied, LXD sequentially copies snapshots into destination volume, from which a new snapshot is created. +Finally, once all snapshots are copied, the source volume is copied into the destination volume. + +(storage-pure-volume-names)= +### Volume names + +Due to a [limitation](storage-pure-limitations) in Pure Storage, volume names cannot exceed 63 characters. +Therefore, the driver uses the volume's {config:option}`storage-pure-volume-conf:volatile.uuid` to generate a shorter volume name. + +For example, a UUID `5a2504b0-6a6c-4849-8ee7-ddb0b674fd14` is first trimmed of any hyphens (`-`), resulting in the string `5a2504b06a6c48498ee7ddb0b674fd14`. +To distinguish volume types and snapshots, special identifiers are prepended and appended to the volume names, as depicted in the table below: + +Type | Identifier | Example +:-- | :--- | :---------- +Container | `c-` | `c-5a2504b06a6c48498ee7ddb0b674fd14` +Virtual machine | `v-` | `v-5a2504b06a6c48498ee7ddb0b674fd14-b` (block volume) and `v-5a2504b06a6c48498ee7ddb0b674fd14` (file system volume) +Image (ISO) | `i-` | `i-5a2504b06a6c48498ee7ddb0b674fd14-i` +Custom volume | `u-` | `u-5a2504b06a6c48498ee7ddb0b674fd14` +Snapshot | `s` | `sc-5a2504b06a6c48498ee7ddb0b674fd14` (container snapshot) + +(storage-pure-limitations)= +### Limitations + +The `pure` driver has the following limitations: + +Volume size constraints +: Minimum volume size (quota) is `1MiB` and must be a multiple of `512B`. + +Snapshots cannot be mounted +: Snapshots cannot be mounted directly to the host. Instead, a temporary volume must be created to access the snapshot's contents. + For internal operations, such as copying instances or exporting snapshots, LXD handles this automatically. + +Sharing the Pure Storage storage pool between installations +: Sharing the same Pure Storage storage pool between multiple LXD installations is not supported. + If a different LXD installation tries to create a storage pool with a name that already exists, an error is returned. + +Recovering Pure Storage storage pools +: Recovery of Pure Storage storage pools using `lxd recover` is currently not supported. + +## Configuration options + +The following configuration options are available for storage pools that use the `pure` driver, as well as storage volumes in these pools. + +(storage-pure-pool-config)= +### Storage pool configuration + +% Include content from [../metadata.txt](../metadata.txt) +```{include} ../metadata.txt + :start-after: + :end-before: +``` + +{{volume_configuration}} + +(storage-pure-vol-config)= +### Storage volume configuration + +% Include content from [../metadata.txt](../metadata.txt) +```{include} ../metadata.txt + :start-after: + :end-before: +``` diff --git a/lxd/metadata/configuration.json b/lxd/metadata/configuration.json index 5040c6e96ef1..7e9031232383 100644 --- a/lxd/metadata/configuration.json +++ b/lxd/metadata/configuration.json @@ -6600,6 +6600,119 @@ ] } }, + "storage-pure": { + "pool-conf": { + "keys": [ + { + "pure.api.token": { + "longdesc": "", + "shortdesc": "API token for Pure Storage gateway authentication", + "type": "string" + } + }, + { + "pure.gateway": { + "longdesc": "", + "shortdesc": "Address of the Pure Storage gateway", + "type": "string" + } + }, + { + "pure.gateway.verify": { + "defaultdesc": "`true`", + "longdesc": "", + "shortdesc": "Whether to verify the Pure Storage gateway's certificate", + "type": "bool" + } + }, + { + "pure.mode": { + "defaultdesc": "the discovered mode", + "longdesc": "The mode to use to map Pure Storage volumes to the local server.\nSupported values are `iscsi` and `nvme`.", + "shortdesc": "How volumes are mapped to the local server", + "type": "string" + } + }, + { + "volume.size": { + "defaultdesc": "`10GiB`", + "longdesc": "Default Pure Storage volume size rounded to 512B. The minimum size is 1MiB.", + "shortdesc": "Size/quota of the storage volume", + "type": "string" + } + } + ] + }, + "volume-conf": { + "keys": [ + { + "block.filesystem": { + "condition": "block-based volume with content type `filesystem`", + "defaultdesc": "same as `volume.block.filesystem`", + "longdesc": "Valid options are: `btrfs`, `ext4`, `xfs`\nIf not set, `ext4` is assumed.", + "shortdesc": "File system of the storage volume", + "type": "string" + } + }, + { + "block.mount_options": { + "condition": "block-based volume with content type `filesystem`", + "defaultdesc": "same as `volume.block.mount_options`", + "longdesc": "", + "shortdesc": "Mount options for block-backed file system volumes", + "type": "string" + } + }, + { + "size": { + "defaultdesc": "same as `volume.size`", + "longdesc": "Default Pure Storage volume size rounded to 512B. The minimum size is 1MiB.", + "shortdesc": "Size/quota of the storage volume", + "type": "string" + } + }, + { + "snapshots.expiry": { + "condition": "custom volume", + "defaultdesc": "same as `volume.snapshots.expiry`", + "longdesc": "Specify an expression like `1M 2H 3d 4w 5m 6y`.", + "scope": "global", + "shortdesc": "When snapshots are to be deleted", + "type": "string" + } + }, + { + "snapshots.pattern": { + "condition": "custom volume", + "defaultdesc": "same as `volume.snapshots.pattern` or `snap%d`", + "longdesc": "You can specify a naming template that is used for scheduled snapshots and unnamed snapshots.\n\nThe `snapshots.pattern` option takes a Pongo2 template string to format the snapshot name.\n\nTo add a time stamp to the snapshot name, use the Pongo2 context variable `creation_date`.\nMake sure to format the date in your template string to avoid forbidden characters in the snapshot name.\nFor example, set `snapshots.pattern` to `{{ creation_date|date:'2006-01-02_15-04-05' }}` to name the snapshots after their time of creation, down to the precision of a second.\n\nAnother way to avoid name collisions is to use the placeholder `%d` in the pattern.\nFor the first snapshot, the placeholder is replaced with `0`.\nFor subsequent snapshots, the existing snapshot names are taken into account to find the highest number at the placeholder's position.\nThis number is then incremented by one for the new name.", + "scope": "global", + "shortdesc": "Template for the snapshot name", + "type": "string" + } + }, + { + "snapshots.schedule": { + "condition": "custom volume", + "defaultdesc": "same as `snapshots.schedule`", + "longdesc": "Specify either a cron expression (`\u003cminute\u003e \u003chour\u003e \u003cdom\u003e \u003cmonth\u003e \u003cdow\u003e`), a comma-separated list of schedule aliases (`@hourly`, `@daily`, `@midnight`, `@weekly`, `@monthly`, `@annually`, `@yearly`), or leave empty to disable automatic snapshots (the default).", + "scope": "global", + "shortdesc": "Schedule for automatic volume snapshots", + "type": "string" + } + }, + { + "volatile.uuid": { + "defaultdesc": "random UUID", + "longdesc": "", + "scope": "global", + "shortdesc": "The volume's UUID", + "type": "string" + } + } + ] + } + }, "storage-zfs": { "bucket-conf": { "keys": [ diff --git a/lxd/storage/backend_lxd.go b/lxd/storage/backend_lxd.go index ee111d241e05..651e7be5ea04 100644 --- a/lxd/storage/backend_lxd.go +++ b/lxd/storage/backend_lxd.go @@ -1154,6 +1154,16 @@ func (b *lxdBackend) CreateInstanceFromCopy(inst instance.Instance, src instance srcVolStorageName := project.Instance(src.Project().Name, src.Name()) srcVol := b.GetVolume(volType, contentType, srcVolStorageName, srcConfig.Volume.Config) + // Set the parent volume's UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(srcVol, src.Project().Name) + if err != nil { + return err + } + + srcVol.SetParentUUID(parentUUID) + } + volCopy := drivers.NewVolumeCopy(vol, targetSnapshots...) srcVolCopy := drivers.NewVolumeCopy(srcVol, sourceSnapshots...) @@ -2015,6 +2025,16 @@ func (b *lxdBackend) CreateInstanceFromImage(inst instance.Instance, fingerprint // will cause a non matching configuration which will always fall back to non optimized storage. vol := b.GetNewVolume(volType, contentType, volStorageName, volumeConfig) + // Set the parent volume UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, inst.Project().Name) + if err != nil { + return err + } + + vol.SetParentUUID(parentUUID) + } + // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), "", volType, false, vol.Config(), inst.CreationDate(), time.Time{}, contentType, true, false) if err != nil { @@ -2365,6 +2385,16 @@ func (b *lxdBackend) CreateInstanceFromMigration(inst instance.Instance, conn io targetSnapshots = append(targetSnapshots, b.GetVolume(volType, contentType, snapshotStorageName, snap.Config)) } + // Set the parent volume's UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, projectName) + if err != nil { + return err + } + + vol.SetParentUUID(parentUUID) + } + volCopy := drivers.NewVolumeCopy(vol, targetSnapshots...) err = b.driver.CreateVolumeFromMigration(volCopy, conn, args, &preFiller, op) @@ -3076,6 +3106,16 @@ func (b *lxdBackend) MigrateInstance(inst instance.Instance, conn io.ReadWriteCl _ = filesystem.SyncFS(inst.RootfsPath()) } + // Set the parent volume UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, inst.Project().Name) + if err != nil { + return err + } + + vol.SetParentUUID(parentUUID) + } + volCopy := drivers.NewVolumeCopy(vol, sourceSnapshots...) err = b.driver.MigrateVolume(volCopy, conn, args, op) @@ -3714,6 +3754,16 @@ func (b *lxdBackend) DeleteInstanceSnapshot(inst instance.Instance, op *operatio vol := b.GetVolume(volType, contentType, snapVolName, dbVol.Config) + // Set the parent volume UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, inst.Project().Name) + if err != nil { + return err + } + + vol.SetParentUUID(parentUUID) + } + volExists, err := b.driver.HasVolume(vol) if err != nil { return err @@ -3915,6 +3965,17 @@ func (b *lxdBackend) MountInstanceSnapshot(inst instance.Instance, op *operation return nil, err } + // Set the parent volume UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, inst.Project().Name) + if err != nil { + return nil, err + } + + vol.SetParentUUID(parentUUID) + } + + // Mount the snapshot. err = b.driver.MountVolumeSnapshot(vol, op) if err != nil { return nil, err @@ -3964,6 +4025,17 @@ func (b *lxdBackend) UnmountInstanceSnapshot(inst instance.Instance, op *operati return err } + // Set the parent volume UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, inst.Project().Name) + if err != nil { + return err + } + + vol.SetParentUUID(parentUUID) + } + + // Unmount volume. _, err = b.driver.UnmountVolumeSnapshot(vol, op) return err @@ -4748,7 +4820,7 @@ func (b *lxdBackend) recoverMinIOKeys(projectName string, bucketName string, op break } - var recoveredKeys []api.StorageBucketKeysPost + recoveredKeys := make([]api.StorageBucketKeysPost, 0, len(svcAccounts)) // Extract bucket keys for each service account. for _, creds := range svcAccounts { @@ -5730,6 +5802,15 @@ func (b *lxdBackend) CreateCustomVolumeFromMigration(projectName string, conn io return err } + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, projectName) + if err != nil { + return err + } + + vol.SetParentUUID(parentUUID) + } + revert := revert.New() defer revert.Fail() @@ -6606,6 +6687,16 @@ func (b *lxdBackend) DeleteCustomVolumeSnapshot(projectName, volName string, op vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, volume.Config) + // Set the parent volume's UUID. + if b.driver.Info().PopulateParentVolumeUUID { + parentUUID, err := b.getParentVolumeUUID(vol, projectName) + if err != nil { + return err + } + + vol.SetParentUUID(parentUUID) + } + // Delete the snapshot from the storage device. // Must come before DB VolumeDBDelete so that the volume ID is still available. volExists, err := b.driver.HasVolume(vol) @@ -7788,3 +7879,30 @@ func (b *lxdBackend) CreateCustomVolumeFromBackup(srcBackup backup.Info, srcData revert.Success() return nil } + +// getParentVolumeUUID returns the UUID of the parent's volume. +// If the volume has no parent, an empty string is returned. +func (b *lxdBackend) getParentVolumeUUID(vol drivers.Volume, projectName string) (string, error) { + parentName, _, isSnapshot := api.GetParentAndSnapshotName(vol.Name()) + if !isSnapshot { + // Volume has no parent. + return "", nil + } + + // Ensure the parent name does not contain a project prefix. + _, parentName = project.StorageVolumeParts(parentName) + + // Load storage volume from the database. + parentDBVol, err := VolumeDBGet(b, projectName, parentName, vol.Type()) + if err != nil { + return "", fmt.Errorf("Failed to extract parent UUID from snapshot %q in project %q: %w", vol.Name(), projectName, err) + } + + // Extract parent volume UUID. + parentUUID := parentDBVol.Config["volatile.uuid"] + if parentUUID == "" { + return "", fmt.Errorf("Parent volume %q of snapshot %q in project %q does not have UUID set)", parentName, projectName, vol.Name()) + } + + return parentUUID, nil +} diff --git a/lxd/storage/connectors/connector.go b/lxd/storage/connectors/connector.go new file mode 100644 index 000000000000..2cb476faf721 --- /dev/null +++ b/lxd/storage/connectors/connector.go @@ -0,0 +1,84 @@ +package connectors + +import ( + "context" +) + +const ( + // TypeUnknown represents an unknown storage connector. + TypeUnknown string = "unknown" + + // TypeISCSI represents an iSCSI storage connector. + TypeISCSI string = "iscsi" + + // TypeNVME represents an NVMe/TCP storage connector. + TypeNVME string = "nvme" + + // TypeSDC represents Dell SDC storage connector. + TypeSDC string = "sdc" +) + +// Connector represents a storage connector that handles connections through +// appropriate storage subsystem. +type Connector interface { + Type() string + Version() (string, error) + QualifiedName() (string, error) + LoadModules() bool + SessionID(targetQN string) (string, error) + Connect(ctx context.Context, targetAddr string, targetQN string) error + ConnectAll(ctx context.Context, targetAddr string) error + Disconnect(targetQN string) error + DisconnectAll() error +} + +// NewConnector instantiates a new connector of the given type. +// The caller needs to ensure connector type is validated before calling this +// function, as common (empty) connector is returned for unknown type. +func NewConnector(connectorType string, serverUUID string) Connector { + common := common{ + serverUUID: serverUUID, + } + + switch connectorType { + case TypeISCSI: + return &connectorISCSI{ + common: common, + } + + case TypeNVME: + return &connectorNVMe{ + common: common, + } + + case TypeSDC: + return &connectorNVMe{ + common: common, + } + + default: + // Return common connector if the type is unknown. This removes + // the need to check for nil or handle the error in the caller. + return &common + } +} + +// GetSupportedVersions returns the versions for the given connector types +// ignoring those that produce an error when version is being retrieved +// (e.g. due to a missing required tools). +func GetSupportedVersions(connectorTypes []string) []string { + versions := make([]string, 0, len(connectorTypes)) + + // Iterate over the supported connectors, extracting version and loading + // kernel module for each of them. + for _, connectorType := range connectorTypes { + version, err := NewConnector(connectorType, "").Version() + if err != nil { + continue + } + + versions = append(versions, version) + } + + return versions +} diff --git a/lxd/storage/connectors/connector_common.go b/lxd/storage/connectors/connector_common.go new file mode 100644 index 000000000000..a875b6725c5d --- /dev/null +++ b/lxd/storage/connectors/connector_common.go @@ -0,0 +1,58 @@ +package connectors + +import ( + "context" + "fmt" +) + +var _ Connector = &common{} + +type common struct { + serverUUID string +} + +// Type returns the name of the connector. +func (c *common) Type() string { + return TypeUnknown +} + +// Version returns the version of the connector. +func (c *common) Version() (string, error) { + return "", fmt.Errorf("Version not implemented") +} + +// QualifiedName returns the qualified name of the connector. +func (c *common) QualifiedName() (string, error) { + return "", fmt.Errorf("QualifiedName not implemented") +} + +// LoadModules loads the necessary kernel modules. +func (c *common) LoadModules() bool { + return true +} + +// SessionID returns the identifier of a session that matches the connector's qualified name. +// If there is no such session, an empty string is returned. +func (c *common) SessionID(targetQN string) (string, error) { + return "", fmt.Errorf("ExistingSession not implemented") +} + +// Connect establishes a connection with the target on the given address. +func (c common) Connect(ctx context.Context, targetAddr string, targetQN string) error { + return fmt.Errorf("Connect not implemented") +} + +// ConnectAll establishes a connection with all targets available on the given address. +func (c common) ConnectAll(ctx context.Context, targetAddr string) error { + return fmt.Errorf("ConnectAll not implemented") +} + +// Disconnect terminates a connection with the target. +func (c common) Disconnect(targetQN string) error { + return fmt.Errorf("Disconnect not implemented") +} + +// DisconnectAll terminates all connections with all targets. +func (c common) DisconnectAll() error { + return fmt.Errorf("DisconnectAll not implemented") +} diff --git a/lxd/storage/connectors/connector_iscsi.go b/lxd/storage/connectors/connector_iscsi.go new file mode 100644 index 000000000000..f8160599fcd4 --- /dev/null +++ b/lxd/storage/connectors/connector_iscsi.go @@ -0,0 +1,217 @@ +package connectors + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/canonical/lxd/lxd/util" + "github.com/canonical/lxd/shared" +) + +var _ Connector = &connectorISCSI{} + +type connectorISCSI struct { + common + + iqn string +} + +// Type returns the type of the connector. +func (c *connectorISCSI) Type() string { + return TypeISCSI +} + +// Version returns the version of the iSCSI CLI (iscsiadm). +func (c *connectorISCSI) Version() (string, error) { + // Detect and record the version of the iSCSI CLI. + // It will fail if the "iscsiadm" is not installed on the host. + out, err := shared.RunCommand("iscsiadm", "--version") + if err != nil { + return "", fmt.Errorf("Failed to get iscsiadm version: %w", err) + } + + fields := strings.Split(strings.TrimSpace(out), " ") + if strings.HasPrefix(out, "iscsiadm version ") && len(fields) > 2 { + version := fmt.Sprintf("%s (iscsiadm)", fields[2]) + return version, nil + } + + return "", fmt.Errorf("Failed to get iscsiadm version: Unexpected output %q", out) +} + +// LoadModules loads the iSCSI kernel modules. +// Returns true if the modules can be loaded. +func (c *connectorISCSI) LoadModules() bool { + return util.LoadModule("iscsi_tcp") == nil +} + +// QualifiedName returns the unique iSCSI Qualified Name (IQN) of the host. +func (c *connectorISCSI) QualifiedName() (string, error) { + if c.iqn != "" { + return c.iqn, nil + } + + // Get the unique iSCSI Qualified Name (IQN) of the host. The iscsiadm + // does not allow providing the IQN directly, so we need to extract it + // from the /etc/iscsi/initiatorname.iscsi file on the host. + filename := shared.HostPath("/etc/iscsi/initiatorname.iscsi") + if !shared.PathExists(filename) { + return "", fmt.Errorf("Failed to extract host IQN: File %q does not exist", filename) + } + + content, err := os.ReadFile(filename) + if err != nil { + return "", err + } + + // Find the IQN line in the file. + lines := strings.Split(string(content), "\n") + for _, line := range lines { + iqn, ok := strings.CutPrefix(line, "InitiatorName=") + if ok { + c.iqn = iqn + return iqn, nil + } + } + + return "", fmt.Errorf(`Failed to extract host IQN: File %q does not contain "InitiatorName"`, filename) +} + +// SessionID returns the ID of an iSCSI session that corresponds +// to the server's qualified name (IQN). If the session is not found, +// an empty string is returned. +func (c *connectorISCSI) SessionID(targetQN string) (string, error) { + // Base path for iSCSI sessions. + basePath := "/sys/class/iscsi_session" + + // Retrieve list of existing iSCSI sessions. + sessions, err := os.ReadDir(basePath) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // No active sessions. + return "", nil + } + + return "", fmt.Errorf("Failed getting a list of existing iSCSI sessions: %w", err) + } + + for _, session := range sessions { + // Get the target IQN of the iSCSI session. + iqnBytes, err := os.ReadFile(filepath.Join(basePath, session.Name(), "targetname")) + if err != nil { + return "", fmt.Errorf("Failed getting the target IQN for session %q: %w", session, err) + } + + sessionIQN := strings.TrimSpace(string(iqnBytes)) + sessionID := strings.TrimPrefix(session.Name(), "session") + + if targetQN == sessionIQN { + // Already connected. + return sessionID, nil + } + } + + return "", nil +} + +// discoverTargets discovers the available iSCSI targets on a given address. +func (c *connectorISCSI) discoverTargets(ctx context.Context, targetAddr string) error { + // Discover the available iSCSI targets on a given address. + _, _, err := shared.RunCommandSplit(ctx, nil, nil, "iscsiadm", "--mode", "discovery", "--type", "sendtargets", "--portal", targetAddr) + if err != nil { + return fmt.Errorf("Failed to discover available iSCSI targets on %q: %w", targetAddr, err) + } + + return nil +} + +// Connect establishes a connection with the target on the given address. +func (c *connectorISCSI) Connect(ctx context.Context, targetAddr string, targetQN string) error { + // Try to find an existing iSCSI session. + sessionID, err := c.SessionID(targetQN) + if err != nil { + return err + } + + if sessionID != "" { + // Already connected. + // Rescan the session to ensure new volumes are detected. + _, err := shared.RunCommand("iscsiadm", "--mode", "session", "--sid", sessionID, "--rescan") + if err != nil { + return err + } + + return nil + } + + err = c.discoverTargets(ctx, targetAddr) + if err != nil { + return err + } + + // Attempt to login into iSCSI target. + _, stderr, err := shared.RunCommandSplit(ctx, nil, nil, "iscsiadm", "--mode", "node", "--targetname", targetQN, "--portal", targetAddr, "--login") + if err != nil { + return fmt.Errorf("Failed to connect to target %q on %q via iSCSI: %w", targetQN, targetAddr, err) + } + + if stderr != "" { + return fmt.Errorf("Failed to connect to target %q on %q via iSCSI: %s", targetQN, targetAddr, stderr) + } + + return nil +} + +// ConnectAll establishes a connection with all targets available on the given address. +func (c *connectorISCSI) ConnectAll(ctx context.Context, targetAddr string) error { + err := c.discoverTargets(ctx, targetAddr) + if err != nil { + return err + } + + // Attempt to login into all iSCSI targets. + _, stderr, err := shared.RunCommandSplit(ctx, nil, nil, "iscsiadm", "--mode", "node", "--portal", targetAddr, "--login") + if err != nil { + return fmt.Errorf("Failed to connect any target on %q via iSCSI: %w", targetAddr, err) + } + + if stderr != "" { + return fmt.Errorf("Failed to connect any target on %q via iSCSI: %s", targetAddr, stderr) + } + + return fmt.Errorf("ConnectAll not implemented") +} + +// Disconnect terminates a connection with the target. +func (c *connectorISCSI) Disconnect(targetQN string) error { + // Find an existing iSCSI session. + sessionID, err := c.SessionID(targetQN) + if err != nil { + return err + } + + // Disconnect from the iSCSI target if there is an existing session. + if sessionID != "" { + _, err := shared.RunCommand("iscsiadm", "--mode", "node", "--targetname", targetQN, "--logout") + if err != nil { + return fmt.Errorf("Failed disconnecting from iSCSI target %q: %w", targetQN, err) + } + } + + return nil +} + +// DisconnectAll terminates all connections with all targets. +func (c *connectorISCSI) DisconnectAll() error { + _, err := shared.RunCommand("iscsiadm", "--mode", "node", "--logoutall", "all") + if err != nil { + return fmt.Errorf("Failed disconnecting from iSCSI targets: %w", err) + } + + return nil +} diff --git a/lxd/storage/connectors/connector_nvme.go b/lxd/storage/connectors/connector_nvme.go new file mode 100644 index 000000000000..cde08ddd1c03 --- /dev/null +++ b/lxd/storage/connectors/connector_nvme.go @@ -0,0 +1,171 @@ +package connectors + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/canonical/lxd/lxd/util" + "github.com/canonical/lxd/shared" +) + +var _ Connector = &connectorNVMe{} + +type connectorNVMe struct { + common +} + +// Type returns the type of the connector. +func (c *connectorNVMe) Type() string { + return TypeNVME +} + +// Version returns the version of the NVMe CLI. +func (c *connectorNVMe) Version() (string, error) { + // Detect and record the version of the NVMe CLI. + out, err := shared.RunCommand("nvme", "version") + if err != nil { + return "", fmt.Errorf("Failed to get nvme-cli version: %w", err) + } + + fields := strings.Split(strings.TrimSpace(out), " ") + if strings.HasPrefix(out, "nvme version ") && len(fields) > 2 { + return fmt.Sprintf("%s (nvme-cli)", fields[2]), nil + } + + return "", fmt.Errorf("Failed to get nvme-cli version: Unexpected output %q", out) +} + +// LoadModules loads the NVMe/TCP kernel modules. +// Returns true if the modules can be loaded. +func (c *connectorNVMe) LoadModules() bool { + err := util.LoadModule("nvme_fabrics") + if err != nil { + return false + } + + err = util.LoadModule("nvme_tcp") + return err == nil +} + +// QualifiedName returns a custom NQN generated from the server UUID. +// Getting the NQN from /etc/nvme/hostnqn would require the nvme-cli +// package to be installed on the host. +func (c *connectorNVMe) QualifiedName() (string, error) { + return fmt.Sprintf("nqn.2014-08.org.nvmexpress:uuid:%s", c.serverUUID), nil +} + +// SessionID returns the target's qualified name (NQN) if a corresponding +// session is found. Otherwise, an empty string is returned. +func (c *connectorNVMe) SessionID(targetQN string) (string, error) { + // Base path for NVMe sessions/subsystems. + basePath := "/sys/devices/virtual/nvme-subsystem" + + // Retrieve list of existing NVMe sessions on this host. + directories, err := os.ReadDir(basePath) + if err != nil { + if os.IsNotExist(err) { + // No active sessions because NVMe subsystems directory does not exist. + return "", nil + } + + return "", fmt.Errorf("Failed getting a list of existing NVMe subsystems: %w", err) + } + + for _, directory := range directories { + subsystemName := directory.Name() + + // Get the target NQN. + nqnBytes, err := os.ReadFile(filepath.Join(basePath, subsystemName, "subsysnqn")) + if err != nil { + return "", fmt.Errorf("Failed getting the target NQN for subystem %q: %w", subsystemName, err) + } + + if strings.Contains(string(nqnBytes), targetQN) { + // Already connected. + return targetQN, nil + } + } + + return "", nil +} + +// Connect establishes a connection with the target on the given address. +func (c *connectorNVMe) Connect(ctx context.Context, targetAddr string, targetQN string) error { + hostNQN, err := c.QualifiedName() + if err != nil { + return err + } + + // Try to find an existing NVMe session. + targetNQN, err := c.SessionID(targetQN) + if err != nil { + return err + } + + if targetNQN != "" { + // Already connected. + return nil + } + + _, stderr, err := shared.RunCommandSplit(ctx, nil, nil, "nvme", "connect", "--transport", "tcp", "--traddr", targetAddr, "--nqn", targetQN, "--hostnqn", hostNQN, "--hostid", c.serverUUID) + if err != nil { + return fmt.Errorf("Failed to connect to target %q on %q via NVMe: %w", targetQN, targetAddr, err) + } + + if stderr != "" { + return fmt.Errorf("Failed to connect to target %q on %q via NVMe: %s", targetQN, targetAddr, stderr) + } + + return nil +} + +// ConnectAll establishes a connection with all targets available on the given address. +func (c *connectorNVMe) ConnectAll(ctx context.Context, targetAddr string) error { + hostNQN, err := c.QualifiedName() + if err != nil { + return err + } + + _, stderr, err := shared.RunCommandSplit(ctx, nil, nil, "nvme", "connect-all", "--transport", "tcp", "--traddr", targetAddr, "--hostnqn", hostNQN, "--hostid", c.serverUUID) + if err != nil { + return fmt.Errorf("Failed to connect to any target on %q via NVMe: %w", targetAddr, err) + } + + if stderr != "" { + return fmt.Errorf("Failed to connect to any target on %q via NVMe: %s", targetAddr, stderr) + } + + return nil +} + +// Disconnect terminates a connection with the target. +func (c *connectorNVMe) Disconnect(targetQN string) error { + // Find an existing NVMe session. + targetNQN, err := c.SessionID(targetQN) + if err != nil { + return err + } + + // Disconnect from the NVMe target if there is an existing session. + if targetNQN != "" { + _, err := shared.RunCommand("nvme", "disconnect", "--nqn", targetNQN) + if err != nil { + return fmt.Errorf("Failed disconnecting from NVMe target %q: %w", targetNQN, err) + } + } + + return nil +} + +// DisconnectAll terminates all connections with all targets. +func (c *connectorNVMe) DisconnectAll() error { + _, err := shared.RunCommand("nvme", "disconnect-all") + if err != nil { + return fmt.Errorf("Failed disconnecting from NVMe targets: %w", err) + } + + return nil +} diff --git a/lxd/storage/connectors/connector_sdc.go b/lxd/storage/connectors/connector_sdc.go new file mode 100644 index 000000000000..3bb074676b28 --- /dev/null +++ b/lxd/storage/connectors/connector_sdc.go @@ -0,0 +1,53 @@ +package connectors + +import ( + "context" +) + +var _ Connector = &connectorSDC{} + +type connectorSDC struct { + common +} + +// Type returns the type of the connector. +func (c *connectorSDC) Type() string { + return TypeSDC +} + +// LoadModules returns true. SDC does not require any kernel modules to be loaded. +func (c *connectorSDC) LoadModules() bool { + return true +} + +// QualifiedName returns an empty string and no error. SDC has no qualified name. +func (c *connectorSDC) QualifiedName() (string, error) { + return "", nil +} + +// SessionID returns an empty string and no error, as connections are handled by SDC. +func (c *connectorSDC) SessionID(targetQN string) (string, error) { + return "", nil +} + +// Connect does nothing. Connections are fully handled by SDC. +func (c *connectorSDC) Connect(ctx context.Context, targetAddr string, targetQN string) error { + // Nothing to do. Connection is handled by Dell SDC. + return nil +} + +// ConnectAll does nothing. Connections are fully handled by SDC. +func (c *connectorSDC) ConnectAll(ctx context.Context, targetAddr string) error { + // Nothing to do. Connection is handled by Dell SDC. + return nil +} + +// Disconnect does nothing. Connections are fully handled by SDC. +func (c *connectorSDC) Disconnect(targetQN string) error { + return nil +} + +// DisconnectAll does nothing. Connections are fully handled by SDC. +func (c *connectorSDC) DisconnectAll() error { + return nil +} diff --git a/lxd/storage/connectors/utils.go b/lxd/storage/connectors/utils.go new file mode 100644 index 000000000000..c7feeef872d9 --- /dev/null +++ b/lxd/storage/connectors/utils.go @@ -0,0 +1,126 @@ +package connectors + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "time" + + "golang.org/x/sys/unix" + + "github.com/canonical/lxd/lxd/resources" + "github.com/canonical/lxd/shared" +) + +// GetDiskDevicePath checks whether the disk device with a given prefix and suffix +// exists in /dev/disk/by-id directory. A device path is returned if the device is +// found, otherwise an error is returned. +func GetDiskDevicePath(diskNamePrefix string, diskNameSuffix string) (string, error) { + devPath, err := findDiskDevicePath(diskNamePrefix, diskNameSuffix) + if err != nil { + return "", err + } + + if devPath == "" { + return "", fmt.Errorf("Device not found") + } + + return devPath, nil +} + +// WaitDiskDevicePath waits for the disk device to appear in /dev/disk/by-id. +// It periodically checks for the device to appear and returns the device path +// once it is found. If the device does not appear within the timeout, an error +// is returned. +func WaitDiskDevicePath(ctx context.Context, diskNamePrefix string, diskNameSuffix string) (string, error) { + var err error + var devPath string + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + for { + // Check if the device is already present. + devPath, err = findDiskDevicePath(diskNamePrefix, diskNameSuffix) + if err != nil && !errors.Is(err, unix.ENOENT) { + return "", err + } + + // If the device is found, return the device path. + if devPath != "" { + break + } + + // Check if context is cancelled. + if ctx.Err() != nil { + return "", ctx.Err() + } + + time.Sleep(500 * time.Millisecond) + } + + if devPath == "" { + return "", fmt.Errorf("Device not found") + } + + return devPath, nil +} + +// findDiskDevivePath iterates over device names in /dev/disk/by-id directory and +// returns the path to the disk device that matches the given prefix and suffix. +// Disk partitions are skipped, and an error is returned if the device is not found. +func findDiskDevicePath(diskNamePrefix string, diskNameSuffix string) (string, error) { + var diskPaths []string + + // If there are no other disks on the system by id, the directory might not + // even be there. Returns ENOENT in case the by-id/ directory does not exist. + diskPaths, err := resources.GetDisksByID(diskNamePrefix) + if err != nil { + return "", err + } + + for _, diskPath := range diskPaths { + // Skip the disk if it is only a partition of the actual volume. + if strings.Contains(diskPath, "-part") { + continue + } + + // Skip volumes that do not contain the desired name suffix. + if !strings.HasSuffix(diskPath, diskNameSuffix) { + continue + } + + // The actual device might not already be created. + // Returns ENOENT in case the device does not exist. + devPath, err := filepath.EvalSymlinks(diskPath) + if err != nil { + return "", err + } + + return devPath, nil + } + + return "", nil +} + +// WaitDiskDeviceGone waits for the disk device to disappear from /dev/disk/by-id. +// It periodically checks for the device to disappear and returns once the device +// is gone. If the device does not disappear within the timeout, an error is returned. +func WaitDiskDeviceGone(ctx context.Context, diskPath string, timeoutSeconds int) bool { + ctx, cancel := context.WithTimeout(ctx, time.Duration(timeoutSeconds)*time.Second) + defer cancel() + + for { + if !shared.PathExists(diskPath) { + return true + } + + if ctx.Err() != nil { + return false + } + + time.Sleep(500 * time.Millisecond) + } +} diff --git a/lxd/storage/drivers/driver_btrfs.go b/lxd/storage/drivers/driver_btrfs.go index 334399828d3f..f16bfc67fa42 100644 --- a/lxd/storage/drivers/driver_btrfs.go +++ b/lxd/storage/drivers/driver_btrfs.go @@ -106,6 +106,7 @@ func (d *btrfs) Info() Info { IOUring: true, MountedRoot: true, Buckets: true, + PopulateParentVolumeUUID: false, } } diff --git a/lxd/storage/drivers/driver_ceph.go b/lxd/storage/drivers/driver_ceph.go index 04ecbfcab163..8db205491ed9 100644 --- a/lxd/storage/drivers/driver_ceph.go +++ b/lxd/storage/drivers/driver_ceph.go @@ -92,6 +92,7 @@ func (d *ceph) Info() Info { DirectIO: true, IOUring: true, MountedRoot: false, + PopulateParentVolumeUUID: false, } } diff --git a/lxd/storage/drivers/driver_cephfs.go b/lxd/storage/drivers/driver_cephfs.go index 19567fdd4947..eaa656900785 100644 --- a/lxd/storage/drivers/driver_cephfs.go +++ b/lxd/storage/drivers/driver_cephfs.go @@ -91,6 +91,7 @@ func (d *cephfs) Info() Info { RunningCopyFreeze: false, DirectIO: true, MountedRoot: true, + PopulateParentVolumeUUID: false, } } diff --git a/lxd/storage/drivers/driver_cephobject.go b/lxd/storage/drivers/driver_cephobject.go index efa344c0c7cc..c83efabf5c25 100644 --- a/lxd/storage/drivers/driver_cephobject.go +++ b/lxd/storage/drivers/driver_cephobject.go @@ -78,18 +78,19 @@ func (d *cephobject) isRemote() bool { // Info returns the pool driver information. func (d *cephobject) Info() Info { return Info{ - Name: "cephobject", - Version: cephobjectVersion, - OptimizedImages: false, - PreservesInodes: false, - Remote: d.isRemote(), - Buckets: true, - VolumeTypes: []VolumeType{}, - VolumeMultiNode: false, - BlockBacking: false, - RunningCopyFreeze: false, - DirectIO: false, - MountedRoot: false, + Name: "cephobject", + Version: cephobjectVersion, + OptimizedImages: false, + PreservesInodes: false, + Remote: d.isRemote(), + Buckets: true, + VolumeTypes: []VolumeType{}, + VolumeMultiNode: false, + BlockBacking: false, + RunningCopyFreeze: false, + DirectIO: false, + MountedRoot: false, + PopulateParentVolumeUUID: false, } } diff --git a/lxd/storage/drivers/driver_dir.go b/lxd/storage/drivers/driver_dir.go index c5f3b70e1b77..0d7108804c78 100644 --- a/lxd/storage/drivers/driver_dir.go +++ b/lxd/storage/drivers/driver_dir.go @@ -49,6 +49,7 @@ func (d *dir) Info() Info { IOUring: true, MountedRoot: true, Buckets: true, + PopulateParentVolumeUUID: false, } } diff --git a/lxd/storage/drivers/driver_lvm.go b/lxd/storage/drivers/driver_lvm.go index 0c85bdb1d5ef..43c68e33e9cd 100644 --- a/lxd/storage/drivers/driver_lvm.go +++ b/lxd/storage/drivers/driver_lvm.go @@ -98,6 +98,7 @@ func (d *lvm) Info() Info { IOUring: true, MountedRoot: false, Buckets: true, + PopulateParentVolumeUUID: false, } } diff --git a/lxd/storage/drivers/driver_mock.go b/lxd/storage/drivers/driver_mock.go index cf00dc4e4063..f49fdfc377d1 100644 --- a/lxd/storage/drivers/driver_mock.go +++ b/lxd/storage/drivers/driver_mock.go @@ -35,6 +35,7 @@ func (d *mock) Info() Info { RunningCopyFreeze: true, DirectIO: true, MountedRoot: true, + PopulateParentVolumeUUID: false, } } diff --git a/lxd/storage/drivers/driver_powerflex.go b/lxd/storage/drivers/driver_powerflex.go index b523302e41ad..92012469f847 100644 --- a/lxd/storage/drivers/driver_powerflex.go +++ b/lxd/storage/drivers/driver_powerflex.go @@ -8,6 +8,7 @@ import ( "github.com/canonical/lxd/lxd/migration" "github.com/canonical/lxd/lxd/operations" + "github.com/canonical/lxd/lxd/storage/connectors" "github.com/canonical/lxd/shared" "github.com/canonical/lxd/shared/api" "github.com/canonical/lxd/shared/validate" @@ -19,10 +20,10 @@ const powerFlexDefaultUser = "admin" // powerFlexDefaultSize represents the default PowerFlex volume size. const powerFlexDefaultSize = "8GiB" -const ( - powerFlexModeNVMe = "nvme" - powerFlexModeSDC = "sdc" -) +var powerflexSupportedConnectors = []string{ + connectors.TypeNVME, + connectors.TypeSDC, +} var powerFlexLoaded bool var powerFlexVersion string @@ -30,6 +31,10 @@ var powerFlexVersion string type powerflex struct { common + // Holds the low level connector for the PowerFlex driver. + // Use powerflex.connector() to retrieve the initialized connector. + storageConnector connectors.Connector + // Holds the low level HTTP client for the PowerFlex API. // Use powerflex.client() to retrieve the client struct. httpClient *powerFlexClient @@ -46,28 +51,29 @@ func (d *powerflex) load() error { return nil } - // Detect and record the version. - // The NVMe CLI is shipped with the snap. - out, err := shared.RunCommand("nvme", "version") - if err != nil { - return fmt.Errorf("Failed to get nvme-cli version: %w", err) - } - - fields := strings.Split(strings.TrimSpace(out), " ") - if strings.HasPrefix(out, "nvme version ") && len(fields) > 2 { - powerFlexVersion = fmt.Sprintf("%s (nvme-cli)", fields[2]) - } + versions := connectors.GetSupportedVersions(powerflexSupportedConnectors) + powerFlexVersion = strings.Join(versions, " / ") + powerFlexLoaded = true - // Load the NVMe/TCP kernel modules. + // Load the kernel modules of the respective connector. // Ignore if the modules cannot be loaded. - // Support for the NVMe/TCP mode is checked during pool creation. + // Support for a specific connector is checked during pool creation. // When a LXD host gets rebooted this ensures that the kernel modules are still loaded. - _ = d.loadNVMeModules() + _ = d.connector().LoadModules() - powerFlexLoaded = true return nil } +// connector retrieves an initialized storage connector based on the configured +// PowerFlex mode. The connector is cached in the driver struct. +func (d *powerflex) connector() connectors.Connector { + if d.storageConnector == nil { + d.storageConnector = connectors.NewConnector(d.config["powerflex.mode"], d.state.ServerUUID) + } + + return d.storageConnector +} + // isRemote returns true indicating this driver uses remote storage. func (d *powerflex) isRemote() bool { return true @@ -89,6 +95,7 @@ func (d *powerflex) Info() Info { DirectIO: true, IOUring: true, MountedRoot: false, + PopulateParentVolumeUUID: false, } } @@ -102,10 +109,13 @@ func (d *powerflex) FillConfig() error { // First try if the NVMe/TCP kernel modules can be loaed. // Second try if the SDC kernel module is setup. if d.config["powerflex.mode"] == "" { - if d.loadNVMeModules() { - d.config["powerflex.mode"] = powerFlexModeNVMe + // Create temporary connector to check if NVMe/TCP kernel modules can be loaded. + nvmeConnector := connectors.NewConnector(connectors.TypeNVME, "") + + if nvmeConnector.LoadModules() { + d.config["powerflex.mode"] = connectors.TypeNVME } else if goscaleio.DrvCfgIsSDCInstalled() { - d.config["powerflex.mode"] = powerFlexModeSDC + d.config["powerflex.mode"] = connectors.TypeSDC } } @@ -139,7 +149,7 @@ func (d *powerflex) Create() error { client := d.client() switch d.config["powerflex.mode"] { - case powerFlexModeNVMe: + case connectors.TypeNVME: // Discover one of the storage pools SDT services. if d.config["powerflex.sdt"] == "" { pool, err := d.resolvePool() @@ -163,7 +173,7 @@ func (d *powerflex) Create() error { d.config["powerflex.sdt"] = relations[0].IPList[0].IP } - case powerFlexModeSDC: + case connectors.TypeSDC: if d.config["powerflex.sdt"] != "" { return fmt.Errorf("The powerflex.sdt config key is specific to the NVMe/TCP mode") } @@ -280,14 +290,23 @@ func (d *powerflex) Validate(config map[string]string) error { return err } + newMode := config["powerflex.mode"] + oldMode := d.config["powerflex.mode"] + + // Ensure powerflex.mode cannot be changed to avoid leaving volume mappings + // and to prevent disturbing running instances. + if oldMode != "" && oldMode != newMode { + return fmt.Errorf("PowerFlex mode cannot be changed") + } + // Check if the selected PowerFlex mode is supported on this node. // Also when forming the storage pool on a LXD cluster, the mode // that got discovered on the creating machine needs to be validated // on the other cluster members too. This can be done here since Validate // gets executed on every cluster member when receiving the cluster // notification to finally create the pool. - if d.config["powerflex.mode"] == powerFlexModeNVMe && !d.loadNVMeModules() { - return fmt.Errorf("NVMe/TCP is not supported") + if newMode != "" && !connectors.NewConnector(newMode, "").LoadModules() { + return fmt.Errorf("PowerFlex mode %q is not supported due to missing kernel modules", newMode) } return nil diff --git a/lxd/storage/drivers/driver_powerflex_utils.go b/lxd/storage/drivers/driver_powerflex_utils.go index 6c8433f01da9..dd01b626c897 100644 --- a/lxd/storage/drivers/driver_powerflex_utils.go +++ b/lxd/storage/drivers/driver_powerflex_utils.go @@ -2,27 +2,20 @@ package drivers import ( "bytes" - "context" "crypto/tls" "encoding/base64" "encoding/json" - "errors" "fmt" "io" "net/http" - "os" - "path/filepath" "strconv" "strings" - "time" "github.com/dell/goscaleio" "github.com/google/uuid" - "golang.org/x/sys/unix" "github.com/canonical/lxd/lxd/locking" - "github.com/canonical/lxd/lxd/resources" - "github.com/canonical/lxd/lxd/util" + "github.com/canonical/lxd/lxd/storage/connectors" "github.com/canonical/lxd/shared" "github.com/canonical/lxd/shared/api" "github.com/canonical/lxd/shared/revert" @@ -65,7 +58,7 @@ type powerFlexError map[string]any // Error tries to return all kinds of errors from the PowerFlex API in a nicely formatted way. func (p *powerFlexError) Error() string { - var errorStrings []string + errorStrings := make([]string, 0, len(*p)) for k, v := range *p { errorStrings = append(errorStrings, fmt.Sprintf("%s: %v", k, v)) } @@ -747,18 +740,6 @@ func (p *powerFlexClient) getHostVolumeMappings(hostID string) ([]powerFlexVolum return actualResponse, nil } -// loadNVMeModules loads the NVMe/TCP kernel modules. -// Returns true if the modules can be loaded. -func (d *powerflex) loadNVMeModules() bool { - err := util.LoadModule("nvme_fabrics") - if err != nil { - return false - } - - err = util.LoadModule("nvme_tcp") - return err == nil -} - // client returns the drivers PowerFlex client. // A new client gets created if it not yet exists. func (d *powerflex) client() *powerFlexClient { @@ -769,13 +750,6 @@ func (d *powerflex) client() *powerFlexClient { return d.httpClient } -// getHostNQN returns the unique NVMe nqn for the current host. -// A custom one is generated from the servers UUID since getting the nqn from /etc/nvme/hostnqn -// requires the nvme-cli package to be installed on the host. -func (d *powerflex) getHostNQN() string { - return fmt.Sprintf("nqn.2014-08.org.nvmexpress:uuid:%s", d.state.ServerUUID) -} - // getHostGUID returns the SDC GUID. // The GUID is unique for a single host. // Cache the GUID as it never changes for a single host. @@ -792,21 +766,6 @@ func (d *powerflex) getHostGUID() (string, error) { return d.sdcGUID, nil } -// getServerName returns the hostname of this host. -// It prefers the value from the daemons state in case LXD is clustered. -func (d *powerflex) getServerName() (string, error) { - if d.state.ServerName != "none" { - return d.state.ServerName, nil - } - - hostname, err := os.Hostname() - if err != nil { - return "", fmt.Errorf("Failed to get hostname: %w", err) - } - - return hostname, nil -} - // getVolumeType returns the selected provisioning type of the volume. // As a default it returns type thin. func (d *powerflex) getVolumeType(vol Volume) powerFlexVolumeType { @@ -825,24 +784,28 @@ func (d *powerflex) getVolumeType(vol Volume) powerFlexVolumeType { // createNVMeHost creates this NVMe host in PowerFlex. func (d *powerflex) createNVMeHost() (string, revert.Hook, error) { var hostID string - nqn := d.getHostNQN() + + targetNQN, err := d.connector().QualifiedName() + if err != nil { + return "", nil, err + } revert := revert.New() defer revert.Fail() client := d.client() - host, err := client.getNVMeHostByNQN(nqn) + host, err := client.getNVMeHostByNQN(targetNQN) if err != nil { if !api.StatusErrorCheck(err, http.StatusNotFound) { return "", nil, err } - hostname, err := d.getServerName() + hostname, err := ResolveServerName(d.state.ServerName) if err != nil { return "", nil, err } - hostID, err = client.createHost(hostname, nqn) + hostID, err = client.createHost(hostname, targetNQN) if err != nil { return "", nil, err } @@ -862,8 +825,13 @@ func (d *powerflex) createNVMeHost() (string, revert.Hook, error) { // deleteNVMeHost deletes this NVMe host in PowerFlex. func (d *powerflex) deleteNVMeHost() error { client := d.client() - nqn := d.getHostNQN() - host, err := client.getNVMeHostByNQN(nqn) + + targetNQN, err := d.connector().QualifiedName() + if err != nil { + return err + } + + host, err := client.getNVMeHostByNQN(targetNQN) if err != nil { // Skip the deletion if the host doesn't exist anymore. if api.StatusErrorCheck(err, http.StatusNotFound) { @@ -884,8 +852,8 @@ func (d *powerflex) mapVolume(vol Volume) (revert.Hook, error) { var hostID string switch d.config["powerflex.mode"] { - case powerFlexModeNVMe: - unlock, err := locking.Lock(d.state.ShutdownCtx, "nvme") + case connectors.TypeNVME: + unlock, err := locking.Lock(d.state.ShutdownCtx, "storage_powerflex_nvme") if err != nil { return nil, err } @@ -899,7 +867,7 @@ func (d *powerflex) mapVolume(vol Volume) (revert.Hook, error) { } reverter.Add(cleanup) - case powerFlexModeSDC: + case connectors.TypeSDC: hostGUID, err := d.getHostGUID() if err != nil { return nil, err @@ -946,19 +914,26 @@ func (d *powerflex) mapVolume(vol Volume) (revert.Hook, error) { reverter.Add(func() { _ = client.deleteHostVolumeMapping(hostID, volumeID) }) } - if d.config["powerflex.mode"] == powerFlexModeNVMe { - // Connect to the NVMe/TCP subsystem. - // We have to connect after the first mapping was established. - // PowerFlex does not offer any discovery log entries until a volume gets mapped to the host. - // This action is idempotent. - cleanup, err := d.connectNVMeSubsys() - if err != nil { - return nil, err - } + pool, err := d.resolvePool() + if err != nil { + return nil, err + } - reverter.Add(cleanup) + domain, err := d.client().getProtectionDomain(pool.ProtectionDomainID) + if err != nil { + return nil, err + } + + targetQN := domain.SystemID + targetAddr := d.config["powerflex.sdt"] + + err = d.connector().Connect(d.state.ShutdownCtx, targetAddr, targetQN) + if err != nil { + return nil, err } + reverter.Add(func() { _ = d.connector().Disconnect(targetQN) }) + cleanup := reverter.Clone().Fail reverter.Success() return cleanup, nil @@ -979,44 +954,6 @@ func (d *powerflex) getMappedDevPath(vol Volume, mapVolume bool) (string, revert revert.Add(cleanup) } - powerFlexVolumes := make(map[string]string) - - // discoverFunc has to be called in a loop with a set timeout to ensure - // all the necessary directories and devices can be discovered. - discoverFunc := func(volumeID string, diskPrefix string) error { - var diskPaths []string - - // If there are no other disks on the system by id, the directory might not even be there. - // Returns ENOENT in case the by-id/ directory does not exist. - diskPaths, err := resources.GetDisksByID(diskPrefix) - if err != nil { - return err - } - - for _, diskPath := range diskPaths { - // Skip the disk if it is only a partition of the actual PowerFlex volume. - if strings.Contains(diskPath, "-part") { - continue - } - - // Skip other volume's that don't match the PowerFlex volume's ID. - if !strings.Contains(diskPath, volumeID) { - continue - } - - // The actual device might not already be created. - // Returns ENOENT in case the device does not exist. - devPath, err := filepath.EvalSymlinks(diskPath) - if err != nil { - return err - } - - powerFlexVolumes[volumeID] = devPath - } - - return nil - } - volumeName, err := d.getVolumeName(vol) if err != nil { return "", nil, err @@ -1027,59 +964,30 @@ func (d *powerflex) getMappedDevPath(vol Volume, mapVolume bool) (string, revert return "", nil, err } - timeout := time.Now().Add(5 * time.Second) - // It might take a while to create the local disk. - // Retry until it can be found. - for { - if time.Now().After(timeout) { - return "", nil, fmt.Errorf("Timeout exceeded for PowerFlex volume discovery: %q", volumeName) - } - - var prefix string - switch d.config["powerflex.mode"] { - case powerFlexModeNVMe: - prefix = "nvme-eui." - case powerFlexModeSDC: - prefix = "emc-vol-" - } - - err := discoverFunc(powerFlexVolumeID, prefix) - if err != nil { - // Try again if on of the directories cannot be found. - if errors.Is(err, unix.ENOENT) { - continue - } - - return "", nil, err - } - - // Exit if the volume got discovered. - _, ok := powerFlexVolumes[powerFlexVolumeID] - if ok { - break - } - - // Exit if the volume wasn't explicitly mapped. - // Doing a retry would run into the timeout when the device isn't mapped. - if !mapVolume { - break - } - - time.Sleep(10 * time.Millisecond) + var prefix string + switch d.config["powerflex.mode"] { + case connectors.TypeNVME: + prefix = "nvme-eui." + case connectors.TypeSDC: + prefix = "emc-vol-" } - if len(powerFlexVolumes) == 0 { - return "", nil, fmt.Errorf("Failed to discover any PowerFlex volume") + var devicePath string + if mapVolume { + // Wait for the device path to appear as the volume has been just mapped to the host. + devicePath, err = connectors.WaitDiskDevicePath(d.state.ShutdownCtx, prefix, powerFlexVolumeID) + } else { + // Get the the device path without waiting. + devicePath, err = connectors.GetDiskDevicePath(prefix, powerFlexVolumeID) } - powerFlexVolumePath, ok := powerFlexVolumes[powerFlexVolumeID] - if !ok { - return "", nil, fmt.Errorf("PowerFlex volume not found: %q", volumeName) + if err != nil { + return "", nil, fmt.Errorf("Failed to locate device for volume %q: %w", vol.name, err) } cleanup := revert.Clone().Fail revert.Success() - return powerFlexVolumePath, cleanup, nil + return devicePath, cleanup, nil } // unmapVolume unmaps the given volume from this host. @@ -1097,20 +1005,24 @@ func (d *powerflex) unmapVolume(vol Volume) error { var host *powerFlexSDC switch d.config["powerflex.mode"] { - case powerFlexModeNVMe: - nqn := d.getHostNQN() - host, err = client.getNVMeHostByNQN(nqn) + case connectors.TypeNVME: + hostNQN, err := d.connector().QualifiedName() if err != nil { return err } - unlock, err := locking.Lock(d.state.ShutdownCtx, "nvme") + host, err = client.getNVMeHostByNQN(hostNQN) + if err != nil { + return err + } + + unlock, err := locking.Lock(d.state.ShutdownCtx, "storage_powerflex_nvme") if err != nil { return err } defer unlock() - case powerFlexModeSDC: + case connectors.TypeSDC: hostGUID, err := d.getHostGUID() if err != nil { return err @@ -1129,28 +1041,35 @@ func (d *powerflex) unmapVolume(vol Volume) error { // Wait until the volume has disappeared. volumePath, _, _ := d.getMappedDevPath(vol, false) - if volumePath != "" { - ctx, cancel := context.WithTimeout(d.state.ShutdownCtx, 10*time.Second) - defer cancel() - - if !waitGone(ctx, volumePath) { - return fmt.Errorf("Timeout whilst waiting for PowerFlex volume to disappear: %q", vol.name) - } + if volumePath != "" && !connectors.WaitDiskDeviceGone(d.state.ShutdownCtx, volumePath, 10) { + return fmt.Errorf("Timeout whilst waiting for PowerFlex volume to disappear: %q", vol.name) } // In case of SDC the driver doesn't manage the underlying connection to PowerFlex. // Therefore if this was the last volume being unmapped from this system // LXD will not try to cleanup the connection. - if d.config["powerflex.mode"] == powerFlexModeNVMe { + if d.config["powerflex.mode"] == connectors.TypeNVME { mappings, err := client.getHostVolumeMappings(host.ID) if err != nil { return err } if len(mappings) == 0 { + pool, err := d.resolvePool() + if err != nil { + return err + } + + domain, err := d.client().getProtectionDomain(pool.ProtectionDomainID) + if err != nil { + return err + } + + targetQN := domain.SystemID + // Disconnect from the NVMe subsystem. // Do this first before removing the host from PowerFlex. - err := d.disconnectNVMeSubsys() + err = d.connector().Disconnect(targetQN) if err != nil { return err } @@ -1167,77 +1086,6 @@ func (d *powerflex) unmapVolume(vol Volume) error { return nil } -// connectNVMeSubsys connects this host to the NVMe subsystem configured in the storage pool. -// The connection can only be established after the first volume is mapped to this host. -// The operation is idempotent and returns nil if already connected to the subsystem. -func (d *powerflex) connectNVMeSubsys() (revert.Hook, error) { - basePath := "/sys/devices/virtual/nvme-subsystem" - - // Retrieve list of existing NVMe subsystems on this host. - directories, err := os.ReadDir(basePath) - if err != nil { - return nil, fmt.Errorf("Failed getting a list of NVMe subsystems: %w", err) - } - - revert := revert.New() - defer revert.Fail() - - pool, err := d.resolvePool() - if err != nil { - return nil, err - } - - domain, err := d.client().getProtectionDomain(pool.ProtectionDomainID) - if err != nil { - return nil, err - } - - for _, directory := range directories { - subsystemName := directory.Name() - - // Get the subsystem's NQN. - nqnBytes, err := os.ReadFile(filepath.Join(basePath, subsystemName, "subsysnqn")) - if err != nil { - return nil, fmt.Errorf("Failed getting the NQN of subystem %q: %w", subsystemName, err) - } - - if strings.Contains(string(nqnBytes), domain.SystemID) { - cleanup := revert.Clone().Fail - revert.Success() - - // Already connected to the NVMe subsystem for the respective PowerFlex system. - return cleanup, nil - } - } - - nqn := d.getHostNQN() - serverUUID := d.state.ServerUUID - _, stderr, err := shared.RunCommandSplit(d.state.ShutdownCtx, nil, nil, "nvme", "connect-all", "-t", "tcp", "-a", d.config["powerflex.sdt"], "-q", nqn, "-I", serverUUID) - if err != nil { - return nil, fmt.Errorf("Failed nvme connect-all: %w", err) - } - - if stderr != "" { - return nil, fmt.Errorf("Failed connecting to PowerFlex NVMe/TCP subsystem: %s", stderr) - } - - revert.Add(func() { _ = d.disconnectNVMeSubsys() }) - - cleanup := revert.Clone().Fail - revert.Success() - return cleanup, nil -} - -// disconnectNVMeSubsys disconnects this host from the NVMe subsystem. -func (d *powerflex) disconnectNVMeSubsys() error { - _, err := shared.RunCommand("nvme", "disconnect-all") - if err != nil { - return fmt.Errorf("Failed disconnecting from PowerFlex NVMe/TCP subsystem: %w", err) - } - - return nil -} - // resolvePool looks up the selected storage pool. // If only the pool is provided, it's expected to be the ID of the pool. // In case both pool and domain are set, the pool will get looked up diff --git a/lxd/storage/drivers/driver_pure.go b/lxd/storage/drivers/driver_pure.go new file mode 100644 index 000000000000..a26754264dda --- /dev/null +++ b/lxd/storage/drivers/driver_pure.go @@ -0,0 +1,352 @@ +package drivers + +import ( + "fmt" + "net/http" + "strings" + + "github.com/canonical/lxd/lxd/migration" + "github.com/canonical/lxd/lxd/operations" + "github.com/canonical/lxd/lxd/storage/connectors" + "github.com/canonical/lxd/shared" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/revert" + "github.com/canonical/lxd/shared/units" + "github.com/canonical/lxd/shared/validate" +) + +// pureLoaded indicates whether load() function was already called for the Pure Storage driver. +var pureLoaded = false + +// pureVersion indicates Pure Storage version. +var pureVersion = "" + +// pureSupportedConnectors is a list of supported Pure Storage connectors. +var pureSupportedConnectors = []string{ + connectors.TypeISCSI, + connectors.TypeNVME, +} + +type pure struct { + common + + // Holds the low level connector for the Pure Storage driver. + // Use pure.connector() to retrieve the initialized connector. + storageConnector connectors.Connector + + // Holds the low level HTTP client for the Pure Storage API. + // Use pure.client() to retrieve the client struct. + httpClient *pureClient + + // apiVersion indicates the Pure Storage API version. + apiVersion string +} + +// load is used initialize the driver. It should be used only once. +func (d *pure) load() error { + // Done if previously loaded. + if pureLoaded { + return nil + } + + versions := connectors.GetSupportedVersions(pureSupportedConnectors) + pureVersion = strings.Join(versions, " / ") + pureLoaded = true + + // Load the kernel modules of the respective connector, ignoring those that cannot + // be loaded. Support for a selected connector is checked during configuration + // validation. However, this ensures that the kernel modules are loaded, even if + // the host has been rebooted. + _ = d.connector().LoadModules() + + return nil +} + +func (d *pure) connector() connectors.Connector { + if d.storageConnector == nil { + d.storageConnector = connectors.NewConnector(d.config["pure.mode"], d.state.ServerUUID) + } + + return d.storageConnector +} + +// client returns the drivers Pure Storage client. A new client is created only if it does not already exist. +func (d *pure) client() *pureClient { + if d.httpClient == nil { + d.httpClient = newPureClient(d) + } + + return d.httpClient +} + +// isRemote returns true indicating this driver uses remote storage. +func (d *pure) isRemote() bool { + return true +} + +// Info returns info about the driver and its environment. +func (d *pure) Info() Info { + return Info{ + Name: "pure", + Version: pureVersion, + DefaultBlockSize: d.defaultBlockVolumeSize(), + DefaultVMBlockFilesystemSize: d.defaultVMBlockFilesystemSize(), + OptimizedImages: true, + PreservesInodes: false, + Remote: d.isRemote(), + VolumeTypes: []VolumeType{VolumeTypeCustom, VolumeTypeVM, VolumeTypeContainer, VolumeTypeImage}, + BlockBacking: true, + RunningCopyFreeze: true, + DirectIO: true, + IOUring: true, + MountedRoot: false, + PopulateParentVolumeUUID: true, + } +} + +// FillConfig populates the storage pool's configuration file with the default values. +func (d *pure) FillConfig() error { + // Use NVMe by default. + if d.config["pure.mode"] == "" { + d.config["pure.mode"] = connectors.TypeNVME + } + + return nil +} + +// Validate checks that all provided keys are supported and there is no conflicting or missing configuration. +func (d *pure) Validate(config map[string]string) error { + rules := map[string]func(value string) error{ + "size": validate.Optional(validate.IsSize), + // lxdmeta:generate(entities=storage-pure; group=pool-conf; key=pure.api.token) + // + // --- + // type: string + // shortdesc: API token for Pure Storage gateway authentication + "pure.api.token": validate.Optional(), + // lxdmeta:generate(entities=storage-pure; group=pool-conf; key=pure.gateway) + // + // --- + // type: string + // shortdesc: Address of the Pure Storage gateway + "pure.gateway": validate.Optional(validate.IsRequestURL), + // lxdmeta:generate(entities=storage-pure; group=pool-conf; key=pure.gateway.verify) + // + // --- + // type: bool + // defaultdesc: `true` + // shortdesc: Whether to verify the Pure Storage gateway's certificate + "pure.gateway.verify": validate.Optional(validate.IsBool), + // lxdmeta:generate(entities=storage-pure; group=pool-conf; key=pure.mode) + // The mode to use to map Pure Storage volumes to the local server. + // Supported values are `iscsi` and `nvme`. + // --- + // type: string + // defaultdesc: the discovered mode + // shortdesc: How volumes are mapped to the local server + "pure.mode": validate.Optional(validate.IsOneOf(connectors.TypeISCSI, connectors.TypeNVME)), + // lxdmeta:generate(entities=storage-pure; group=pool-conf; key=volume.size) + // Default Pure Storage volume size rounded to 512B. The minimum size is 1MiB. + // --- + // type: string + // defaultdesc: `10GiB` + // shortdesc: Size/quota of the storage volume + "volume.size": validate.Optional(validate.IsMultipleOfUnit("512B")), + } + + err := d.validatePool(config, rules, d.commonVolumeRules()) + if err != nil { + return err + } + + newMode := config["pure.mode"] + oldMode := d.config["pure.mode"] + + // Ensure pure.mode cannot be changed to avoid leaving volume mappings + // and to prevent disturbing running instances. + if oldMode != "" && oldMode != newMode { + return fmt.Errorf("Pure Storage mode cannot be changed") + } + + // Check if the selected Pure Storage mode is supported on this node. + // Also when forming the storage pool on a LXD cluster, the mode + // that got discovered on the creating machine needs to be validated + // on the other cluster members too. This can be done here since Validate + // gets executed on every cluster member when receiving the cluster + // notification to finally create the pool. + if newMode != "" && !connectors.NewConnector(newMode, "").LoadModules() { + return fmt.Errorf("Pure Storage mode %q is not supported due to missing kernel modules", newMode) + } + + return nil +} + +// Create is called during pool creation and is effectively using an empty driver struct. +// WARNING: The Create() function cannot rely on any of the struct attributes being set. +func (d *pure) Create() error { + err := d.FillConfig() + if err != nil { + return err + } + + revert := revert.New() + defer revert.Fail() + + // Validate required Pure Storage configuration keys and return an error if they are + // not set. Since those keys are not cluster member specific, the general validation + // rules allow empty strings in order to create the pending storage pools. + if d.config["pure.gateway"] == "" { + return fmt.Errorf("The pure.gateway cannot be empty") + } + + if d.config["pure.api.token"] == "" { + return fmt.Errorf("The pure.api.token cannot be empty") + } + + poolSizeBytes, err := units.ParseByteSizeString(d.config["size"]) + if err != nil { + return fmt.Errorf("Failed to parse storage size: %w", err) + } + + // Create the storage pool. + err = d.client().createStoragePool(d.name, poolSizeBytes) + if err != nil { + return err + } + + revert.Add(func() { _ = d.client().deleteStoragePool(d.name) }) + + revert.Success() + + return nil +} + +// Update applies any driver changes required from a configuration change. +func (d *pure) Update(changedConfig map[string]string) error { + newPoolSizeBytes, err := units.ParseByteSizeString(changedConfig["size"]) + if err != nil { + return fmt.Errorf("Failed to parse storage size: %w", err) + } + + oldPoolSizeBytes, err := units.ParseByteSizeString(d.config["size"]) + if err != nil { + return fmt.Errorf("Failed to parse old storage size: %w", err) + } + + if newPoolSizeBytes != oldPoolSizeBytes { + err = d.client().updateStoragePool(d.name, newPoolSizeBytes) + if err != nil { + return err + } + } + + return nil +} + +// Delete removes the storage pool (Pure Storage pod). +func (d *pure) Delete(op *operations.Operation) error { + // First delete the storage pool on Pure Storage. + err := d.client().deleteStoragePool(d.name) + if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { + return err + } + + // If the user completely destroyed it, call it done. + if !shared.PathExists(GetPoolMountPath(d.name)) { + return nil + } + + // On delete, wipe everything in the directory. + return wipeDirectory(GetPoolMountPath(d.name)) +} + +// Mount mounts the storage pool. +func (d *pure) Mount() (bool, error) { + // Nothing to do here. + return true, nil +} + +// Unmount unmounts the storage pool. +func (d *pure) Unmount() (bool, error) { + // Nothing to do here. + return true, nil +} + +// GetResources returns the pool resource usage information. +func (d *pure) GetResources() (*api.ResourcesStoragePool, error) { + pool, err := d.client().getStoragePool(d.name) + if err != nil { + return nil, err + } + + res := &api.ResourcesStoragePool{} + + res.Space.Total = uint64(pool.Quota) + res.Space.Used = uint64(pool.Space.UsedBytes) + + if pool.Quota == 0 { + // If quota is set to 0, it means that the storage pool is unbounded. Therefore, + // collect the total capacity of arrays where storage pool provisioned. + arrayNames := make([]string, 0, len(pool.Arrays)) + for _, array := range pool.Arrays { + arrayNames = append(arrayNames, array.Name) + } + + arrays, err := d.client().getStorageArrays(arrayNames...) + if err != nil { + return nil, err + } + + for _, array := range arrays { + res.Space.Total += uint64(array.Capacity) + } + } + + return res, nil +} + +// MigrationTypes returns the type of transfer methods to be used when doing migrations between pools in preference order. +func (d *pure) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool) []migration.Type { + var rsyncFeatures []string + + // Do not pass compression argument to rsync if the associated + // config key, that is rsync.compression, is set to false. + if shared.IsFalse(d.Config()["rsync.compression"]) { + rsyncFeatures = []string{"xattrs", "delete", "bidirectional"} + } else { + rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"} + } + + if refresh { + var transportType migration.MigrationFSType + + if IsContentBlock(contentType) { + transportType = migration.MigrationFSType_BLOCK_AND_RSYNC + } else { + transportType = migration.MigrationFSType_RSYNC + } + + return []migration.Type{ + { + FSType: transportType, + Features: rsyncFeatures, + }, + } + } + + if contentType == ContentTypeBlock { + return []migration.Type{ + { + FSType: migration.MigrationFSType_BLOCK_AND_RSYNC, + Features: rsyncFeatures, + }, + } + } + + return []migration.Type{ + { + FSType: migration.MigrationFSType_RSYNC, + Features: rsyncFeatures, + }, + } +} diff --git a/lxd/storage/drivers/driver_pure_util.go b/lxd/storage/drivers/driver_pure_util.go new file mode 100644 index 000000000000..edc342c19317 --- /dev/null +++ b/lxd/storage/drivers/driver_pure_util.go @@ -0,0 +1,1203 @@ +package drivers + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/canonical/lxd/lxd/locking" + "github.com/canonical/lxd/lxd/storage/connectors" + "github.com/canonical/lxd/shared" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/logger" + "github.com/canonical/lxd/shared/revert" +) + +// pureVolTypePrefixes maps volume type to storage volume name prefix. +// Use smallest possible prefixes since Pure Storage volume names are limited to 63 characters. +var pureVolTypePrefixes = map[VolumeType]string{ + VolumeTypeContainer: "c", + VolumeTypeVM: "v", + VolumeTypeImage: "i", + VolumeTypeCustom: "u", +} + +// pureContentTypeSuffixes maps volume's content type to storage volume name suffix. +var pureContentTypeSuffixes = map[ContentType]string{ + // Suffix used for block content type volumes. + ContentTypeBlock: "b", + + // Suffix used for ISO content type volumes. + ContentTypeISO: "i", +} + +// pureSnapshotPrefix is a prefix used for Pure Storage snapshots to avoid name conflicts +// when creating temporary volume from the snapshot. +var pureSnapshotPrefix = "s" + +// pureError represents an error responses from Pure Storage API. +type pureError struct { + // List of errors returned by the Pure Storage API. + Errors []struct { + Context string `json:"context"` + Message string `json:"message"` + } `json:"errors"` + + // StatusCode is not part of the response body but is used + // to store the HTTP status code. + StatusCode int `json:"-"` +} + +// Error returns the first error message from the Pure Storage API error. +func (p *pureError) Error() string { + if p == nil || len(p.Errors) == 0 { + return "" + } + + // Return the first error message without the trailing dot. + return strings.TrimSuffix(p.Errors[0].Message, ".") +} + +// isPureErrorOf checks if the given error is of type pureError, has the specified status code, +// and its error messages contain any of the provided substrings. Note that the error message +// comparison is case-insensitive. +func isPureErrorOf(err error, statusCode int, substrings ...string) bool { + perr, ok := err.(*pureError) + if !ok { + return false + } + + if perr.StatusCode != statusCode { + return false + } + + if len(substrings) == 0 { + // Error matches the given status code and no substrings are provided. + return true + } + + // Check if any error message contains a provided substring. + // Perform case-insensitive matching by converting both the + // error message and the substring to lowercase. + for _, err := range perr.Errors { + errMsg := strings.ToLower(err.Message) + + for _, substring := range substrings { + if strings.Contains(errMsg, strings.ToLower(substring)) { + return true + } + } + } + + return false +} + +// pureIsNotFoundError returns true if the error is of type pureError, its status code is 400 (bad request), +// and the error message contains a substring indicating the resource was not found. +func isPureErrorNotFound(err error) bool { + return isPureErrorOf(err, http.StatusBadRequest, "Not found", "Does not exist", "No such volume or snapshot") +} + +// pureResponse wraps the response from the Pure Storage API. In most cases, the response +// contains a list of items, even if only one item is returned. +type pureResponse[T any] struct { + Items []T `json:"items"` +} + +// pureEntity represents a generic entity in Pure Storage. +type pureEntity struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// pureSpace represents the usage data of Pure Storage resource. +type pureSpace struct { + // Total reserved space. + // For volumes, this is the available space or quota. + // For storage pools, this is the total reserved space (not the quota). + TotalBytes int64 `json:"total_provisioned"` + + // Amount of logically written data that a volume or a snapshot references. + // This value is compared against the quota, therefore, it should be used for + // showing the actual used space. Although, the actual used space is most likely + // less than this value due to the data reduction that is done by Pure Storage. + UsedBytes int64 `json:"virtual"` +} + +// pureStorageArray represents a storage array in Pure Storage. +type pureStorageArray struct { + ID string `json:"id"` + Name string `json:"name"` + Capacity int64 `json:"capacity"` + Space pureSpace `json:"space"` +} + +// pureStoragePool represents a storage pool (pod) in Pure Storage. +type pureStoragePool struct { + ID string `json:"id"` + Name string `json:"name"` + IsDestroyed bool `json:"destroyed"` + Quota int64 `json:"quota_limit"` + Space pureSpace `json:"space"` + Arrays []pureEntity `json:"arrays"` +} + +// pureVolume represents a volume in Pure Storage. +type pureVolume struct { + ID string `json:"id"` + Name string `json:"name"` + Serial string `json:"serial"` + IsDestroyed bool `json:"destroyed"` + Space pureSpace `json:"space"` +} + +// pureHost represents a host in Pure Storage. +type pureHost struct { + Name string `json:"name"` + IQNs []string `json:"iqns"` + NQNs []string `json:"nqns"` + ConnectionCount int `json:"connection_count"` +} + +// pureTarget represents a target in Pure Storage. +type pureTarget struct { + IQN *string `json:"iqn"` + NQN *string `json:"nqn"` + Portal *string `json:"portal"` +} + +// pureClient holds the Pure Storage HTTP client and an access token. +type pureClient struct { + driver *pure + accessToken string +} + +// newPureClient creates a new instance of the HTTP Pure Storage client. +func newPureClient(driver *pure) *pureClient { + return &pureClient{ + driver: driver, + } +} + +// createBodyReader creates a reader for the given request body contents. +func (p *pureClient) createBodyReader(contents map[string]any) (io.Reader, error) { + body := &bytes.Buffer{} + + err := json.NewEncoder(body).Encode(contents) + if err != nil { + return nil, fmt.Errorf("Failed to write request body: %w", err) + } + + return body, nil +} + +// request issues a HTTP request against the Pure Storage gateway. +func (p *pureClient) request(method string, urlPath string, reqBody io.Reader, reqHeaders map[string]string, respBody any, respHeaders map[string]string) error { + // Extract scheme and host from the gateway URL. + urlParts := strings.Split(p.driver.config["pure.gateway"], "://") + if len(urlParts) != 2 { + return fmt.Errorf("Invalid Pure Storage gateway URL: %q", p.driver.config["pure.gateway"]) + } + + // Construct the request URL. + url := api.NewURL().Scheme(urlParts[0]).Host(urlParts[1]).URL + + // Prefixes the given path with the API version in the format "/api//". + // If the path is "/api/api_version", the API version is not included as this path + // is used to retrieve supported API versions. + if urlPath == "/api/api_version" { + url.Path = urlPath + } else { + // If API version is not known yet, retrieve and cache it first. + if p.driver.apiVersion == "" { + apiVersions, err := p.getAPIVersions() + if err != nil { + return fmt.Errorf("Failed to retrieve supported Pure Storage API versions: %w", err) + } + + // Use the latest available API version. + p.driver.apiVersion = apiVersions[len(apiVersions)-1] + } + + url.Path = path.Join("api", p.driver.apiVersion, urlPath) + } + + req, err := http.NewRequest(method, url.String(), reqBody) + if err != nil { + return fmt.Errorf("Failed to create request: %w", err) + } + + // Set custom request headers. + for k, v := range reqHeaders { + req.Header.Add(k, v) + } + + req.Header.Add("Accept", "application/json") + if reqBody != nil { + req.Header.Add("Content-Type", "application/json") + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: shared.IsFalse(p.driver.config["pure.gateway.verify"]), + }, + }, + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("Failed to send request: %w", err) + } + + defer resp.Body.Close() + + // The unauthorized error is reported when an invalid (or expired) access token is provided. + // Wrap unauthorized requests into an API status error to allow easier checking for expired + // token in the requestAuthenticated function. + if resp.StatusCode == http.StatusUnauthorized { + return api.StatusErrorf(http.StatusUnauthorized, "Unauthorized request") + } + + // Overwrite the response data type if an error is detected. + if resp.StatusCode != http.StatusOK { + respBody = &pureError{} + } + + // Extract the response body if requested. + if respBody != nil { + err = json.NewDecoder(resp.Body).Decode(respBody) + if err != nil { + return fmt.Errorf("Failed to read response body from %q: %w", urlPath, err) + } + } + + // Extract the response headers if requested. + if respHeaders != nil { + for k, v := range resp.Header { + respHeaders[k] = strings.Join(v, ",") + } + } + + // Return the formatted error from the body + pureErr, ok := respBody.(*pureError) + if ok { + pureErr.StatusCode = resp.StatusCode + return pureErr + } + + return nil +} + +// requestAuthenticated issues an authenticated HTTP request against the Pure Storage gateway. +// In case the access token is expired, the function will try to obtain a new one. +func (p *pureClient) requestAuthenticated(method string, path string, reqBody io.Reader, respBody any) error { + // If request fails with an unauthorized error, the request will be retried after + // requesting a new access token. + retries := 1 + + for { + // Ensure we are logged into the Pure Storage. + err := p.login() + if err != nil { + return err + } + + // Set access token as request header. + reqHeaders := map[string]string{ + "X-Auth-Token": p.accessToken, + } + + // Initiate request. + err = p.request(method, path, reqBody, reqHeaders, respBody, nil) + if err != nil { + if api.StatusErrorCheck(err, http.StatusUnauthorized) && retries > 0 { + // Access token seems to be expired. + // Reset the token and try one more time. + p.accessToken = "" + retries-- + continue + } + + // Either the error is not of type unauthorized or the maximum number of + // retries has been exceeded. + return err + } + + return nil + } +} + +// getAPIVersion returns the list of API versions that are supported by the Pure Storage. +func (p *pureClient) getAPIVersions() ([]string, error) { + var resp struct { + APIVersions []string `json:"version"` + } + + err := p.request(http.MethodGet, "/api/api_version", nil, nil, &resp, nil) + if err != nil { + return nil, fmt.Errorf("Failed to retrieve available API versions from Pure Storage: %w", err) + } + + if len(resp.APIVersions) == 0 { + return nil, fmt.Errorf("Pure Storage does not support any API versions") + } + + return resp.APIVersions, nil +} + +// login initiates an authentication request against the Pure Storage using the API token. If successful, +// an access token is retrieved and stored within a client. The access token is then used for further +// authentication. +func (p *pureClient) login() error { + if p.accessToken != "" { + // Token has been already obtained. + return nil + } + + reqHeaders := map[string]string{ + "api-token": p.driver.config["pure.api.token"], + } + + respHeaders := make(map[string]string) + + err := p.request(http.MethodPost, "/login", nil, reqHeaders, nil, respHeaders) + if err != nil { + return fmt.Errorf("Failed to login: %w", err) + } + + p.accessToken = respHeaders["X-Auth-Token"] + if p.accessToken == "" { + return errors.New("Failed to obtain access token") + } + + return nil +} + +// getStorageArray returns the list of storage arrays. +// If arrayNames are provided, only those are returned. +func (p *pureClient) getStorageArrays(arrayNames ...string) ([]pureStorageArray, error) { + var resp pureResponse[pureStorageArray] + err := p.requestAuthenticated(http.MethodGet, fmt.Sprintf("/arrays?names=%s", strings.Join(arrayNames, ",")), nil, &resp) + if err != nil { + return nil, fmt.Errorf("Failed to get storage arrays: %w", err) + } + + return resp.Items, nil +} + +// getStoragePool returns the storage pool with the given name. +func (p *pureClient) getStoragePool(poolName string) (*pureStoragePool, error) { + var resp pureResponse[pureStoragePool] + err := p.requestAuthenticated(http.MethodGet, fmt.Sprintf("/pods?names=%s", poolName), nil, &resp) + if err != nil { + if isPureErrorNotFound(err) { + return nil, api.StatusErrorf(http.StatusNotFound, "Storage pool %q not found", poolName) + } + + return nil, fmt.Errorf("Failed to get storage pool %q: %w", poolName, err) + } + + if len(resp.Items) == 0 { + return nil, api.StatusErrorf(http.StatusNotFound, "Storage pool %q not found", poolName) + } + + return &resp.Items[0], nil +} + +// createStoragePool creates a storage pool (Pure Storage pod). +func (p *pureClient) createStoragePool(poolName string, size int64) error { + reqBody := make(map[string]any) + if size > 0 { + reqBody["quota_limit"] = size + } + + pool, err := p.getStoragePool(poolName) + if err == nil && pool.IsDestroyed { + // Storage pool exists in destroyed state, therefore, restore it. + reqBody["destroyed"] = false + + req, err := p.createBodyReader(reqBody) + if err != nil { + return err + } + + err = p.requestAuthenticated(http.MethodPatch, fmt.Sprintf("/pods?names=%s", poolName), req, nil) + if err != nil { + return fmt.Errorf("Failed to restore storage pool %q: %w", poolName, err) + } + + logger.Info("Storage pool has been restored", logger.Ctx{"pool": poolName}) + return nil + } + + req, err := p.createBodyReader(reqBody) + if err != nil { + return err + } + + // Storage pool does not exist in destroyed state, therefore, try to create a new one. + err = p.requestAuthenticated(http.MethodPost, fmt.Sprintf("/pods?names=%s", poolName), req, nil) + if err != nil { + return fmt.Errorf("Failed to create storage pool %q: %w", poolName, err) + } + + return nil +} + +// updateStoragePool updates an existing storage pool (Pure Storage pod). +func (p *pureClient) updateStoragePool(poolName string, size int64) error { + reqBody := make(map[string]any) + if size > 0 { + reqBody["quota_limit"] = size + } + + req, err := p.createBodyReader(reqBody) + if err != nil { + return err + } + + err = p.requestAuthenticated(http.MethodPatch, fmt.Sprintf("/pods?names=%s", poolName), req, nil) + if err != nil { + return fmt.Errorf("Failed to update storage pool %q: %w", poolName, err) + } + + return nil +} + +// deleteStoragePool deletes a storage pool (Pure Storage pod). +func (p *pureClient) deleteStoragePool(poolName string) error { + pool, err := p.getStoragePool(poolName) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + // Storage pool has been already removed. + return nil + } + + return err + } + + // To delete the storage pool, we need to destroy it first by setting the destroyed property to true. + // In addition, we want to destroy all of its contents to allow the pool to be deleted. + // If the pool is already destroyed, we can skip this step. + if !pool.IsDestroyed { + req, err := p.createBodyReader(map[string]any{ + "destroyed": true, + }) + if err != nil { + return err + } + + err = p.requestAuthenticated(http.MethodPatch, fmt.Sprintf("/pods?names=%s&destroy_contents=true", poolName), req, nil) + if err != nil { + if isPureErrorNotFound(err) { + return nil + } + + return fmt.Errorf("Failed to destroy storage pool %q: %w", poolName, err) + } + } + + // Eradicate the storage pool by permanently deleting it along all of its contents. + err = p.requestAuthenticated(http.MethodDelete, fmt.Sprintf("/pods?names=%s&eradicate_contents=true", poolName), nil, nil) + if err != nil { + if isPureErrorNotFound(err) { + return nil + } + + if isPureErrorOf(err, http.StatusBadRequest, "Cannot eradicate pod") { + // Eradication failed, therefore the pool remains in the destroyed state. + // However, we still consider it as deleted because Pure Storage SafeMode + // may be enabled, which prevents immediate eradication of the pool. + logger.Warn("Storage pool is left in destroyed state", logger.Ctx{"pool": poolName, "err": err}) + return nil + } + + return fmt.Errorf("Failed to delete storage pool %q: %w", poolName, err) + } + + return nil +} + +// getVolume returns the volume behind volumeID. +func (p *pureClient) getVolume(poolName string, volName string) (*pureVolume, error) { + var resp pureResponse[pureVolume] + + err := p.requestAuthenticated(http.MethodGet, fmt.Sprintf("/volumes?names=%s::%s", poolName, volName), nil, &resp) + if err != nil { + if isPureErrorNotFound(err) { + return nil, api.StatusErrorf(http.StatusNotFound, "Volume %q not found", volName) + } + + return nil, fmt.Errorf("Failed to get volume %q: %w", volName, err) + } + + if len(resp.Items) == 0 { + return nil, api.StatusErrorf(http.StatusNotFound, "Volume %q not found", volName) + } + + return &resp.Items[0], nil +} + +// createVolume creates a new volume in the given storage pool. The volume is created with +// supplied size in bytes. Upon successful creation, volume's ID is returned. +func (p *pureClient) createVolume(poolName string, volName string, sizeBytes int64) error { + req, err := p.createBodyReader(map[string]any{ + "provisioned": sizeBytes, + }) + if err != nil { + return err + } + + // Prevent default protection groups to be applied on the new volume, which can + // prevent us from eradicating the volume once deleted. + err = p.requestAuthenticated(http.MethodPost, fmt.Sprintf("/volumes?names=%s::%s&with_default_protection=false", poolName, volName), req, nil) + if err != nil { + return fmt.Errorf("Failed to create volume %q in storage pool %q: %w", volName, poolName, err) + } + + return nil +} + +// deleteVolume deletes an exisiting volume in the given storage pool. +func (p *pureClient) deleteVolume(poolName string, volName string) error { + req, err := p.createBodyReader(map[string]any{ + "destroyed": true, + }) + if err != nil { + return err + } + + // To destroy the volume, we need to patch it by setting the destroyed to true. + err = p.requestAuthenticated(http.MethodPatch, fmt.Sprintf("/volumes?names=%s::%s", poolName, volName), req, nil) + if err != nil { + return fmt.Errorf("Failed to destroy volume %q in storage pool %q: %w", volName, poolName, err) + } + + // Afterwards, we can eradicate the volume. If this operation fails, the volume will remain + // in the destroyed state. + err = p.requestAuthenticated(http.MethodDelete, fmt.Sprintf("/volumes?names=%s::%s", poolName, volName), nil, nil) + if err != nil { + return fmt.Errorf("Failed to delete volume %q in storage pool %q: %w", volName, poolName, err) + } + + return nil +} + +// resizeVolume resizes an existing volume. This function does not resize any filesystem inside the volume. +func (p *pureClient) resizeVolume(poolName string, volName string, sizeBytes int64, truncate bool) error { + req, err := p.createBodyReader(map[string]any{ + "provisioned": sizeBytes, + }) + if err != nil { + return err + } + + err = p.requestAuthenticated(http.MethodPatch, fmt.Sprintf("/volumes?names=%s::%s&truncate=%v", poolName, volName, truncate), req, nil) + if err != nil { + return fmt.Errorf("Failed to resize volume %q in storage pool %q: %w", volName, poolName, err) + } + + return nil +} + +// copyVolume copies a source volume into destination volume. If overwrite is set to true, +// the destination volume will be overwritten if it already exists. +func (p *pureClient) copyVolume(srcPoolName string, srcVolName string, dstPoolName string, dstVolName string, overwrite bool) error { + req, err := p.createBodyReader(map[string]any{ + "source": map[string]string{ + "name": fmt.Sprintf("%s::%s", srcPoolName, srcVolName), + }, + }) + if err != nil { + return err + } + + url := fmt.Sprintf("/volumes?names=%s::%s&overwrite=%v", dstPoolName, dstVolName, overwrite) + + if !overwrite { + // Disable default protection groups when creating a new volume to avoid potential issues + // when deleting the volume because protection group may prevent volume eridication. + url = fmt.Sprintf("%s&with_default_protection=false", url) + } + + err = p.requestAuthenticated(http.MethodPost, url, req, nil) + if err != nil { + return fmt.Errorf(`Failed to copy volume "%s/%s" to "%s/%s": %w`, srcPoolName, srcVolName, dstPoolName, dstVolName, err) + } + + return nil +} + +// getVolumeSnapshots retrieves all existing snapshot for the given storage volume. +func (p *pureClient) getVolumeSnapshots(poolName string, volName string) ([]pureVolume, error) { + var resp pureResponse[pureVolume] + + err := p.requestAuthenticated(http.MethodGet, fmt.Sprintf("/volume-snapshots?source_names=%s::%s", poolName, volName), nil, &resp) + if err != nil { + if isPureErrorNotFound(err) { + return nil, api.StatusErrorf(http.StatusNotFound, "Volume %q not found", volName) + } + + return nil, fmt.Errorf("Failed to retrieve snapshots for volume %q in storage pool %q: %w", volName, poolName, err) + } + + return resp.Items, nil +} + +// getVolumeSnapshot retrieves an existing snapshot for the given storage volume. +func (p *pureClient) getVolumeSnapshot(poolName string, volName string, snapshotName string) (*pureVolume, error) { + var resp pureResponse[pureVolume] + + err := p.requestAuthenticated(http.MethodGet, fmt.Sprintf("/volume-snapshots?names=%s::%s.%s", poolName, volName, snapshotName), nil, &resp) + if err != nil { + if isPureErrorNotFound(err) { + return nil, api.StatusErrorf(http.StatusNotFound, "Snapshot %q not found", snapshotName) + } + + return nil, fmt.Errorf("Failed to retrieve snapshot %q for volume %q in storage pool %q: %w", snapshotName, volName, poolName, err) + } + + if len(resp.Items) == 0 { + return nil, api.StatusErrorf(http.StatusNotFound, "Snapshot %q not found", snapshotName) + } + + return &resp.Items[0], nil +} + +// createVolumeSnapshot creates a new snapshot for the given storage volume. +func (p *pureClient) createVolumeSnapshot(poolName string, volName string, snapshotName string) error { + req, err := p.createBodyReader(map[string]any{ + "suffix": snapshotName, + }) + if err != nil { + return err + } + + err = p.requestAuthenticated(http.MethodPost, fmt.Sprintf("/volume-snapshots?source_names=%s::%s", poolName, volName), req, nil) + if err != nil { + return fmt.Errorf("Failed to create snapshot %q for volume %q in storage pool %q: %w", snapshotName, volName, poolName, err) + } + + return nil +} + +// deleteVolumeSnapshot deletes an existing snapshot for the given storage volume. +func (p *pureClient) deleteVolumeSnapshot(poolName string, volName string, snapshotName string) error { + snapshot, err := p.getVolumeSnapshot(poolName, volName, snapshotName) + if err != nil { + return err + } + + if !snapshot.IsDestroyed { + // First destroy the snapshot. + req, err := p.createBodyReader(map[string]any{ + "destroyed": true, + }) + if err != nil { + return err + } + + // Destroy snapshot. + err = p.requestAuthenticated(http.MethodPatch, fmt.Sprintf("/volume-snapshots?names=%s::%s.%s", poolName, volName, snapshotName), req, nil) + if err != nil { + return fmt.Errorf("Failed to destroy snapshot %q for volume %q in storage pool %q: %w", snapshotName, volName, poolName, err) + } + } + + // Delete (eradicate) snapshot. + err = p.requestAuthenticated(http.MethodDelete, fmt.Sprintf("/volume-snapshots?names=%s::%s.%s", poolName, volName, snapshotName), nil, nil) + if err != nil { + return fmt.Errorf("Failed to delete snapshot %q for volume %q in storage pool %q: %w", snapshotName, volName, poolName, err) + } + + return nil +} + +// restoreVolumeSnapshot restores the volume by copying the volume snapshot into its parent volume. +func (p *pureClient) restoreVolumeSnapshot(poolName string, volName string, snapshotName string) error { + return p.copyVolume(poolName, fmt.Sprintf("%s.%s", volName, snapshotName), poolName, volName, true) +} + +// copyVolumeSnapshot copies the volume snapshot into destination volume. Destination volume is overwritten +// if already exists. +func (p *pureClient) copyVolumeSnapshot(srcPoolName string, srcVolName string, srcSnapshotName string, dstPoolName string, dstVolName string) error { + return p.copyVolume(srcPoolName, fmt.Sprintf("%s.%s", srcVolName, srcSnapshotName), dstPoolName, dstVolName, true) +} + +// getHosts retrieves an existing Pure Storage host. +func (p *pureClient) getHosts() ([]pureHost, error) { + var resp pureResponse[pureHost] + + err := p.requestAuthenticated(http.MethodGet, "/hosts", nil, &resp) + if err != nil { + return nil, fmt.Errorf("Failed to get hosts: %w", err) + } + + return resp.Items, nil +} + +// getCurrentHost retrieves the Pure Storage host linked to the current LXD host. +// The Pure Storage host is considered a match if it includes the fully qualified +// name of the LXD host that is determined by the configured mode. +func (p *pureClient) getCurrentHost() (*pureHost, error) { + qn, err := p.driver.connector().QualifiedName() + if err != nil { + return nil, err + } + + hosts, err := p.getHosts() + if err != nil { + return nil, err + } + + mode := p.driver.config["pure.mode"] + + for _, host := range hosts { + if mode == connectors.TypeISCSI && slices.Contains(host.IQNs, qn) { + return &host, nil + } + + if mode == connectors.TypeNVME && slices.Contains(host.NQNs, qn) { + return &host, nil + } + } + + return nil, api.StatusErrorf(http.StatusNotFound, "Host with qualified name %q not found", qn) +} + +// createHost creates a new host with provided initiator qualified names that can be associated +// with specific volumes. +func (p *pureClient) createHost(hostName string, qns []string) error { + body := make(map[string]any, 1) + mode := p.driver.config["pure.mode"] + + switch mode { + case connectors.TypeISCSI: + body["iqns"] = qns + case connectors.TypeNVME: + body["nqns"] = qns + default: + return fmt.Errorf("Unsupported Pure Storage mode %q", mode) + } + + req, err := p.createBodyReader(body) + if err != nil { + return err + } + + err = p.requestAuthenticated(http.MethodPost, fmt.Sprintf("/hosts?names=%s", hostName), req, nil) + if err != nil { + if isPureErrorOf(err, http.StatusBadRequest, "Host already exists.") { + return api.StatusErrorf(http.StatusConflict, "Host %q already exists", hostName) + } + + return fmt.Errorf("Failed to create host %q: %w", hostName, err) + } + + return nil +} + +// updateHost updates an existing host. +func (p *pureClient) updateHost(hostName string, qns []string) error { + body := make(map[string]any, 1) + mode := p.driver.config["pure.mode"] + + switch mode { + case connectors.TypeISCSI: + body["iqns"] = qns + case connectors.TypeNVME: + body["nqns"] = qns + default: + return fmt.Errorf("Unsupported Pure Storage mode %q", mode) + } + + req, err := p.createBodyReader(body) + if err != nil { + return err + } + + // To destroy the volume, we need to patch it by setting the destroyed to true. + err = p.requestAuthenticated(http.MethodPatch, fmt.Sprintf("/hosts?names=%s", hostName), req, nil) + if err != nil { + return fmt.Errorf("Failed to update host %q: %w", hostName, err) + } + + return nil +} + +// deleteHost deletes an existing host. +func (p *pureClient) deleteHost(hostName string) error { + err := p.requestAuthenticated(http.MethodDelete, fmt.Sprintf("/hosts?names=%s", hostName), nil, nil) + if err != nil { + return fmt.Errorf("Failed to delete host %q: %w", hostName, err) + } + + return nil +} + +// connectHostToVolume creates a connection between a host and volume. It returns true if the connection +// was created, and false if it already existed. +func (p *pureClient) connectHostToVolume(poolName string, volName string, hostName string) (bool, error) { + err := p.requestAuthenticated(http.MethodPost, fmt.Sprintf("/connections?host_names=%s&volume_names=%s::%s", hostName, poolName, volName), nil, nil) + if err != nil { + if isPureErrorOf(err, http.StatusBadRequest, "Connection already exists.") { + // Do not error out if connection already exists. + return false, nil + } + + return false, fmt.Errorf("Failed to connect volume %q with host %q: %w", volName, hostName, err) + } + + return true, nil +} + +// disconnectHostFromVolume deletes a connection between a host and volume. +func (p *pureClient) disconnectHostFromVolume(poolName string, volName string, hostName string) error { + err := p.requestAuthenticated(http.MethodDelete, fmt.Sprintf("/connections?host_names=%s&volume_names=%s::%s", hostName, poolName, volName), nil, nil) + if err != nil { + if isPureErrorNotFound(err) { + return api.StatusErrorf(http.StatusNotFound, "Connection between host %q and volume %q not found", volName, hostName) + } + + return fmt.Errorf("Failed to disconnect volume %q from host %q: %w", volName, hostName, err) + } + + return nil +} + +// getTarget retrieves the Pure Storage address and the its qualified name for the configured mode. +func (p *pureClient) getTarget() (targetAddr string, targetQN string, err error) { + var resp pureResponse[pureTarget] + + err = p.requestAuthenticated(http.MethodGet, "/ports", nil, &resp) + if err != nil { + return "", "", fmt.Errorf("Failed to retrieve Pure Storage targets: %w", err) + } + + mode := p.driver.config["pure.mode"] + + // Find and return the target that has address (portal) and qualified name configured. + for _, target := range resp.Items { + if target.Portal == nil { + continue + } + + // Strip the port from the portal address. + portal := strings.Split(*target.Portal, ":")[0] + + if mode == connectors.TypeISCSI && target.IQN != nil { + return portal, *target.IQN, nil + } + + if mode == connectors.TypeNVME && target.NQN != nil { + return portal, *target.NQN, nil + } + } + + return "", "", api.StatusErrorf(http.StatusNotFound, "No Pure Storage target found") +} + +// ensureHost returns a name of the host that is configured with a given IQN. If such host +// does not exist, a new one is created, where host's name equals to the server name with a +// mode included as a suffix because Pure Storage does not allow mixing IQNs, NQNs, and WWNs +// on a single host. +func (d *pure) ensureHost() (hostName string, cleanup revert.Hook, err error) { + var hostname string + + qn, err := d.connector().QualifiedName() + if err != nil { + return "", nil, err + } + + revert := revert.New() + defer revert.Fail() + + // Fetch an existing Pure Storage host. + host, err := d.client().getCurrentHost() + if err != nil { + if !api.StatusErrorCheck(err, http.StatusNotFound) { + return "", nil, err + } + + // The Pure Storage host with a qualified name of the current LXD host does not exist. + // Therefore, create a new one and name it after the server name. + serverName, err := ResolveServerName(d.state.ServerName) + if err != nil { + return "", nil, err + } + + // Append the mode to the server name because Pure Storage does not allow mixing + // NQNs, IQNs, and WWNs for a single host. + hostname = serverName + "-" + d.config["pure.mode"] + + err = d.client().createHost(hostname, []string{qn}) + if err != nil { + if !api.StatusErrorCheck(err, http.StatusConflict) { + return "", nil, err + } + + // The host with the given name already exists, update it instead. + err = d.client().updateHost(hostname, []string{qn}) + if err != nil { + return "", nil, err + } + } else { + revert.Add(func() { _ = d.client().deleteHost(hostname) }) + } + } else { + // Hostname already exists with the given IQN. + hostname = host.Name + } + + cleanup = revert.Clone().Fail + revert.Success() + return hostname, cleanup, nil +} + +// mapVolume maps the given volume onto this host. +func (d *pure) mapVolume(vol Volume) error { + revert := revert.New() + defer revert.Fail() + + volName, err := d.getVolumeName(vol) + if err != nil { + return err + } + + // Use lock to prevent a concurrent operation from the same of different + // storage pool from disconnecting the volume or even removing the Pure + // Storage host. This can happen if the last volume is unmapped just + // before this volume is mapped. + unlock, err := locking.Lock(d.state.ShutdownCtx, fmt.Sprintf("storage_pure_%s", d.config["pure.mode"])) + if err != nil { + return err + } + + defer unlock() + + // Ensure the host exists and is configured with the correct QN. + hostname, cleanup, err := d.ensureHost() + if err != nil { + return err + } + + revert.Add(cleanup) + + // Ensure the volume is connected to the host. + connCreated, err := d.client().connectHostToVolume(vol.pool, volName, hostname) + if err != nil { + return err + } + + if connCreated { + revert.Add(func() { _ = d.client().disconnectHostFromVolume(vol.pool, volName, hostname) }) + } + + // Find the array's qualified name for the configured mode. + targetAddr, targetQN, err := d.client().getTarget() + if err != nil { + return err + } + + // Connect to the array. Note that the connection can only be established + // when at least one volume is mapped with the corresponding Pure Storage + // host. + err = d.connector().Connect(d.state.ShutdownCtx, targetAddr, targetQN) + if err != nil { + return err + } + + revert.Success() + return nil +} + +// unmapVolume unmaps the given volume from this host. +func (d *pure) unmapVolume(vol Volume) error { + volName, err := d.getVolumeName(vol) + if err != nil { + return err + } + + unlock, err := locking.Lock(d.state.ShutdownCtx, fmt.Sprintf("storage_pure_%s", d.config["pure.mode"])) + if err != nil { + return err + } + + defer unlock() + + host, err := d.client().getCurrentHost() + if err != nil { + return err + } + + // Disconnect the volume from the host and ignore error if connection does not exist. + err = d.client().disconnectHostFromVolume(vol.pool, volName, host.Name) + if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { + return err + } + + volumePath, _, _ := d.getMappedDevPath(vol, false) + if volumePath != "" { + if d.config["pure.mode"] == connectors.TypeISCSI { + // When volume is disconnected from the host, the device will remain on the system. + // + // To remove the device, we need to either logout from the session or remove the + // device manually. Logging out of the session is not desired as it would disconnect + // from all connected volumes. Therefore, we need to manually remove the device. + split := strings.Split(filepath.Base(volumePath), "/") + devName := split[len(split)-1] + + path := fmt.Sprintf("/sys/block/%s/device/delete", devName) + if shared.PathExists(path) { + err := os.WriteFile(path, []byte("1"), 0400) + if err != nil { + return fmt.Errorf("Failed to unmap volume %q: Failed to remove device %q: %w", vol.name, devName, err) + } + } + } + + // Wait until the volume has disappeared. + ctx, cancel := context.WithTimeout(d.state.ShutdownCtx, 10*time.Second) + defer cancel() + + if !waitGone(ctx, volumePath) { + return fmt.Errorf("Timeout exceeded waiting for Pure Storage volume %q to disappear on path %q", vol.name, volumePath) + } + } + + // If this was the last volume being unmapped from this system, terminate + // an active session and remove the host from Pure Storage. + if host.ConnectionCount == 1 { + _, targetQN, err := d.client().getTarget() + if err != nil { + return err + } + + // Disconnect from the target. + err = d.connector().Disconnect(targetQN) + if err != nil { + return err + } + + // Remove the host from Pure Storage. + err = d.client().deleteHost(host.Name) + if err != nil { + return err + } + } + + return nil +} + +// getMappedDevPath returns the local device path for the given volume. +// Indicate with mapVolume if the volume should get mapped to the system if it isn't present. +func (d *pure) getMappedDevPath(vol Volume, mapVolume bool) (string, revert.Hook, error) { + revert := revert.New() + defer revert.Fail() + + if mapVolume { + err := d.mapVolume(vol) + if err != nil { + return "", nil, err + } + + revert.Add(func() { _ = d.unmapVolume(vol) }) + } + + volName, err := d.getVolumeName(vol) + if err != nil { + return "", nil, err + } + + var diskPrefix string + var diskSuffix string + var devicePath string + + switch d.config["pure.mode"] { + case connectors.TypeISCSI: + diskPrefix = "scsi-" + diskSuffix = fmt.Sprintf("%s::%s", vol.pool, volName) + case connectors.TypeNVME: + diskPrefix = "nvme-eui." + + pureVol, err := d.client().getVolume(vol.pool, volName) + if err != nil { + return "", nil, err + } + + // The serial number is used to identify the device. The last 10 characters + // of the serial number appear as a disk device suffix. This check ensures + // we do not panic if the reported serial number is too short for parsing. + if len(pureVol.Serial) <= 10 { + // Serial number is too short. + return "", nil, fmt.Errorf("Failed to locate device for volume %q: Invalid serial number %q", vol.name, pureVol.Serial) + } + + // Extract the last 10 characters of the serial number. Also convert + // it to lower case, as on host the device ID is completely lower case. + diskSuffix = strings.ToLower(pureVol.Serial[len(pureVol.Serial)-10:]) + default: + return "", nil, fmt.Errorf("Unsupported Pure Storage mode %q", d.config["pure.mode"]) + } + + // Get the device path. + if mapVolume { + // Wait for the device path to appear as the volume has been just mapped to the host. + devicePath, err = connectors.WaitDiskDevicePath(d.state.ShutdownCtx, diskPrefix, diskSuffix) + } else { + // Get the the device path without waiting. + devicePath, err = connectors.GetDiskDevicePath(diskPrefix, diskSuffix) + } + + if err != nil { + return "", nil, fmt.Errorf("Failed to locate device for volume %q: %w", vol.name, err) + } + + cleanup := revert.Clone().Fail + revert.Success() + return devicePath, cleanup, nil +} + +// getVolumeName returns the fully qualified name derived from the volume's UUID. +func (d *pure) getVolumeName(vol Volume) (string, error) { + volUUID, err := uuid.Parse(vol.config["volatile.uuid"]) + if err != nil { + return "", fmt.Errorf(`Failed parsing "volatile.uuid" from volume %q: %w`, vol.name, err) + } + + // Remove hypens from the UUID to create a volume name. + volName := strings.ReplaceAll(volUUID.String(), "-", "") + + // Search for the volume type prefix, and if found, prepend it to the volume name. + volumeTypePrefix, ok := pureVolTypePrefixes[vol.volType] + if ok { + volName = fmt.Sprintf("%s-%s", volumeTypePrefix, volName) + } + + // Search for the content type suffix, and if found, append it to the volume name. + contentTypeSuffix, ok := pureContentTypeSuffixes[vol.contentType] + if ok { + volName = fmt.Sprintf("%s-%s", volName, contentTypeSuffix) + } + + // If volume is snapshot, prepend snapshot prefix to its name. + if vol.IsSnapshot() { + volName = fmt.Sprintf("%s%s", pureSnapshotPrefix, volName) + } + + return volName, nil +} diff --git a/lxd/storage/drivers/driver_pure_util_test.go b/lxd/storage/drivers/driver_pure_util_test.go new file mode 100644 index 000000000000..3fabf570a356 --- /dev/null +++ b/lxd/storage/drivers/driver_pure_util_test.go @@ -0,0 +1,117 @@ +package drivers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_pure_serverName(t *testing.T) { + // newTestVol creates a new Volume with the given UUID, VolumeType and ContentType. + newTestVol := func(volName string, volType VolumeType, contentType ContentType, uuid string) Volume { + config := map[string]string{ + "volatile.uuid": uuid, + } + + return NewVolume(nil, "testpool", volType, contentType, volName, config, nil) + } + + tests := []struct { + Name string + Volume Volume + WantVolName string + WantError string + }{ + { + Name: "Incorrect UUID length", + Volume: newTestVol("vol-err-1", VolumeTypeContainer, ContentTypeFS, "uuid"), + WantError: "invalid UUID length: 4", + }, + { + Name: "Invalid UUID format", + Volume: newTestVol("vol-err-2", VolumeTypeContainer, ContentTypeFS, "abcdefgh-1234-abcd-1234-abcdefgh"), + WantError: "invalid UUID format", + }, + { + Name: "Container FS", + Volume: newTestVol("c-fs", VolumeTypeContainer, ContentTypeFS, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "c-a5289556c903409a8aa04af18a46738d", + }, + { + Name: "VM FS", + Volume: newTestVol("vm-fs", VolumeTypeVM, ContentTypeFS, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "v-a5289556c903409a8aa04af18a46738d", + }, + { + Name: "VM Block", + Volume: newTestVol("vm-block", VolumeTypeVM, ContentTypeBlock, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "v-a5289556c903409a8aa04af18a46738d-b", + }, + { + Name: "Image FS", + Volume: newTestVol("img-fs", VolumeTypeImage, ContentTypeFS, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "i-a5289556c903409a8aa04af18a46738d", + }, + { + Name: "Image Block", + Volume: newTestVol("img-block", VolumeTypeImage, ContentTypeBlock, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "i-a5289556c903409a8aa04af18a46738d-b", + }, + { + Name: "Custom FS", + Volume: newTestVol("custom-fs", VolumeTypeCustom, ContentTypeFS, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "u-a5289556c903409a8aa04af18a46738d", + }, + { + Name: "Custom Block", + Volume: newTestVol("custom-block", VolumeTypeCustom, ContentTypeBlock, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "u-a5289556c903409a8aa04af18a46738d-b", + }, + { + Name: "Custom ISO", + Volume: newTestVol("custom-iso", VolumeTypeCustom, ContentTypeISO, "a5289556-c903-409a-8aa0-4af18a46738d"), + WantVolName: "u-a5289556c903409a8aa04af18a46738d-i", + }, + { + Name: "Snapshot Container FS", + Volume: newTestVol("c-fs/snap0", VolumeTypeContainer, ContentTypeFS, "fd87f109-767d-4f2f-ae18-66c34276f351"), + WantVolName: "sc-fd87f109767d4f2fae1866c34276f351", + }, + { + Name: "Snapshot VM FS", + Volume: newTestVol("vm-fs/snap0", VolumeTypeVM, ContentTypeFS, "fd87f109-767d-4f2f-ae18-66c34276f351"), + WantVolName: "sv-fd87f109767d4f2fae1866c34276f351", + }, + { + Name: "Snapshot VM Block", + Volume: newTestVol("vm-block/snap0", VolumeTypeVM, ContentTypeBlock, "fd87f109-767d-4f2f-ae18-66c34276f351"), + WantVolName: "sv-fd87f109767d4f2fae1866c34276f351-b", + }, + { + Name: "Snapshot Custom Block", + Volume: newTestVol("custom-block/snap0", VolumeTypeCustom, ContentTypeBlock, "fd87f109-767d-4f2f-ae18-66c34276f351"), + WantVolName: "su-fd87f109767d4f2fae1866c34276f351-b", + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + d := &pure{} + + volName, err := d.getVolumeName(test.Volume) + if err != nil { + if test.WantError != "" { + assert.ErrorContains(t, err, test.WantError) + } else { + t.Errorf("pure.getVolumeName() unexpected error: %v", err) + } + } else { + if test.WantError != "" { + t.Errorf("pure.getVolumeName() expected error %q, but got none", err) + } else { + assert.Equal(t, test.WantVolName, volName) + } + } + }) + } +} diff --git a/lxd/storage/drivers/driver_pure_volumes.go b/lxd/storage/drivers/driver_pure_volumes.go new file mode 100644 index 000000000000..37b42f7f251b --- /dev/null +++ b/lxd/storage/drivers/driver_pure_volumes.go @@ -0,0 +1,1395 @@ +package drivers + +import ( + "fmt" + "io" + "net/http" + "os" + "slices" + "strings" + + "golang.org/x/sys/unix" + + "github.com/canonical/lxd/lxd/backup" + "github.com/canonical/lxd/lxd/instancewriter" + "github.com/canonical/lxd/lxd/migration" + "github.com/canonical/lxd/lxd/operations" + "github.com/canonical/lxd/lxd/storage/filesystem" + "github.com/canonical/lxd/shared" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/logger" + "github.com/canonical/lxd/shared/revert" + "github.com/canonical/lxd/shared/units" + "github.com/canonical/lxd/shared/validate" +) + +// commonVolumeRules returns validation rules which are common for pool and volume. +func (d *pure) commonVolumeRules() map[string]func(value string) error { + return map[string]func(value string) error{ + // lxdmeta:generate(entities=storage-pure; group=volume-conf; key=block.filesystem) + // Valid options are: `btrfs`, `ext4`, `xfs` + // If not set, `ext4` is assumed. + // --- + // type: string + // condition: block-based volume with content type `filesystem` + // defaultdesc: same as `volume.block.filesystem` + // shortdesc: File system of the storage volume + "block.filesystem": validate.Optional(validate.IsOneOf(blockBackedAllowedFilesystems...)), + // lxdmeta:generate(entities=storage-pure; group=volume-conf; key=block.mount_options) + // + // --- + // type: string + // condition: block-based volume with content type `filesystem` + // defaultdesc: same as `volume.block.mount_options` + // shortdesc: Mount options for block-backed file system volumes + "block.mount_options": validate.IsAny, + // lxdmeta:generate(entities=storage-pure; group=volume-conf; key=size) + // Default Pure Storage volume size rounded to 512B. The minimum size is 1MiB. + // --- + // type: string + // defaultdesc: same as `volume.size` + // shortdesc: Size/quota of the storage volume + "size": validate.Optional(validate.IsMultipleOfUnit("512B")), + } +} + +// CreateVolume creates an empty volume and can optionally fill it by executing the supplied filler function. +func (d *pure) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { + client := d.client() + + revert := revert.New() + defer revert.Fail() + + volName, err := d.getVolumeName(vol) + if err != nil { + return err + } + + sizeBytes, err := units.ParseByteSizeString(vol.ConfigSize()) + if err != nil { + return err + } + + // Create the volume. + err = client.createVolume(vol.pool, volName, sizeBytes) + if err != nil { + return err + } + + revert.Add(func() { _ = client.deleteVolume(vol.pool, volName) }) + + volumeFilesystem := vol.ConfigBlockFilesystem() + if vol.contentType == ContentTypeFS { + devPath, cleanup, err := d.getMappedDevPath(vol, true) + if err != nil { + return err + } + + revert.Add(cleanup) + + _, err = makeFSType(devPath, volumeFilesystem, nil) + if err != nil { + return err + } + } + + // For VMs, also create the filesystem volume. + if vol.IsVMBlock() { + fsVol := vol.NewVMBlockFilesystemVolume() + + err := d.CreateVolume(fsVol, nil, op) + if err != nil { + return err + } + + revert.Add(func() { _ = d.DeleteVolume(fsVol, op) }) + } + + err = vol.MountTask(func(mountPath string, op *operations.Operation) error { + // Run the volume filler function if supplied. + if filler != nil && filler.Fill != nil { + var err error + var devPath string + + if IsContentBlock(vol.contentType) { + // Get the device path. + devPath, err = d.GetVolumeDiskPath(vol) + if err != nil { + return err + } + } + + allowUnsafeResize := false + if vol.volType == VolumeTypeImage { + // Allow filler to resize initial image volume as needed. + // Some storage drivers don't normally allow image volumes to be resized due to + // them having read-only snapshots that cannot be resized. However when creating + // the initial image volume and filling it before the snapshot is taken resizing + // can be allowed and is required in order to support unpacking images larger than + // the default volume size. The filler function is still expected to obey any + // volume size restrictions configured on the pool. + // Unsafe resize is also needed to disable filesystem resize safety checks. + // This is safe because if for some reason an error occurs the volume will be + // discarded rather than leaving a corrupt filesystem. + allowUnsafeResize = true + } + + // Run the filler. + err = d.runFiller(vol, devPath, filler, allowUnsafeResize) + if err != nil { + return err + } + + // Move the GPT alt header to end of disk if needed. + if vol.IsVMBlock() { + err = d.moveGPTAltHeader(devPath) + if err != nil { + return err + } + } + } + + if vol.contentType == ContentTypeFS { + // Run EnsureMountPath again after mounting and filling to ensure the mount directory has + // the correct permissions set. + err = vol.EnsureMountPath() + if err != nil { + return err + } + } + + return nil + }, op) + if err != nil { + return err + } + + revert.Success() + return nil +} + +// CreateVolumeFromBackup re-creates a volume from its exported state. +func (d *pure) CreateVolumeFromBackup(vol VolumeCopy, srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) (VolumePostHook, revert.Hook, error) { + return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, op) +} + +// CreateVolumeFromCopy provides same-pool volume copying functionality. +func (d *pure) CreateVolumeFromCopy(vol VolumeCopy, srcVol VolumeCopy, allowInconsistent bool, op *operations.Operation) error { + revert := revert.New() + defer revert.Fail() + + // Function to run once the volume is created, which will ensure appropriate permissions + // on the mount path inside the volume, and resize the volume to specified size. + postCreateTasks := func(v Volume) error { + if vol.contentType == ContentTypeFS { + // Mount the volume and ensure the permissions are set correctly inside the mounted volume. + err := v.MountTask(func(_ string, _ *operations.Operation) error { + return v.EnsureMountPath() + }, op) + if err != nil { + return err + } + } + + // Resize volume to the size specified. + err := d.SetVolumeQuota(v, v.ConfigSize(), false, op) + if err != nil { + return err + } + + return nil + } + + // For VMs, also copy the filesystem volume. + if vol.IsVMBlock() { + // Ensure that the volume's snapshots are also replaced with their filesystem counterpart. + fsVolSnapshots := make([]Volume, 0, len(vol.Snapshots)) + for _, snapshot := range vol.Snapshots { + fsVolSnapshots = append(fsVolSnapshots, snapshot.NewVMBlockFilesystemVolume()) + } + + srcFsVolSnapshots := make([]Volume, 0, len(srcVol.Snapshots)) + for _, snapshot := range srcVol.Snapshots { + srcFsVolSnapshots = append(srcFsVolSnapshots, snapshot.NewVMBlockFilesystemVolume()) + } + + fsVol := NewVolumeCopy(vol.NewVMBlockFilesystemVolume(), fsVolSnapshots...) + srcFSVol := NewVolumeCopy(srcVol.NewVMBlockFilesystemVolume(), srcFsVolSnapshots...) + + // Ensure parent UUID is retained for the filesystem volumes. + fsVol.SetParentUUID(vol.parentUUID) + srcFSVol.SetParentUUID(srcVol.parentUUID) + + err := d.CreateVolumeFromCopy(fsVol, srcFSVol, false, op) + if err != nil { + return err + } + + revert.Add(func() { _ = d.DeleteVolume(fsVol.Volume, op) }) + } + + poolName := vol.pool + srcPoolName := srcVol.pool + + volName, err := d.getVolumeName(vol.Volume) + if err != nil { + return err + } + + srcVolName, err := d.getVolumeName(srcVol.Volume) + if err != nil { + return err + } + + // Since snapshots are first copied into destination volume from which a new snapshot is created, + // we need to also remove the destination volume if an error occurs during copying of snapshots. + deleteVolCopy := true + + // Copy volume snapshots. + // Pure Storage does not allow copying snapshots along with the volume. Therefore, we + // copy the snapshots sequentially. Each snapshot is first copied into destination + // volume from which a new snapshot is created. The process is repeted until all + // snapshots are copied. + if !srcVol.IsSnapshot() { + for _, snapshot := range vol.Snapshots { + _, snapshotShortName, _ := api.GetParentAndSnapshotName(snapshot.name) + + // Find the corresponding source snapshot. + var srcSnapshot *Volume + for _, srcSnap := range srcVol.Snapshots { + _, srcSnapshotShortName, _ := api.GetParentAndSnapshotName(snapshot.name) + if snapshotShortName == srcSnapshotShortName { + srcSnapshot = &srcSnap + break + } + } + + if srcSnapshot == nil { + return fmt.Errorf("Failed to copy snapshot %q: Source snapshot does not exist", snapshotShortName) + } + + srcSnapshotName, err := d.getVolumeName(*srcSnapshot) + if err != nil { + return err + } + + // Copy the snapshot. + err = d.client().copyVolumeSnapshot(srcPoolName, srcVolName, srcSnapshotName, poolName, volName) + if err != nil { + return fmt.Errorf("Failed copying snapshot %q: %w", snapshot.name, err) + } + + if deleteVolCopy { + // If at least one snapshot is copied into destination volume, we need to remove + // that volume as well in case of an error. + revert.Add(func() { _ = d.DeleteVolume(vol.Volume, op) }) + deleteVolCopy = false + } + + // Set snapshot's parent UUID and retain source snapshot UUID. + snapshot.SetParentUUID(vol.config["volatile.uuid"]) + + // Create snapshot from a new volume (that was created from the source snapshot). + // However, do not create VM's filesystem volume snapshot, as filesystem volume is + // copied before block volume. + err = d.createVolumeSnapshot(snapshot, false, op) + if err != nil { + return err + } + } + } + + // Finally, copy the source volume (or snapshot) into destination volume snapshots. + if srcVol.IsSnapshot() { + // Get snapshot parent volume name. + srcParentVol := getSnapshotParentVolume(srcVol.Volume) + srcParentVolName, err := d.getVolumeName(srcParentVol) + if err != nil { + return err + } + + // Copy the source snapshot into destination volume. + err = d.client().copyVolumeSnapshot(srcPoolName, srcParentVolName, srcVolName, poolName, volName) + if err != nil { + return err + } + } else { + err = d.client().copyVolume(srcPoolName, srcVolName, poolName, volName, true) + if err != nil { + return err + } + } + + // Add reverted to delete destination volume, if not already added. + if deleteVolCopy { + revert.Add(func() { _ = d.DeleteVolume(vol.Volume, op) }) + } + + err = postCreateTasks(vol.Volume) + if err != nil { + return err + } + + revert.Success() + return nil +} + +// CreateVolumeFromMigration creates a volume being sent via a migration. +func (d *pure) CreateVolumeFromMigration(vol VolumeCopy, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { + // When performing a cluster member move prepare the volumes on the target side. + if volTargetArgs.ClusterMoveSourceName != "" { + err := vol.EnsureMountPath() + if err != nil { + return err + } + + if vol.IsVMBlock() { + fsVol := NewVolumeCopy(vol.NewVMBlockFilesystemVolume()) + err := d.CreateVolumeFromMigration(fsVol, conn, volTargetArgs, preFiller, op) + if err != nil { + return err + } + } + + return nil + } + + _, err := genericVFSCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op) + return err +} + +// RefreshVolume updates an existing volume to match the state of another. +func (d *pure) RefreshVolume(vol VolumeCopy, srcVol VolumeCopy, refreshSnapshots []string, allowInconsistent bool, op *operations.Operation) error { + revert := revert.New() + defer revert.Fail() + + // For VMs, also copy the filesystem volume. + if vol.IsVMBlock() { + // Ensure that the volume's snapshots are also replaced with their filesystem counterpart. + fsVolSnapshots := make([]Volume, 0, len(vol.Snapshots)) + for _, snapshot := range vol.Snapshots { + fsVolSnapshots = append(fsVolSnapshots, snapshot.NewVMBlockFilesystemVolume()) + } + + srcFsVolSnapshots := make([]Volume, 0, len(srcVol.Snapshots)) + for _, snapshot := range srcVol.Snapshots { + srcFsVolSnapshots = append(srcFsVolSnapshots, snapshot.NewVMBlockFilesystemVolume()) + } + + fsVol := NewVolumeCopy(vol.NewVMBlockFilesystemVolume(), fsVolSnapshots...) + srcFSVol := NewVolumeCopy(srcVol.NewVMBlockFilesystemVolume(), srcFsVolSnapshots...) + + cleanup, err := d.refreshVolume(fsVol, srcFSVol, refreshSnapshots, allowInconsistent, op) + if err != nil { + return err + } + + revert.Add(cleanup) + } + + cleanup, err := d.refreshVolume(vol, srcVol, refreshSnapshots, allowInconsistent, op) + if err != nil { + return err + } + + revert.Add(cleanup) + + revert.Success() + return nil +} + +// refreshVolume updates an existing volume to match the state of another. For VMs, this function +// refreshes either block or filesystem volume, depending on the volume type. Therefore, the caller +// needs to ensure it is called twice - once for each volume type. +func (d *pure) refreshVolume(vol VolumeCopy, srcVol VolumeCopy, refreshSnapshots []string, allowInconsistent bool, op *operations.Operation) (revert.Hook, error) { + revert := revert.New() + defer revert.Fail() + + // Function to run once the volume is created, which will ensure appropriate permissions + // on the mount path inside the volume, and resize the volume to specified size. + postCreateTasks := func(v Volume) error { + if vol.contentType == ContentTypeFS { + // Mount the volume and ensure the permissions are set correctly inside the mounted volume. + err := v.MountTask(func(_ string, _ *operations.Operation) error { + return v.EnsureMountPath() + }, op) + if err != nil { + return err + } + } + + // Resize volume to the size specified. + err := d.SetVolumeQuota(vol.Volume, vol.ConfigSize(), false, op) + if err != nil { + return err + } + + return nil + } + + srcPoolName := srcVol.pool + poolName := vol.pool + + srcVolName, err := d.getVolumeName(srcVol.Volume) + if err != nil { + return nil, err + } + + volName, err := d.getVolumeName(vol.Volume) + if err != nil { + return nil, err + } + + // Create new reverter snapshot, which is used to revert the original volume in case of + // an error. Snapshots are also required to be first copied into destination volume, + // from which a new snapshot is created to effectively copy a snapshot. If any error + // occurs, the destination volume has been already modified and needs reverting. + reverterSnapshotName := "lxd-reverter-snapshot" + + // Remove existing reverter snapshot. + err = d.client().deleteVolumeSnapshot(vol.pool, volName, reverterSnapshotName) + if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { + return nil, err + } + + // Create new reverter snapshot. + err = d.client().createVolumeSnapshot(vol.pool, volName, reverterSnapshotName) + if err != nil { + return nil, err + } + + revert.Add(func() { + // Restore destination volume from reverter snapshot and remove the snapshot afterwards. + _ = d.client().restoreVolumeSnapshot(vol.pool, volName, reverterSnapshotName) + _ = d.client().deleteVolumeSnapshot(vol.pool, volName, reverterSnapshotName) + }) + + if !srcVol.IsSnapshot() && len(refreshSnapshots) > 0 { + var refreshedSnapshots []string + + // Refresh volume snapshots. + // Pure Storage does not allow copying snapshots along with the volume. Therefore, + // we copy the missing snapshots sequentially. Each snapshot is first copied into + // destination volume from which a new snapshot is created. The process is repeted + // until all of the missing snapshots are copied. + for _, snapshot := range vol.Snapshots { + // Remove volume name prefix from the snapshot name, and check whether it + // has to be refreshed. + _, snapshotShortName, _ := api.GetParentAndSnapshotName(snapshot.name) + if !slices.Contains(refreshSnapshots, snapshotShortName) { + // Skip snapshot if it doesn't have to be refreshed. + continue + } + + // Find the corresponding source snapshot. + var srcSnapshot *Volume + for _, srcSnap := range srcVol.Snapshots { + _, srcSnapshotShortName, _ := api.GetParentAndSnapshotName(srcSnap.name) + if snapshotShortName == srcSnapshotShortName { + srcSnapshot = &srcSnap + break + } + } + + if srcSnapshot == nil { + return nil, fmt.Errorf("Failed to refresh snapshot %q: Source snapshot does not exist", snapshotShortName) + } + + srcSnapshotName, err := d.getVolumeName(*srcSnapshot) + if err != nil { + return nil, err + } + + // Overwrite existing destination volume with snapshot. + err = d.client().copyVolumeSnapshot(srcPoolName, srcVolName, srcSnapshotName, poolName, volName) + if err != nil { + return nil, err + } + + // Set snapshot's parent UUID. + snapshot.SetParentUUID(vol.config["volatile.uuid"]) + + // Create snapshot of a new volume. Do not copy VM's filesystem volume snapshot, + // as FS volumes are already copied by this point. + err = d.createVolumeSnapshot(snapshot, false, op) + if err != nil { + return nil, err + } + + revert.Add(func() { _ = d.DeleteVolumeSnapshot(snapshot, op) }) + + // Append snapshot to the list of successfully refreshed snapshots. + refreshedSnapshots = append(refreshedSnapshots, snapshotShortName) + } + + // Ensure all snapshots were successfully refreshed. + missing := shared.RemoveElementsFromSlice(refreshSnapshots, refreshedSnapshots...) + if len(missing) > 0 { + return nil, fmt.Errorf("Failed to refresh snapshots %v", missing) + } + } + + // Finally, copy the source volume (or snapshot) into destination volume snapshots. + if srcVol.IsSnapshot() { + // Find snapshot parent volume. + srcParentVol := getSnapshotParentVolume(srcVol.Volume) + srcParentVolName, err := d.getVolumeName(srcParentVol) + if err != nil { + return nil, err + } + + // Copy the source snapshot into destination volume. + err = d.client().copyVolumeSnapshot(srcPoolName, srcParentVolName, srcVolName, poolName, volName) + if err != nil { + return nil, err + } + } else { + err = d.client().copyVolume(srcPoolName, srcVolName, poolName, volName, true) + if err != nil { + return nil, err + } + } + + err = postCreateTasks(vol.Volume) + if err != nil { + return nil, err + } + + cleanup := revert.Clone().Fail + revert.Success() + + // Remove temporary reverter snapshot. + _ = d.client().deleteVolumeSnapshot(vol.pool, volName, reverterSnapshotName) + + return cleanup, err +} + +// DeleteVolume deletes the volume and all associated snapshots. +func (d *pure) DeleteVolume(vol Volume, op *operations.Operation) error { + volExists, err := d.HasVolume(vol) + if err != nil { + return err + } + + if !volExists { + return nil + } + + volName, err := d.getVolumeName(vol) + if err != nil { + return err + } + + host, err := d.client().getCurrentHost() + if err != nil { + if !api.StatusErrorCheck(err, http.StatusNotFound) { + return err + } + } else { + // Dicsconnect the volume from the host. + err = d.client().disconnectHostFromVolume(vol.pool, volName, host.Name) + if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { + return err + } + } + + err = d.client().deleteVolume(vol.pool, volName) + if err != nil { + return err + } + + // For VMs, also delete the filesystem volume. + if vol.IsVMBlock() { + fsVol := vol.NewVMBlockFilesystemVolume() + + err := d.DeleteVolume(fsVol, op) + if err != nil { + return err + } + } + + mountPath := vol.MountPath() + + if vol.contentType == ContentTypeFS && shared.PathExists(mountPath) { + err := wipeDirectory(mountPath) + if err != nil { + return err + } + + err = os.Remove(mountPath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("Failed to remove %q: %w", mountPath, err) + } + } + + return nil +} + +// HasVolume indicates whether a specific volume exists on the storage pool. +func (d *pure) HasVolume(vol Volume) (bool, error) { + volName, err := d.getVolumeName(vol) + if err != nil { + return false, err + } + + // If volume represents a snapshot, also retrieve (encoded) volume name of the parent, + // and check if the snapshot exists. + if vol.IsSnapshot() { + parentVol := getSnapshotParentVolume(vol) + parentVolName, err := d.getVolumeName(parentVol) + if err != nil { + return false, err + } + + _, err = d.client().getVolumeSnapshot(vol.pool, parentVolName, volName) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + return false, nil + } + + return false, err + } + + return true, nil + } + + // Otherwise, check if the volume exists. + _, err = d.client().getVolume(vol.pool, volName) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + return false, nil + } + + return false, err + } + + return true, nil +} + +// FillVolumeConfig populate volume with default config. +func (d *pure) FillVolumeConfig(vol Volume) error { + // Copy volume.* configuration options from pool. + // Exclude 'block.filesystem' and 'block.mount_options' + // as these ones are handled below in this function and depend on the volume's type. + err := d.fillVolumeConfig(&vol, "block.filesystem", "block.mount_options") + if err != nil { + return err + } + + // Only validate filesystem config keys for filesystem volumes or VM block volumes (which have an + // associated filesystem volume). + if vol.ContentType() == ContentTypeFS || vol.IsVMBlock() { + // VM volumes will always use the default filesystem. + if vol.IsVMBlock() { + vol.config["block.filesystem"] = DefaultFilesystem + } else { + // Inherit filesystem from pool if not set. + if vol.config["block.filesystem"] == "" { + vol.config["block.filesystem"] = d.config["volume.block.filesystem"] + } + + // Default filesystem if neither volume nor pool specify an override. + if vol.config["block.filesystem"] == "" { + // Unchangeable volume property: Set unconditionally. + vol.config["block.filesystem"] = DefaultFilesystem + } + } + + // Inherit filesystem mount options from pool if not set. + if vol.config["block.mount_options"] == "" { + vol.config["block.mount_options"] = d.config["volume.block.mount_options"] + } + + // Default filesystem mount options if neither volume nor pool specify an override. + if vol.config["block.mount_options"] == "" { + // Unchangeable volume property: Set unconditionally. + vol.config["block.mount_options"] = "discard" + } + } + + return nil +} + +// ValidateVolume validates the supplied volume config. +func (d *pure) ValidateVolume(vol Volume, removeUnknownKeys bool) error { + // When creating volumes from ISO images, round its size to the next multiple of 512B. + if vol.ContentType() == ContentTypeISO { + sizeBytes, err := units.ParseByteSizeString(vol.ConfigSize()) + if err != nil { + return err + } + + // If the remainder when dividing by 512 is greater than 0, round the size up + // to the next multiple of 512. + remainder := sizeBytes % 512 + if remainder > 0 { + sizeBytes = (sizeBytes/512 + 1) * 512 + vol.SetConfigSize(fmt.Sprintf("%d", sizeBytes)) + } + } + + commonRules := d.commonVolumeRules() + + // Disallow block.* settings for regular custom block volumes. These settings only make sense + // when using custom filesystem volumes. LXD will create the filesystem for these volumes, + // and use the mount options. When attaching a regular block volume to a VM, these are not + // mounted by LXD and therefore don't need these config keys. + if vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { + delete(commonRules, "block.filesystem") + delete(commonRules, "block.mount_options") + } + + return d.validateVolume(vol, commonRules, removeUnknownKeys) +} + +// UpdateVolume applies config changes to the volume. +func (d *pure) UpdateVolume(vol Volume, changedConfig map[string]string) error { + newSize, sizeChanged := changedConfig["size"] + if sizeChanged { + err := d.SetVolumeQuota(vol, newSize, false, nil) + if err != nil { + return err + } + } + + return nil +} + +// GetVolumeUsage returns the disk space used by the volume. +func (d *pure) GetVolumeUsage(vol Volume) (int64, error) { + volName, err := d.getVolumeName(vol) + if err != nil { + return -1, err + } + + pureVol, err := d.client().getVolume(vol.pool, volName) + if err != nil { + return -1, err + } + + return pureVol.Space.UsedBytes, nil +} + +// SetVolumeQuota applies a size limit on volume. +// Does nothing if supplied with an non-positive size. +func (d *pure) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { + revert := revert.New() + defer revert.Fail() + + // Convert to bytes. + sizeBytes, err := units.ParseByteSizeString(size) + if err != nil { + return err + } + + // Do nothing if size isn't specified. + if sizeBytes <= 0 { + return nil + } + + volName, err := d.getVolumeName(vol) + if err != nil { + return err + } + + // Get volume and retrieve current size. + pureVol, err := d.client().getVolume(vol.pool, volName) + if err != nil { + return err + } + + oldSizeBytes := pureVol.Space.TotalBytes + + // Do nothing if volume is already specified size (+/- 512 bytes). + if oldSizeBytes+512 > sizeBytes && oldSizeBytes-512 < sizeBytes { + return nil + } + + devPath, cleanup, err := d.getMappedDevPath(vol, true) + if err != nil { + return err + } + + revert.Add(cleanup) + + inUse := vol.MountInUse() + truncate := sizeBytes < oldSizeBytes + + // Resize filesystem if needed. + if vol.contentType == ContentTypeFS { + fsType := vol.ConfigBlockFilesystem() + + if sizeBytes < oldSizeBytes { + if !filesystemTypeCanBeShrunk(fsType) { + return fmt.Errorf("Filesystem %q cannot be shrunk: %w", fsType, ErrCannotBeShrunk) + } + + if inUse { + // We don't allow online shrinking of filesytem volumes. + // Returning this error ensures the disk is resized next + // time the instance is started. + return ErrInUse + } + + // Shrink filesystem first. + err = shrinkFileSystem(fsType, devPath, vol, sizeBytes, allowUnsafeResize) + if err != nil { + return err + } + + // Shrink the block device. + err = d.client().resizeVolume(vol.pool, volName, sizeBytes, truncate) + if err != nil { + return err + } + } else { + // Grow block device first. + err = d.client().resizeVolume(vol.pool, volName, sizeBytes, truncate) + if err != nil { + return err + } + + // Grow the filesystem to fill the block device. + err = growFileSystem(fsType, devPath, vol) + if err != nil { + return err + } + } + } else { + // Only perform pre-resize checks if we are not in "unsafe" mode. + // In unsafe mode we expect the caller to know what they are doing and understand the risks. + if !allowUnsafeResize { + if sizeBytes < oldSizeBytes { + return fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) + } + + if inUse { + // We don't allow online shrinking of filesytem volumes. + // Returning this error ensures the disk is resized next + // time the instance is started. + return ErrInUse + } + } + + // Resize block device. + err = d.client().resizeVolume(vol.pool, volName, sizeBytes, truncate) + if err != nil { + return err + } + + // Move the VM GPT alt header to end of disk if needed (not needed in unsafe resize mode as it is + // expected the caller will do all necessary post resize actions themselves). + if vol.IsVMBlock() && !allowUnsafeResize { + err = d.moveGPTAltHeader(devPath) + if err != nil { + return err + } + } + } + + return nil +} + +// GetVolumeDiskPath returns the location of a root disk block device. +func (d *pure) GetVolumeDiskPath(vol Volume) (string, error) { + if vol.IsVMBlock() || (vol.volType == VolumeTypeCustom && IsContentBlock(vol.contentType)) { + devPath, _, err := d.getMappedDevPath(vol, false) + return devPath, err + } + + return "", ErrNotSupported +} + +// ListVolumes returns a list of LXD volumes in storage pool. +func (d *pure) ListVolumes() ([]Volume, error) { + return []Volume{}, nil +} + +// MountVolume mounts a volume and increments ref counter. Please call UnmountVolume() when done with the volume. +func (d *pure) MountVolume(vol Volume, op *operations.Operation) error { + unlock, err := vol.MountLock() + if err != nil { + return err + } + + defer unlock() + + revert := revert.New() + defer revert.Fail() + + // Activate Pure Storage volume if needed. + volDevPath, cleanup, err := d.getMappedDevPath(vol, true) + if err != nil { + return err + } + + revert.Add(cleanup) + + if vol.contentType == ContentTypeFS { + mountPath := vol.MountPath() + if !filesystem.IsMountPoint(mountPath) { + err = vol.EnsureMountPath() + if err != nil { + return err + } + + fsType := vol.ConfigBlockFilesystem() + + if vol.mountFilesystemProbe { + fsType, err = fsProbe(volDevPath) + if err != nil { + return fmt.Errorf("Failed probing filesystem: %w", err) + } + } + + mountFlags, mountOptions := filesystem.ResolveMountOptions(strings.Split(vol.ConfigBlockMountOptions(), ",")) + err = TryMount(volDevPath, mountPath, fsType, mountFlags, mountOptions) + if err != nil { + return err + } + + d.logger.Debug("Mounted Pure Storage volume", logger.Ctx{"volName": vol.name, "dev": volDevPath, "path": mountPath, "options": mountOptions}) + } + } else if vol.contentType == ContentTypeBlock { + // For VMs, mount the filesystem volume. + if vol.IsVMBlock() { + fsVol := vol.NewVMBlockFilesystemVolume() + err := d.MountVolume(fsVol, op) + if err != nil { + return err + } + } + } + + vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. + revert.Success() + return nil +} + +// UnmountVolume simulates unmounting a volume. +// keepBlockDev indicates if backing block device should not be unmapped if volume is unmounted. +func (d *pure) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { + unlock, err := vol.MountLock() + if err != nil { + return false, err + } + + defer unlock() + + ourUnmount := false + mountPath := vol.MountPath() + refCount := vol.MountRefCountDecrement() + + // Attempt to unmount the volume. + if vol.contentType == ContentTypeFS && filesystem.IsMountPoint(mountPath) { + if refCount > 0 { + d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) + return false, ErrInUse + } + + err := TryUnmount(mountPath, unix.MNT_DETACH) + if err != nil { + return false, err + } + + // Attempt to unmap. + if !keepBlockDev { + err = d.unmapVolume(vol) + if err != nil { + return false, err + } + } + + ourUnmount = true + } else if vol.contentType == ContentTypeBlock { + // For VMs, unmount the filesystem volume. + if vol.IsVMBlock() { + fsVol := vol.NewVMBlockFilesystemVolume() + ourUnmount, err = d.UnmountVolume(fsVol, false, op) + if err != nil { + return false, err + } + } + + if !keepBlockDev { + // Check if device is currently mapped (but don't map if not). + devPath, _, _ := d.getMappedDevPath(vol, false) + if devPath != "" && shared.PathExists(devPath) { + if refCount > 0 { + d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) + return false, ErrInUse + } + + // Attempt to unmap. + err := d.unmapVolume(vol) + if err != nil { + return false, err + } + + ourUnmount = true + } + } + } + + return ourUnmount, nil +} + +// RenameVolume renames a volume and its snapshots. +func (d *pure) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { + // Renaming a volume won't change an actual name of the Pure Storage volume. + return nil +} + +// RestoreVolume restores a volume from a snapshot. +func (d *pure) RestoreVolume(vol Volume, snapVol Volume, op *operations.Operation) error { + ourUnmount, err := d.UnmountVolume(vol, false, op) + if err != nil { + return err + } + + if ourUnmount { + defer func() { _ = d.MountVolume(vol, op) }() + } + + volName, err := d.getVolumeName(vol) + if err != nil { + return err + } + + snapVolName, err := d.getVolumeName(snapVol) + if err != nil { + return err + } + + // Overwrite existing volume by copying the given snapshot content into it. + err = d.client().restoreVolumeSnapshot(vol.pool, volName, snapVolName) + if err != nil { + return err + } + + // For VMs, also restore the filesystem volume. + if vol.IsVMBlock() { + fsVol := vol.NewVMBlockFilesystemVolume() + snapFSVol := snapVol.NewVMBlockFilesystemVolume() + err := d.RestoreVolume(fsVol, snapFSVol, op) + if err != nil { + return err + } + } + + return nil +} + +// MigrateVolume sends a volume for migration. +func (d *pure) MigrateVolume(vol VolumeCopy, conn io.ReadWriteCloser, volSrcArgs *migration.VolumeSourceArgs, op *operations.Operation) error { + // When performing a cluster member move don't do anything on the source member. + if volSrcArgs.ClusterMove { + return nil + } + + return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) +} + +// BackupVolume creates an exported version of a volume. +func (d *pure) BackupVolume(vol VolumeCopy, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots []string, op *operations.Operation) error { + return genericVFSBackupVolume(d, vol, tarWriter, snapshots, op) +} + +// CreateVolumeSnapshot creates a snapshot of a volume. +func (d *pure) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { + return d.createVolumeSnapshot(snapVol, true, op) +} + +// createVolumeSnapshot creates a snapshot of a volume. If snapshotVMfilesystem is false, a VM's filesystem volume +// is not copied. +func (d *pure) createVolumeSnapshot(snapVol Volume, snapshotVMfilesystem bool, op *operations.Operation) error { + revert := revert.New() + defer revert.Fail() + + parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) + sourcePath := GetVolumeMountPath(d.name, snapVol.volType, parentName) + + if filesystem.IsMountPoint(sourcePath) { + // Attempt to sync and freeze filesystem, but do not error if not able to freeze (as filesystem + // could still be busy), as we do not guarantee the consistency of a snapshot. This is costly but + // try to ensure that all cached data has been committed to disk. If we don't then the snapshot + // of the underlying filesystem can be inconsistent or, in the worst case, empty. + unfreezeFS, err := d.filesystemFreeze(sourcePath) + if err == nil { + defer func() { _ = unfreezeFS() }() + } + } + + // Create the parent directory. + err := createParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) + if err != nil { + return err + } + + err = snapVol.EnsureMountPath() + if err != nil { + return err + } + + parentVol := getSnapshotParentVolume(snapVol) + parentVolName, err := d.getVolumeName(parentVol) + if err != nil { + return err + } + + snapVolName, err := d.getVolumeName(snapVol) + if err != nil { + return err + } + + err = d.client().createVolumeSnapshot(snapVol.pool, parentVolName, snapVolName) + if err != nil { + return err + } + + revert.Add(func() { _ = d.DeleteVolumeSnapshot(snapVol, op) }) + + // For VMs, create a snapshot of the filesystem volume too. + // Skip if snapshotVMfilesystem is false to prevent overwriting separately copied volumes. + if snapVol.IsVMBlock() && snapshotVMfilesystem { + fsVol := snapVol.NewVMBlockFilesystemVolume() + + // Set the parent volume's UUID. + fsVol.SetParentUUID(snapVol.parentUUID) + + err := d.CreateVolumeSnapshot(fsVol, op) + if err != nil { + return err + } + + revert.Add(func() { _ = d.DeleteVolumeSnapshot(fsVol, op) }) + } + + revert.Success() + return nil +} + +// DeleteVolumeSnapshot removes a snapshot from the storage device. +func (d *pure) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { + parentVol := getSnapshotParentVolume(snapVol) + parentVolName, err := d.getVolumeName(parentVol) + if err != nil { + return err + } + + snapVolName, err := d.getVolumeName(snapVol) + if err != nil { + return err + } + + err = d.client().deleteVolumeSnapshot(snapVol.pool, parentVolName, snapVolName) + if err != nil { + return err + } + + mountPath := snapVol.MountPath() + + if snapVol.contentType == ContentTypeFS && shared.PathExists(mountPath) { + err = wipeDirectory(mountPath) + if err != nil { + return err + } + + err = os.Remove(mountPath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("Failed to remove %q: %w", mountPath, err) + } + } + + // Remove the parent snapshot directory if this is the last snapshot being removed. + err = deleteParentSnapshotDirIfEmpty(d.name, snapVol.volType, parentVol.name) + if err != nil { + return err + } + + // For VM images, delete the filesystem volume too. + if snapVol.IsVMBlock() { + fsVol := snapVol.NewVMBlockFilesystemVolume() + fsVol.SetParentUUID(snapVol.parentUUID) + + err := d.DeleteVolumeSnapshot(fsVol, op) + if err != nil { + return err + } + } + + return nil +} + +// MountVolumeSnapshot creates a new temporary volume from a volume snapshot to allow mounting it. +func (d *pure) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { + revert := revert.New() + defer revert.Fail() + + parentVol := getSnapshotParentVolume(snapVol) + + // Get the parent volume name. + parentVolName, err := d.getVolumeName(parentVol) + if err != nil { + return err + } + + // Get the snapshot volume name. + snapVolName, err := d.getVolumeName(snapVol) + if err != nil { + return err + } + + // A Pure Storage snapshot cannot be mounted. To mount a snapshot, a new volume + // has to be created from the snapshot. + err = d.client().copyVolumeSnapshot(snapVol.pool, parentVolName, snapVolName, snapVol.pool, snapVolName) + if err != nil { + return err + } + + // Ensure temporary snapshot volume is remooved in case of an error. + revert.Add(func() { _ = d.client().deleteVolume(snapVol.pool, snapVolName) }) + + // For VMs, also create the temporary filesystem volume snapshot. + if snapVol.IsVMBlock() { + snapFsVol := snapVol.NewVMBlockFilesystemVolume() + snapFsVol.SetParentUUID(snapVol.parentUUID) + + parentFsVol := getSnapshotParentVolume(snapFsVol) + + snapFsVolName, err := d.getVolumeName(snapFsVol) + if err != nil { + return err + } + + parentFsVolName, err := d.getVolumeName(parentFsVol) + if err != nil { + return err + } + + err = d.client().copyVolumeSnapshot(snapVol.pool, parentFsVolName, snapFsVolName, snapVol.pool, snapFsVolName) + if err != nil { + return err + } + + revert.Add(func() { _ = d.client().deleteVolume(snapVol.pool, snapFsVolName) }) + } + + err = d.MountVolume(snapVol, op) + if err != nil { + return err + } + + revert.Success() + return nil +} + +// UnmountVolumeSnapshot unmountes and deletes volume that was temporary created from a snapshot +// to allow mounting it. +func (d *pure) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { + ourUnmount, err := d.UnmountVolume(snapVol, false, op) + if err != nil { + return false, err + } + + if !ourUnmount { + return false, nil + } + + snapVolName, err := d.getVolumeName(snapVol) + if err != nil { + return true, err + } + + // Cleanup temporary snapshot volume. + err = d.client().deleteVolume(snapVol.pool, snapVolName) + if err != nil { + return true, err + } + + // For VMs, also cleanup the temporary volume for a filesystem snapshot. + if snapVol.IsVMBlock() { + snapFsVol := snapVol.NewVMBlockFilesystemVolume() + snapFsVolName, err := d.getVolumeName(snapFsVol) + if err != nil { + return true, err + } + + err = d.client().deleteVolume(snapVol.pool, snapFsVolName) + if err != nil { + return true, err + } + } + + return ourUnmount, nil +} + +// VolumeSnapshots returns a list of Pure Storage snapshot names for the given volume (in no particular order). +func (d *pure) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { + volName, err := d.getVolumeName(vol) + if err != nil { + return nil, err + } + + volumeSnapshots, err := d.client().getVolumeSnapshots(vol.pool, volName) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + return nil, nil + } + + return nil, err + } + + snapshotNames := make([]string, 0, len(volumeSnapshots)) + for _, snapshot := range volumeSnapshots { + // Snapshot name contains storage pool and volume names as prefix. + // Storage pool is delimited with double colon (::) and volume with a dot. + _, volAndSnapName, _ := strings.Cut(snapshot.Name, "::") + _, snapshotName, _ := strings.Cut(volAndSnapName, ".") + + snapshotNames = append(snapshotNames, snapshotName) + } + + return snapshotNames, nil +} + +// CheckVolumeSnapshots checks that the volume's snapshots, according to the storage driver, +// match those provided. Note that additional snapshots may exist within the Pure Storage pool +// if protection groups are configured outside of LXD. +func (d *pure) CheckVolumeSnapshots(vol Volume, snapVols []Volume, op *operations.Operation) error { + // Get all of the volume's snapshots in base64 encoded format. + storageSnapshotNames, err := vol.driver.VolumeSnapshots(vol, op) + if err != nil { + return err + } + + // Check if the provided list of volume snapshots matches the ones from the storage. + for _, snap := range snapVols { + snapName, err := d.getVolumeName(snap) + if err != nil { + return err + } + + if !slices.Contains(storageSnapshotNames, snapName) { + return fmt.Errorf("Snapshot %q expected but not in storage", snapName) + } + } + + return nil +} + +// RenameVolumeSnapshot renames a volume snapshot. +func (d *pure) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { + // Renaming a volume snapshot won't change an actual name of the Pure Storage volume snapshot. + return nil +} + +// getSnapshotParentVolume returns a parent volume that has volatile.uuid set to the snapshot's parent UUID. +func getSnapshotParentVolume(snapVol Volume) Volume { + parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) + parentVolConfig := map[string]string{ + "volatile.uuid": snapVol.parentUUID, + } + + return NewVolume(snapVol.driver, snapVol.pool, snapVol.volType, snapVol.contentType, parentName, parentVolConfig, nil) +} diff --git a/lxd/storage/drivers/driver_types.go b/lxd/storage/drivers/driver_types.go index cd842308d25c..8654eef08106 100644 --- a/lxd/storage/drivers/driver_types.go +++ b/lxd/storage/drivers/driver_types.go @@ -19,6 +19,7 @@ type Info struct { DirectIO bool // Whether the driver supports direct I/O. IOUring bool // Whether the driver supports io_uring. MountedRoot bool // Whether the pool directory itself is a mount. + PopulateParentVolumeUUID bool // Whether the volume should have parent UUID populated before any action. } // VolumeFiller provides a struct for filling a volume. diff --git a/lxd/storage/drivers/driver_zfs.go b/lxd/storage/drivers/driver_zfs.go index 3bd1f76b05d1..9d30ec24ab85 100644 --- a/lxd/storage/drivers/driver_zfs.go +++ b/lxd/storage/drivers/driver_zfs.go @@ -130,6 +130,7 @@ func (d *zfs) Info() Info { DirectIO: zfsDirectIO, MountedRoot: false, Buckets: true, + PopulateParentVolumeUUID: false, } return info diff --git a/lxd/storage/drivers/load.go b/lxd/storage/drivers/load.go index 198b11f5d084..187945a04e50 100644 --- a/lxd/storage/drivers/load.go +++ b/lxd/storage/drivers/load.go @@ -13,6 +13,7 @@ var drivers = map[string]func() driver{ "dir": func() driver { return &dir{} }, "lvm": func() driver { return &lvm{} }, "powerflex": func() driver { return &powerflex{} }, + "pure": func() driver { return &pure{} }, "zfs": func() driver { return &zfs{} }, } diff --git a/lxd/storage/drivers/utils.go b/lxd/storage/drivers/utils.go index 954ace3074ab..4b8c174002cf 100644 --- a/lxd/storage/drivers/utils.go +++ b/lxd/storage/drivers/utils.go @@ -894,3 +894,18 @@ func roundAbove(above, val int64) int64 { return rounded } + +// ResolveServerName returns the given server name if it is not "none". +// If the server name is "none", it retrieves and returns the server's hostname. +func ResolveServerName(serverName string) (string, error) { + if serverName != "none" { + return serverName, nil + } + + hostname, err := os.Hostname() + if err != nil { + return "", fmt.Errorf("Failed to get hostname: %w", err) + } + + return hostname, nil +} diff --git a/lxd/storage/utils.go b/lxd/storage/utils.go index e6684a0bda5f..882fcca4fe5d 100644 --- a/lxd/storage/utils.go +++ b/lxd/storage/utils.go @@ -535,7 +535,7 @@ func poolAndVolumeCommonRules(vol *drivers.Volume) map[string]func(string) error // shortdesc: Size/quota of the storage bucket // scope: local "size": validate.Optional(validate.IsSize), - // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex; group=volume-conf; key=snapshots.expiry) + // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex,storage-pure; group=volume-conf; key=snapshots.expiry) // Specify an expression like `1M 2H 3d 4w 5m 6y`. // --- // type: string @@ -548,7 +548,7 @@ func poolAndVolumeCommonRules(vol *drivers.Volume) map[string]func(string) error _, err := shared.GetExpiry(time.Time{}, value) return err }, - // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex; group=volume-conf; key=snapshots.schedule) + // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex,storage-pure; group=volume-conf; key=snapshots.schedule) // Specify either a cron expression (` `), a comma-separated list of schedule aliases (`@hourly`, `@daily`, `@midnight`, `@weekly`, `@monthly`, `@annually`, `@yearly`), or leave empty to disable automatic snapshots (the default). // --- // type: string @@ -557,7 +557,7 @@ func poolAndVolumeCommonRules(vol *drivers.Volume) map[string]func(string) error // shortdesc: Schedule for automatic volume snapshots // scope: global "snapshots.schedule": validate.Optional(validate.IsCron([]string{"@hourly", "@daily", "@midnight", "@weekly", "@monthly", "@annually", "@yearly"})), - // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex; group=volume-conf; key=snapshots.pattern) + // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex,storage-pure; group=volume-conf; key=snapshots.pattern) // You can specify a naming template that is used for scheduled snapshots and unnamed snapshots. // // {{snapshot_pattern_detail}} @@ -608,7 +608,7 @@ func poolAndVolumeCommonRules(vol *drivers.Volume) map[string]func(string) error // Those keys are only valid for volumes. if vol != nil { - // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex; group=volume-conf; key=volatile.uuid) + // lxdmeta:generate(entities=storage-btrfs,storage-cephfs,storage-ceph,storage-dir,storage-lvm,storage-zfs,storage-powerflex,storage-pure; group=volume-conf; key=volatile.uuid) // // --- // type: string diff --git a/test/backends/pure.sh b/test/backends/pure.sh new file mode 100644 index 000000000000..b6e4bcfc8b85 --- /dev/null +++ b/test/backends/pure.sh @@ -0,0 +1,64 @@ +pure_setup() { + local LXD_DIR + + LXD_DIR=$1 + + echo "==> Setting up Pure Storage backend in ${LXD_DIR}" +} + +# pure_configure creates Pure Storage storage pool and configures instance root disk +# device in default profile to use that storage pool. +pure_configure() { + local LXD_DIR + + LXD_DIR=$1 + + echo "==> Configuring Pure Storage backend in ${LXD_DIR}" + + # Create pure storage storage pool. + lxc storage create "lxdtest-$(basename "${LXD_DIR}")" pure \ + pure.gateway="${PURE_GATEWAY}" \ + pure.gateway.verify="${PURE_GATEWAY_VERIFY:-true}" \ + pure.api.token="${PURE_API_TOKEN}" \ + pure.mode="${PURE_MODE:-nvme}" \ + volume.size=25MiB + + # Add the storage pool to the default profile. + lxc profile device add default root disk path="/" pool="lxdtest-$(basename "${LXD_DIR}")" +} + +# configure_pure_pool creates new Pure Storage storage pool with a given name. +# Additional arguments are appended to the lxc storage create command. +# If there is anything on the stdin, the content is passed to the lxc storage create command as stdin as well. +configure_pure_pool() { + poolName=$1 + shift 1 + + if [ -p /dev/stdin ]; then + # Use heredoc if there's input on stdin + lxc storage create "${poolName}" pure \ + pure.gateway="${PURE_GATEWAY}" \ + pure.gateway.verify="${PURE_GATEWAY_VERIFY:-true}" \ + pure.api.token="${PURE_API_TOKEN}" \ + pure.mode="${PURE_MODE:-nvme}" \ + "$@" < Tearing down Pure Storage backend in ${LXD_DIR}" +} diff --git a/test/extras/stresstest.sh b/test/extras/stresstest.sh index 22112fe4b951..c492f2bdcf84 100755 --- a/test/extras/stresstest.sh +++ b/test/extras/stresstest.sh @@ -21,7 +21,6 @@ fi echo "==> Running the LXD testsuite" -BASEURL=https://127.0.0.1:18443 my_curl() { curl -k -s --cert "${LXD_CONF}/client.crt" --key "${LXD_CONF}/client.key" "${@}" } diff --git a/test/includes/storage.sh b/test/includes/storage.sh index 9f3f001c8faa..37e412f4e024 100644 --- a/test/includes/storage.sh +++ b/test/includes/storage.sh @@ -39,6 +39,10 @@ available_storage_backends() { backends="dir" # always available + if [ -n "${PURE_GATEWAY:-}" ] && [ -n "${PURE_API_TOKEN}" ]; then + backends="$backends pure" + fi + storage_backends="btrfs lvm zfs" if [ -n "${LXD_CEPH_CLUSTER:-}" ]; then storage_backends="${storage_backends} ceph" @@ -182,4 +186,4 @@ delete_object_storage_pool() { # shellcheck disable=SC2154 deconfigure_loop_device "${loop_file}" "${loop_device}" fi -} \ No newline at end of file +} diff --git a/test/main.sh b/test/main.sh index 76457adb205a..118c4954b864 100755 --- a/test/main.sh +++ b/test/main.sh @@ -395,6 +395,7 @@ if [ "${1:-"all"}" != "cluster" ]; then run_test test_storage_driver_cephfs "cephfs storage driver" run_test test_storage_driver_dir "dir storage driver" run_test test_storage_driver_zfs "zfs storage driver" + run_test test_storage_driver_pure "pure storage driver" run_test test_storage_buckets "storage buckets" run_test test_storage_volume_import "storage volume import" run_test test_storage_volume_initial_config "storage volume initial configuration" diff --git a/test/suites/backup.sh b/test/suites/backup.sh index 772b17c53682..4fa4c7ab9371 100644 --- a/test/suites/backup.sh +++ b/test/suites/backup.sh @@ -6,6 +6,11 @@ test_storage_volume_recover() { poolName=$(lxc profile device get default root pool) poolDriver=$(lxc storage show "${poolName}" | awk '/^driver:/ {print $2}') + if [ "${poolDriver}" = "pure" ]; then + echo "==> SKIP: Storage driver does not support recovery" + return + fi + # Create custom block volume. lxc storage volume create "${poolName}" vol1 --type=block @@ -77,6 +82,11 @@ test_container_recover() { LXD_DIR=${LXD_IMPORT_DIR} lxd_backend=$(storage_backend "$LXD_DIR") + if [ "${lxd_backend}" = "pure" ]; then + echo "==> SKIP: Storage driver does not support recovery" + return + fi + ensure_import_testimage poolName=$(lxc profile device get default root pool) @@ -1008,6 +1018,13 @@ test_backup_volume_expiry() { } test_backup_export_import_recover() { + lxd_backend=$(storage_backend "$LXD_DIR") + + if [ "$lxd_backend" = "pure" ]; then + echo "==> SKIP: Storage driver does not support recovery" + return + fi + ( set -e diff --git a/test/suites/container_move.sh b/test/suites/container_move.sh index ecc117b563e8..db8bf60cb594 100644 --- a/test/suites/container_move.sh +++ b/test/suites/container_move.sh @@ -11,7 +11,11 @@ test_container_move() { # Setup. lxc project create "${project}" - lxc storage create "${pool2}" "${lxd_backend}" + if [ "${lxd_backend}" = "pure" ]; then + configure_pure_pool "${pool2}" + else + lxc storage create "${pool2}" "${lxd_backend}" + fi lxc profile create "${profile}" --project "${project}" lxc profile device add "${profile}" root disk pool="${pool2}" path=/ --project "${project}" diff --git a/test/suites/storage_driver_pure.sh b/test/suites/storage_driver_pure.sh new file mode 100644 index 000000000000..bf021f4120ab --- /dev/null +++ b/test/suites/storage_driver_pure.sh @@ -0,0 +1,128 @@ +test_storage_driver_pure() { + local LXD_STORAGE_DIR lxd_backend + + lxd_backend=$(storage_backend "$LXD_DIR") + if [ "$lxd_backend" != "pure" ]; then + return + fi + + LXD_STORAGE_DIR=$(mktemp -d -p "${TEST_DIR}" XXXXXXXXX) + chmod +x "${LXD_STORAGE_DIR}" + spawn_lxd "${LXD_STORAGE_DIR}" false + + ( + set -eux + # shellcheck disable=2030 + LXD_DIR="${LXD_STORAGE_DIR}" + + # Create 2 storage pools. + poolName1="lxdtest-$(basename "${LXD_DIR}")-pool1" + poolName2="lxdtest-$(basename "${LXD_DIR}")-pool2" + configure_pure_pool "${poolName1}" + configure_pure_pool "${poolName2}" + + # Configure default volume size for pools. + lxc storage set "${poolName1}" volume.size=25MiB + lxc storage set "${poolName2}" volume.size=25MiB + + # Set default storage pool for image import. + lxc profile device add default root disk path="/" pool="${poolName1}" + + # Import image into default storage pool. + ensure_import_testimage + + # Muck around with some containers on various pools. + lxc init testimage c1pool1 -s "${poolName1}" + lxc list -c b c1pool1 | grep "${poolName1}" + + lxc init testimage c2pool2 -s "${poolName2}" + lxc list -c b c2pool2 | grep "${poolName2}" + + lxc launch images:alpine/edge c3pool1 -s "${poolName1}" + lxc list -c b c3pool1 | grep "${poolName1}" + + lxc launch images:alpine/edge c4pool2 -s "${poolName2}" + lxc list -c b c4pool2 | grep "${poolName2}" + + lxc storage set "${poolName1}" volume.block.filesystem xfs + + # xfs is unhappy with block devices < 300 MiB. + lxc storage set "${poolName1}" volume.size 300MiB + lxc init testimage c5pool1 -s "${poolName1}" + + # Test whether dependency tracking is working correctly. We should be able + # to create a container, copy it, which leads to a dependency relation + # between the source container's storage volume and the copied container's + # storage volume. Now, we delete the source container which will trigger a + # rename operation and not an actual delete operation. Now we create a + # container of the same name as the source container again, create a copy of + # it to introduce another dependency relation. Now we delete the source + # container again. This should work. If it doesn't it means the rename + # operation tries to map the two source to the same name. + lxc init testimage a -s "${poolName1}" + lxc copy a b + lxc delete a + lxc init testimage a -s "${poolName1}" + lxc copy a c + lxc delete a + lxc delete b + lxc delete c + + lxc storage volume create "${poolName1}" c1pool1 + lxc storage volume attach "${poolName1}" c1pool1 c1pool1 testDevice /opt + ! lxc storage volume attach "${poolName1}" c1pool1 c1pool1 testDevice2 /opt || false + lxc storage volume detach "${poolName1}" c1pool1 c1pool1 + lxc storage volume attach "${poolName1}" custom/c1pool1 c1pool1 testDevice /opt + ! lxc storage volume attach "${poolName1}" custom/c1pool1 c1pool1 testDevice2 /opt || false + lxc storage volume detach "${poolName1}" c1pool1 c1pool1 + + lxc storage volume create "${poolName1}" c2pool2 + lxc storage volume attach "${poolName1}" c2pool2 c2pool2 testDevice /opt + ! lxc storage volume attach "${poolName1}" c2pool2 c2pool2 testDevice2 /opt || false + lxc storage volume detach "${poolName1}" c2pool2 c2pool2 + lxc storage volume attach "${poolName1}" custom/c2pool2 c2pool2 testDevice /opt + ! lxc storage volume attach "${poolName1}" custom/c2pool2 c2pool2 testDevice2 /opt || false + lxc storage volume detach "${poolName1}" c2pool2 c2pool2 + + lxc storage volume create "${poolName2}" c3pool1 + lxc storage volume attach "${poolName2}" c3pool1 c3pool1 testDevice /opt + ! lxc storage volume attach "${poolName2}" c3pool1 c3pool1 testDevice2 /opt || false + lxc storage volume detach "${poolName2}" c3pool1 c3pool1 + lxc storage volume attach "${poolName2}" c3pool1 c3pool1 testDevice /opt + ! lxc storage volume attach "${poolName2}" c3pool1 c3pool1 testDevice2 /opt || false + lxc storage volume detach "${poolName2}" c3pool1 c3pool1 + + lxc storage volume create "${poolName2}" c4pool2 + lxc storage volume attach "${poolName2}" c4pool2 c4pool2 testDevice /opt + ! lxc storage volume attach "${poolName2}" c4pool2 c4pool2 testDevice2 /opt || false + lxc storage volume detach "${poolName2}" c4pool2 c4pool2 + lxc storage volume attach "${poolName2}" custom/c4pool2 c4pool2 testDevice /opt + ! lxc storage volume attach "${poolName2}" custom/c4pool2 c4pool2 testDevice2 /opt || false + lxc storage volume detach "${poolName2}" c4pool2 c4pool2 + lxc storage volume rename "${poolName2}" c4pool2 c4pool2-renamed + lxc storage volume rename "${poolName2}" c4pool2-renamed c4pool2 + + lxc delete -f c1pool1 + lxc delete -f c3pool1 + lxc delete -f c5pool1 + + lxc delete -f c4pool2 + lxc delete -f c2pool2 + + lxc storage volume set "${poolName1}" c1pool1 size 500MiB + lxc storage volume unset "${poolName1}" c1pool1 size + + lxc storage volume delete "${poolName1}" c1pool1 + lxc storage volume delete "${poolName1}" c2pool2 + lxc storage volume delete "${poolName2}" c3pool1 + lxc storage volume delete "${poolName2}" c4pool2 + + lxc image delete testimage + lxc profile device remove default root + lxc storage delete "${poolName1}" + lxc storage delete "${poolName2}" + ) + + # shellcheck disable=SC2031 + kill_lxd "${LXD_STORAGE_DIR}" +} diff --git a/test/suites/storage_local_volume_handling.sh b/test/suites/storage_local_volume_handling.sh index 456eebd8b7e4..09c5621be6f5 100755 --- a/test/suites/storage_local_volume_handling.sh +++ b/test/suites/storage_local_volume_handling.sh @@ -36,6 +36,10 @@ test_storage_local_volume_handling() { lxc storage create "${pool_base}-zfs" zfs size=1GiB fi + if storage_backend_available "pure"; then + configure_pure_pool "${pool_base}-pure" + fi + # Test all combinations of our storage drivers driver="${lxd_backend}" @@ -51,11 +55,13 @@ test_storage_local_volume_handling() { pool_opts="volume.size=25MiB ceph.osd.pg_num=16" fi - if [ "$driver" = "lvm" ]; then + if [ "$driver" = "lvm" ] || [ "$driver" = "pure" ]; then pool_opts="volume.size=25MiB" fi - if [ -n "${pool_opts}" ]; then + if [ "$driver" = "pure" ]; then + configure_pure_pool "${pool}1" "${pool_opts}" + elif [ -n "${pool_opts}" ]; then # shellcheck disable=SC2086 lxc storage create "${pool}1" "${driver}" $pool_opts else @@ -179,8 +185,8 @@ test_storage_local_volume_handling() { lxc storage volume delete "${pool}1" vol1 lxc storage delete "${pool}1" - for source_driver in "btrfs" "ceph" "cephfs" "dir" "lvm" "zfs"; do - for target_driver in "btrfs" "ceph" "cephfs" "dir" "lvm" "zfs"; do + for source_driver in "btrfs" "ceph" "cephfs" "dir" "lvm" "zfs" "pure"; do + for target_driver in "btrfs" "ceph" "cephfs" "dir" "lvm" "zfs" "pure"; do # shellcheck disable=SC2235 if [ "$source_driver" != "$target_driver" ] \ && ([ "$lxd_backend" = "$source_driver" ] || ([ "$lxd_backend" = "ceph" ] && [ "$source_driver" = "cephfs" ] && [ -n "${LXD_CEPH_CEPHFS:-}" ])) \ diff --git a/test/suites/storage_snapshots.sh b/test/suites/storage_snapshots.sh index 28a19b4adccc..5134da6cd34e 100644 --- a/test/suites/storage_snapshots.sh +++ b/test/suites/storage_snapshots.sh @@ -14,7 +14,12 @@ test_storage_volume_snapshots() { storage_pool2="${storage_pool}2" storage_volume="${storage_pool}-vol" - lxc storage create "$storage_pool" "$lxd_backend" + if [ "${lxd_backend}" = "pure" ]; then + # Pure Storage needs some additional configuration, therefore create it using a helper function. + configure_pure_pool "${storage_pool}" + else + lxc storage create "$storage_pool" "$lxd_backend" + fi lxc storage volume create "${storage_pool}" "${storage_volume}" lxc launch testimage c1 -s "${storage_pool}" lxc storage volume attach "${storage_pool}" "${storage_volume}" c1 /mnt