diff --git a/terraform/files/bin/apply_cindercsi.sh b/terraform/files/bin/apply_cindercsi.sh index 28e375b4..cb83b21b 100755 --- a/terraform/files/bin/apply_cindercsi.sh +++ b/terraform/files/bin/apply_cindercsi.sh @@ -34,32 +34,29 @@ else fi echo "# Install Cinder CSI persistent storage support $CCSI_VERSION to $CLUSTER_NAME" -if test -n "$SNAP_VERSION"; then - # deploy snapshot CRDs - for name in snapshot.storage.k8s.io_volumesnapshotcontents.yaml snapshot.storage.k8s.io_volumesnapshotclasses.yaml snapshot.storage.k8s.io_volumesnapshots.yaml; do - NAME=${name%.yaml}-$SNAP_VERSION.yaml - if ! test -s $NAME; then - curl -L https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$SNAP_VERSION/client/config/crd/$name -o $NAME - echo -e "\n---" >> $NAME - fi - done - # FIXME: Should we ignore non-working snapshots? - cat snapshot.storage.k8s.io_volumesnapshot* > cindercsi-snapshot-$SNAP_VERSION.yaml +SNAP_VERSION="master" +# deploy snapshot CRDs +for name in snapshot.storage.k8s.io_volumesnapshotcontents.yaml snapshot.storage.k8s.io_volumesnapshotclasses.yaml snapshot.storage.k8s.io_volumesnapshots.yaml; do + NAME=${name%.yaml}-$SNAP_VERSION.yaml + if ! test -s $NAME; then + curl -L https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$SNAP_VERSION/client/config/crd/$name -o $NAME + echo -e "\n---" >> $NAME + fi +done +# FIXME: Should we ignore non-working snapshots? +cat snapshot.storage.k8s.io_volumesnapshot* > cindercsi-snapshot-$SNAP_VERSION.yaml - # deploy snapshot controller - for name in rbac-snapshot-controller.yaml setup-snapshot-controller.yaml; do - NAME=${name%.yaml}-$SNAP_VERSION.yaml - if ! test -s $NAME; then - curl -L https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$SNAP_VERSION/deploy/kubernetes/snapshot-controller/$name -o $NAME - echo -e "\n---" >> $NAME - fi - cat $NAME >> cindercsi-snapshot-$SNAP_VERSION.yaml - done +# deploy snapshot controller +for name in rbac-snapshot-controller.yaml setup-snapshot-controller.yaml; do + NAME=${name%.yaml}-$SNAP_VERSION.yaml + if ! test -s $NAME; then + curl -L https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$SNAP_VERSION/deploy/kubernetes/snapshot-controller/$name -o $NAME + echo -e "\n---" >> $NAME + fi + cat $NAME >> cindercsi-snapshot-$SNAP_VERSION.yaml +done - cp -p cindercsi-snapshot-$SNAP_VERSION.yaml ~/${CLUSTER_NAME}/deployed-manifests.d/cindercsi-snapshot.yaml -else - cp -p external-snapshot-crds.yaml ~/$CLUSTER_NAME/deployed-manifests.d/cindercsi-snapshot.yaml -fi +cp -p cindercsi-snapshot-$SNAP_VERSION.yaml ~/${CLUSTER_NAME}/deployed-manifests.d/cindercsi-snapshot.yaml if test -n "$CCSI_VERSION"; then # Now get cinder diff --git a/terraform/files/bin/openstack-kube-versions.inc b/terraform/files/bin/openstack-kube-versions.inc index 1cbdc8dd..eabc3a82 100644 --- a/terraform/files/bin/openstack-kube-versions.inc +++ b/terraform/files/bin/openstack-kube-versions.inc @@ -3,13 +3,13 @@ # (c) Kurt Garloff , 3/2022 # SPDX-License-Identifier: Apache-2.0 # Images from https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images -k8s_versions=("v1.18.20" "v1.19.16" "v1.20.15" "v1.21.14" "v1.22.17" "v1.23.16" "v1.24.15" "v1.25.15" "v1.26.11" "v1.27.8" "v1.28.4") +k8s_versions=("v1.21.14" "v1.22.17" "v1.23.16" "v1.24.15" "v1.25.15" "v1.26.11" "v1.27.8" "v1.28.4") # OCCM, CCM-RBAC, Cinder CSI, Cinder-Snapshot (TODO: Manila CSI) -occm_versions=("" "" "v1.21.1" "v1.21.1" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") -#ccmr_versions=("" "" "" "" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") -ccmr_versions=("" "" "v1.22.2" "v1.22.2" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") -ccsi_versions=("" "" "v1.20.5" "v1.21.1" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") -min_snapshot_master="v1.21.0" +occm_versions=("v1.21.1" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") +#ccmr_versions=("" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") +ccmr_versions=("v1.22.2" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") +ccsi_versions=("v1.21.1" "v1.22.2" "v1.23.4" "v1.24.6" "v1.25.6" "v1.26.4" "v1.27.3" "v1.28.1") + # Versions that require a --allow-preview-versions flag techprev_versions=("v1.28" "v2") @@ -83,8 +83,6 @@ find_openstack_versions() NUMV=${#k8s_versions[*]} k8min=$(dotversion ${k8s_versions[0]%.*}) k8max=$(dotversion ${k8s_versions[$((NUMV-1))]%.*}) - snapmaster=$(dotversion $min_snapshot_master) - if test $k8vers -lt $snapmaster; then SNAP_VERSION=""; else SNAP_VERSION="master"; fi #echo "$k8vers $k8min $k8max" if test $k8vers -lt $k8min; then setversions 0; return 0; fi if test $k8vers -gt $((k8max+99)); then setversions $((NUMV-1)); return 0; fi diff --git a/terraform/files/kubernetes-manifests.d/external-snapshot-crds.yaml b/terraform/files/kubernetes-manifests.d/external-snapshot-crds.yaml deleted file mode 100644 index dbddcbb7..00000000 --- a/terraform/files/kubernetes-manifests.d/external-snapshot-crds.yaml +++ /dev/null @@ -1,420 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" - creationTimestamp: null - name: volumesnapshotclasses.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshotClass - listKind: VolumeSnapshotClassList - plural: volumesnapshotclasses - singular: volumesnapshotclass - scope: Cluster - preserveUnknownFields: false - validation: - openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage - system uses when creating a volume snapshot. A specific VolumeSnapshotClass - is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses - are non-namespaced - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent created - through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot - is deleted. Supported values are "Retain" and "Delete". "Retain" means - that the VolumeSnapshotContent and its physical snapshot on underlying - storage system are kept. "Delete" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are deleted. Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the storage driver that handles this - VolumeSnapshotClass. Required. - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - parameters: - additionalProperties: - type: string - description: parameters is a key-value map with storage driver specific - parameters for creating snapshots. These values are opaque to Kubernetes. - type: object - required: - - deletionPolicy - - driver - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" - creationTimestamp: null - name: volumesnapshotcontents.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshotContent - listKind: VolumeSnapshotContentList - plural: volumesnapshotcontents - singular: volumesnapshotcontent - scope: Cluster - subresources: - status: {} - preserveUnknownFields: false - validation: - openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot - object in the underlying storage system - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - spec: - description: spec defines properties of a VolumeSnapshotContent created - by the underlying storage system. Required. - properties: - deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent - and its physical snapshot on the underlying storage system should - be deleted when its bound VolumeSnapshot is deleted. Supported values - are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are kept. "Delete" - means that the VolumeSnapshotContent and its physical snapshot on - underlying storage system are deleted. In dynamic snapshot creation - case, this field will be filled in with the "DeletionPolicy" field - defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For - pre-existing snapshots, users MUST specify this field when creating - the VolumeSnapshotContent object. Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the CSI driver used to create the - physical snapshot on the underlying storage system. This MUST be the - same as the name returned by the CSI GetPluginName() call for that - driver. Required. - type: string - source: - description: source specifies from where a snapshot will be created. - This field is immutable after creation. Required. - properties: - snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of a - pre-existing snapshot on the underlying storage system. This field - is immutable. - type: string - volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the volume - from which a snapshot should be dynamically taken from. This field - is immutable. - type: string - type: object - volumeSnapshotClassName: - description: name of the VolumeSnapshotClass to which this snapshot - belongs. - type: string - volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object to - which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName - field must reference to this VolumeSnapshotContent's name for the - bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent - object, name and namespace of the VolumeSnapshot object MUST be provided - for binding to happen. This field is immutable after creation. Required. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an - entire object, this string should contain a valid JSON/Go field - access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen only - to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change - in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is - made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - required: - - deletionPolicy - - driver - - source - - volumeSnapshotRef - type: object - status: - description: status represents the current information of a snapshot. - properties: - creationTime: - description: creationTime is the timestamp when the point-in-time snapshot - is taken by the underlying storage system. In dynamic snapshot creation - case, this field will be filled in with the "creation_time" value - returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing - snapshot, this field will be filled with the "creation_time" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it. If not specified, it indicates the creation time is unknown. The - format of this field is a Unix nanoseconds time encoded as an int64. - On Unix, the command `date +%s%N` returns the current time in nanoseconds - since 1970-01-01 00:00:00 UTC. - format: int64 - type: integer - error: - description: error is the latest observed error during snapshot creation, - if any. - properties: - message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be logged, - and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if a snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in with the "ready_to_use" value returned from CSI - "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this - field will be filled with the "ready_to_use" value returned from the - CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, - this field will be set to "True". If not specified, it means the readiness - of a snapshot is unknown. - type: boolean - restoreSize: - description: restoreSize represents the complete size of the snapshot - in bytes. In dynamic snapshot creation case, this field will be filled - in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "size_bytes" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. When restoring a volume from - this snapshot, the size of the volume MUST NOT be smaller than the - restoreSize if it is specified, otherwise the restoration will fail. - If not specified, it indicates that the size is unknown. - format: int64 - minimum: 0 - type: integer - snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot on - the underlying storage system. If not specified, it indicates that - dynamic snapshot creation has either failed or it is still in progress. - type: string - type: object - required: - - spec - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" - creationTimestamp: null - name: volumesnapshots.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshot - listKind: VolumeSnapshotList - plural: volumesnapshots - singular: volumesnapshot - scope: Namespaced - subresources: - status: {} - preserveUnknownFields: false - validation: - openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time - snapshot of a persistent volume, or binding to a pre-existing snapshot. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - spec: - description: 'spec defines the desired characteristics of a snapshot requested - by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots - Required.' - properties: - source: - description: source specifies where a snapshot will be created from. - This field is immutable after creation. Required. - properties: - persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the - PersistentVolumeClaim object in the same namespace as the VolumeSnapshot - object where the snapshot should be dynamically taken from. This - field is immutable. - type: string - volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a pre-existing - VolumeSnapshotContent object. This field is immutable. - type: string - type: object - volumeSnapshotClassName: - description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass - requested by the VolumeSnapshot. If not specified, the default snapshot - class will be used if one exists. If not specified, and there is no - default snapshot class, dynamic snapshot creation will fail. Empty - string is not allowed for this field. TODO(xiangqian): a webhook validation - on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' - type: string - required: - - source - type: object - status: - description: 'status represents the current information of a snapshot. NOTE: - status can be modified by sources other than system controllers, and must - not be depended upon for accuracy. Controllers should only use information - from the VolumeSnapshotContent object after verifying that the binding - is accurate and complete.' - properties: - boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName represents the name of - the VolumeSnapshotContent object to which the VolumeSnapshot object - is bound. If not specified, it indicates that the VolumeSnapshot object - has not been successfully bound to a VolumeSnapshotContent object - yet. NOTE: Specified boundVolumeSnapshotContentName alone does not - mean binding is valid. Controllers MUST always verify bidirectional - binding between VolumeSnapshot and VolumeSnapshotContent to - avoid possible security issues.' - type: string - creationTime: - description: creationTime is the timestamp when the point-in-time snapshot - is taken by the underlying storage system. In dynamic snapshot creation - case, this field will be filled in with the "creation_time" value - returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing - snapshot, this field will be filled with the "creation_time" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it. If not specified, it indicates that the creation time of the snapshot - is unknown. - format: date-time - type: string - error: - description: error is the last observed error during snapshot creation, - if any. This field could be helpful to upper level controllers(i.e., - application controller) to decide whether they should continue on - waiting for the snapshot to be created based on the type of error - reported. - properties: - message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be logged, - and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if a snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in with the "ready_to_use" value returned from CSI - "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this - field will be filled with the "ready_to_use" value returned from the - CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, - this field will be set to "True". If not specified, it means the readiness - of a snapshot is unknown. - type: boolean - restoreSize: - description: restoreSize represents the complete size of the snapshot - in bytes. In dynamic snapshot creation case, this field will be filled - in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "size_bytes" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. When restoring a volume from - this snapshot, the size of the volume MUST NOT be smaller than the - restoreSize if it is specified, otherwise the restoration will fail. - If not specified, it indicates that the size is unknown. - type: string - type: object - required: - - spec - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: []