send/recv: open up additional stream feature flags

The docs for drr_versioninfo have marked the top 32 bits as "reserved"
since its introduction (illumos/illumos-gate@9e69d7d). There's no
indication of why they're reserved, so it seems uncontroversial to make
a lot more flags available.

I'm keeping the top eight reserved, and explicitly calling them out as
such, so we can extend the header further in the future if we run out of
flags or want to do some kind of change that isn't about feature flags.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <[email protected]>

include/sys/zfs_ioctl.h

@@ -70,10 +70,36 @@ extern "C" {
 #define	ZFS_ACLTYPE_POSIX		1
 #define	ZFS_ACLTYPE_NFSV4		2
 
+/*
+ * The drr_versioninfo field of the dmu_replay_record has the
+ * following layout:
+ *
+ *	64	56	48	40	32	24	16	8	0
+ *	+-------+-------+-------+-------+-------+-------+-------+-------+
+ *	|reserve|		  feature-flags			    |C|S|
+ *	+-------+-------+-------+-------+-------+-------+-------+-------+
+ *
+ * The low order two bits indicate the header type: SUBSTREAM (0x1)
+ * or COMPOUNDSTREAM (0x2).  Using two bits for this is historical:
+ * this field used to be a version number, where the two version types
+ * were 1 and 2.  Using two bits for this allows earlier versions of
+ * the code to be able to recognize send streams that don't use any
+ * of the features indicated by feature flags.
+ *
+ * The top 8 bits are reserved for future expansion. At time of writing there
+ * are no plans for these. If you want to use them, please reach out to the
+ * OpenZFS community, e.g., on GitHub or Slack.
+ */
+
 /*
  * Field manipulation macros for the drr_versioninfo field of the
  * send stream header.
  */
+#define	DMU_GET_STREAM_HDRTYPE(vi)	BF64_GET((vi), 0, 2)
+#define	DMU_SET_STREAM_HDRTYPE(vi, x)	BF64_SET((vi), 0, 2, x)
+
+#define	DMU_GET_FEATUREFLAGS(vi)	BF64_GET((vi), 2, 56)
+#define	DMU_SET_FEATUREFLAGS(vi, x)	BF64_SET((vi), 2, 56, x)
 
 /*
  * Header types for zfs send streams.
@@ -83,16 +109,9 @@ typedef enum drr_headertype {
 	DMU_COMPOUNDSTREAM = 0x2
 } drr_headertype_t;
 
-#define	DMU_GET_STREAM_HDRTYPE(vi)	BF64_GET((vi), 0, 2)
-#define	DMU_SET_STREAM_HDRTYPE(vi, x)	BF64_SET((vi), 0, 2, x)
-
-#define	DMU_GET_FEATUREFLAGS(vi)	BF64_GET((vi), 2, 30)
-#define	DMU_SET_FEATUREFLAGS(vi, x)	BF64_SET((vi), 2, 30, x)
-
 /*
  * Feature flags for zfs send streams (flags in drr_versioninfo)
  */
-
 #define	DMU_BACKUP_FEATURE_DEDUP		(1 << 0)
 #define	DMU_BACKUP_FEATURE_DEDUPPROPS		(1 << 1)
 #define	DMU_BACKUP_FEATURE_SA_SPILL		(1 << 2)
@@ -125,12 +144,6 @@ typedef enum drr_headertype {
  */
 #define	DMU_BACKUP_FEATURE_SWITCH_TO_LARGE_BLOCKS (1 << 27)
 /* flag #28 is reserved for a Nutanix feature */
-/*
- * flag #29 is the last unused bit. It is reserved to indicate a to-be-designed
- * extension to the stream format which will accomodate more feature flags.
- * If you need to add another feature flag, please reach out to the OpenZFS
- * community, e.g., on GitHub or Slack.
- */
 
 /*
  * Mask of all supported backup features
@@ -150,23 +163,6 @@ typedef enum dmu_send_resume_token_version {
 	ZFS_SEND_RESUME_TOKEN_VERSION = 1
 } dmu_send_resume_token_version_t;
 
-/*
- * The drr_versioninfo field of the dmu_replay_record has the
- * following layout:
- *
- *	64	56	48	40	32	24	16	8	0
- *	+-------+-------+-------+-------+-------+-------+-------+-------+
- *	|		reserved	|        feature-flags	    |C|S|
- *	+-------+-------+-------+-------+-------+-------+-------+-------+
- *
- * The low order two bits indicate the header type: SUBSTREAM (0x1)
- * or COMPOUNDSTREAM (0x2).  Using two bits for this is historical:
- * this field used to be a version number, where the two version types
- * were 1 and 2.  Using two bits for this allows earlier versions of
- * the code to be able to recognize send streams that don't use any
- * of the features indicated by feature flags.
- */
-
 #define	DMU_BACKUP_MAGIC 0x2F5bacbacULL
 
 /*
@@ -230,10 +226,6 @@ typedef enum dmu_send_resume_token_version {
 	((drro)->drr_raw_bonuslen != 0 ? \
 	(drro)->drr_raw_bonuslen : P2ROUNDUP((drro)->drr_bonuslen, 8))
 
-/*
- * zfs ioctl command structure
- */
-
 /* Header is used in C++ so can't forward declare untagged struct */
 struct drr_begin {
 	uint64_t drr_magic;
@@ -477,6 +469,10 @@ typedef enum zfs_case {
 	ZFS_CASE_MIXED
 } zfs_case_t;
 
+/*
+ * zfs ioctl command structure
+ */
+
 /*
  * Note: this struct must have the same layout in 32-bit and 64-bit, so
  * that 32-bit processes (like /sbin/zfs) can pass it to the 64-bit