ddt: add support for prefetching tables into the ARC

This change adds a new `zpool prefetch -t ddt $pool` command which
causes a pool's DDT to be loaded into the ARC. The primary goal is to
remove the need to "warm" a pool's cache before deduplication stops
slowing write performance. It may also provide a way to reload portions
of a DDT if they have been flushed due to inactivity.

Sponsored-by: iXsystems, Inc.
Sponsored-by: Catalogics, Inc.
Sponsored-by: Klara, Inc.

Co-authored-by: Will Andrews <[email protected]>
Co-authored-by: Allan Jude <[email protected]>
Co-authored-by: Don Brady <[email protected]>
Signed-off-by: Allan Jude <[email protected]>
Signed-off-by: Will Andrews <[email protected]>
Signed-off-by: Fred Weigel <[email protected]>
Signed-off-by: Rob Norris <[email protected]>
Signed-off-by: Don Brady <[email protected]>

cmd/zdb/zdb.c

@@ -1934,8 +1934,8 @@ dump_ddt(ddt_t *ddt, ddt_type_t type, ddt_class_t class)
 	(void) printf("%s: %llu entries, size %llu on disk, %llu in core\n",
 	    name,
 	    (u_longlong_t)count,
-	    (u_longlong_t)(dspace / count),
-	    (u_longlong_t)(mspace / count));
+	    (u_longlong_t)dspace,
+	    (u_longlong_t)mspace);
 
 	if (dump_opt['D'] < 3)
 		return;

cmd/zpool/zpool_main.c

@@ -32,7 +32,7 @@
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019, loli10K <[email protected]>
  * Copyright (c) 2021, Colm Buckley <[email protected]>
- * Copyright (c) 2021, Klara Inc.
+ * Copyright (c) 2021, 2023, Klara Inc.
  * Copyright [2021] Hewlett Packard Enterprise Development LP
  */
 
@@ -87,6 +87,7 @@ static int zpool_do_remove(int, char **);
 static int zpool_do_labelclear(int, char **);
 
 static int zpool_do_checkpoint(int, char **);
+static int zpool_do_prefetch(int, char **);
 
 static int zpool_do_list(int, char **);
 static int zpool_do_iostat(int, char **);
@@ -173,6 +174,7 @@ typedef enum {
 	HELP_LIST,
 	HELP_OFFLINE,
 	HELP_ONLINE,
+	HELP_PREFETCH,
 	HELP_REPLACE,
 	HELP_REMOVE,
 	HELP_INITIALIZE,
@@ -304,6 +306,7 @@ static zpool_command_t command_table[] = {
 	{ "labelclear",	zpool_do_labelclear,	HELP_LABELCLEAR		},
 	{ NULL },
 	{ "checkpoint",	zpool_do_checkpoint,	HELP_CHECKPOINT		},
+	{ "prefetch",	zpool_do_prefetch,	HELP_PREFETCH		},
 	{ NULL },
 	{ "list",	zpool_do_list,		HELP_LIST		},
 	{ "iostat",	zpool_do_iostat,	HELP_IOSTAT		},
@@ -395,6 +398,9 @@ get_usage(zpool_help_t idx)
 		return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 		    "[-T d|u] [pool] ... \n"
 		    "\t    [interval [count]]\n"));
+	case HELP_PREFETCH:
+		return (gettext("\tprefetch -t <type> [<type opts>] <pool>\n"
+		    "\t    -t ddt <pool>\n"));
 	case HELP_OFFLINE:
 		return (gettext("\toffline [--power]|[[-f][-t]] <pool> "
 		    "<device> ...\n"));
@@ -3694,6 +3700,72 @@ zpool_do_checkpoint(int argc, char **argv)
 
 #define	CHECKPOINT_OPT	1024
 
+/*
+ * zpool prefetch <type> [<type opts>] <pool>
+ *
+ * Prefetchs a particular type of data in the specified pool.
+ */
+int
+zpool_do_prefetch(int argc, char **argv)
+{
+	int c;
+	char *poolname;
+	char *typestr = NULL;
+	zpool_prefetch_type_t type;
+	zpool_handle_t *zhp;
+	int err = 0;
+
+	while ((c = getopt(argc, argv, "t:")) != -1) {
+		switch (c) {
+		case 't':
+			typestr = optarg;
+			break;
+		case ':':
+			(void) fprintf(stderr, gettext("missing argument for "
+			    "'%c' option\n"), optopt);
+			usage(B_FALSE);
+			break;
+		case '?':
+			(void) fprintf(stderr, gettext("invalid option '%c'\n"),
+			    optopt);
+			usage(B_FALSE);
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 1) {
+		(void) fprintf(stderr, gettext("missing pool name argument\n"));
+		usage(B_FALSE);
+	}
+
+	if (argc > 1) {
+		(void) fprintf(stderr, gettext("too many arguments\n"));
+		usage(B_FALSE);
+	}
+
+	poolname = argv[0];
+
+	argc--;
+	argv++;
+
+	if (strcmp(typestr, "ddt") == 0) {
+		type = ZPOOL_PREFETCH_DDT;
+	} else {
+		(void) fprintf(stderr, gettext("unsupported prefetch type\n"));
+		usage(B_FALSE);
+	}
+
+	if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
+		return (1);
+
+	err = zpool_prefetch(zhp, type);
+
+	zpool_close(zhp);
+
+	return (err);
+}
+
 /*
  * zpool import [-d dir] [-D]
  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
@@ -6316,6 +6388,7 @@ print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
 	case ZPOOL_PROP_EXPANDSZ:
 	case ZPOOL_PROP_CHECKPOINT:
 	case ZPOOL_PROP_DEDUPRATIO:
+	case ZPOOL_PROP_DEDUPCACHED:
 		if (value == 0)
 			(void) strlcpy(propval, "-", sizeof (propval));
 		else
@@ -8661,13 +8734,17 @@ print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
 }
 
 static void
-print_dedup_stats(nvlist_t *config)
+print_dedup_stats(zpool_handle_t *zhp, nvlist_t *config, boolean_t literal)
 {
 	ddt_histogram_t *ddh;
 	ddt_stat_t *dds;
 	ddt_object_t *ddo;
 	uint_t c;
-	char dspace[6], mspace[6];
+	/* Extra space provided for literal display */
+	char dspace[32], mspace[32], cspace[32];
+	uint64_t cspace_prop;
+	enum zfs_nicenum_format format;
+	zprop_source_t src;
 
 	/*
 	 * If the pool was faulted then we may not have been able to
@@ -8685,12 +8762,26 @@ print_dedup_stats(nvlist_t *config)
 		return;
 	}
 
-	zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
-	zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
-	(void) printf("DDT entries %llu, size %s on disk, %s in core\n",
+	/*
+	 * Squash cached size into in-core size to handle race.
+	 * Only include cached size if it is available.
+	 */
+	cspace_prop = MIN(zpool_get_prop_int(zhp, ZPOOL_PROP_DEDUPCACHED, &src),
+	    ddo->ddo_mspace);
+	format = literal ? ZFS_NICENUM_RAW : ZFS_NICENUM_1024;
+	zfs_nicenum_format(cspace_prop, cspace, sizeof (cspace), format);
+	zfs_nicenum_format(ddo->ddo_dspace, dspace, sizeof (dspace), format);
+	zfs_nicenum_format(ddo->ddo_mspace, mspace, sizeof (mspace), format);
+	(void) printf("DDT entries %llu, size %s on disk, %s in core",
 	    (u_longlong_t)ddo->ddo_count,
 	    dspace,
 	    mspace);
+	if (src != ZPROP_SRC_DEFAULT) {
+		(void) printf(", %s cached (%.02f%%)",
+		    cspace,
+		    (double)cspace_prop / (double)ddo->ddo_mspace * 100.0);
+	}
+	(void) printf("\n");
 
 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
 	    (uint64_t **)&dds, &c) == 0);
@@ -8726,6 +8817,10 @@ status_callback(zpool_handle_t *zhp, void *data)
 	uint_t c;
 	vdev_stat_t *vs;
 
+	/* If dedup stats were requested, also fetch dedupcached. */
+	if (cbp->cb_dedup_stats > 1)
+		zpool_add_propname(zhp, ZPOOL_DEDUPCACHED_PROP_NAME);
+
 	config = zpool_get_config(zhp, NULL);
 	reason = zpool_get_status(zhp, &msgid, &errata);
 
@@ -9207,7 +9302,7 @@ status_callback(zpool_handle_t *zhp, void *data)
 		}
 
 		if (cbp->cb_dedup_stats)
-			print_dedup_stats(config);
+			print_dedup_stats(zhp, config, cbp->cb_literal);
 	} else {
 		(void) printf(gettext("config: The configuration cannot be "
 		    "determined.\n"));
@@ -9281,7 +9376,8 @@ zpool_do_status(int argc, char **argv)
 			cmd = optarg;
 			break;
 		case 'D':
-			cb.cb_dedup_stats = B_TRUE;
+			if (++cb.cb_dedup_stats  > 2)
+				cb.cb_dedup_stats = 2;
 			break;
 		case 'e':
 			cb.cb_print_unhealthy = B_TRUE;
@@ -10991,6 +11087,19 @@ zpool_do_set(int argc, char **argv)
 		cb.cb_type = ZFS_TYPE_VDEV;
 	}
 
+	/* argv[1], when supplied, is vdev name */
+	if (argc == 2) {
+		if (!are_vdevs_in_pool(1, argv + 1, argv[0], &cb.cb_vdevs)) {
+			(void) fprintf(stderr, gettext(
+			    "cannot find '%s' in '%s': device not in pool\n"),
+			    argv[1], argv[0]);
+			return (EINVAL);
+		}
+		cb.cb_vdevs.cb_names = argv + 1;
+		cb.cb_vdevs.cb_names_count = 1;
+		cb.cb_type = ZFS_TYPE_VDEV;
+	}
+
 	error = for_each_pool(1, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
 	    B_FALSE, set_callback, &cb);
 

cmd/ztest.c

@@ -26,6 +26,7 @@
  * Copyright (c) 2014 Integros [integros.com]
  * Copyright 2017 Joyent, Inc.
  * Copyright (c) 2017, Intel Corporation.
+ * Copyright (c) 2023, Klara, Inc.
  */
 
 /*
@@ -446,6 +447,7 @@ ztest_func_t ztest_blake3;
 ztest_func_t ztest_fletcher;
 ztest_func_t ztest_fletcher_incr;
 ztest_func_t ztest_verify_dnode_bt;
+ztest_func_t ztest_pool_prefetch_ddt;
 
 static uint64_t zopt_always = 0ULL * NANOSEC;		/* all the time */
 static uint64_t zopt_incessant = 1ULL * NANOSEC / 10;	/* every 1/10 second */
@@ -501,6 +503,7 @@ static ztest_info_t ztest_info[] = {
 	ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
 	ZTI_INIT(ztest_fletcher_incr, 1, &zopt_rarely),
 	ZTI_INIT(ztest_verify_dnode_bt, 1, &zopt_sometimes),
+	ZTI_INIT(ztest_pool_prefetch_ddt, 1, &zopt_rarely),
 };
 
 #define	ZTEST_FUNCS	(sizeof (ztest_info) / sizeof (ztest_info_t))
@@ -6999,6 +7002,21 @@ ztest_fletcher_incr(ztest_ds_t *zd, uint64_t id)
 	}
 }
 
+void
+ztest_pool_prefetch_ddt(ztest_ds_t *zd, uint64_t id)
+{
+	(void) zd, (void) id;
+	spa_t *spa;
+
+	(void) pthread_rwlock_rdlock(&ztest_name_lock);
+	VERIFY0(spa_open(ztest_opts.zo_pool, &spa, FTAG));
+
+	ddt_prefetch_all(spa);
+
+	spa_close(spa, FTAG);
+	(void) pthread_rwlock_unlock(&ztest_name_lock);
+}
+
 static int
 ztest_set_global_vars(void)
 {

include/libzfs.h

@@ -458,6 +458,7 @@ _LIBZFS_H nvlist_t *zpool_get_config(zpool_handle_t *, nvlist_t **);
 _LIBZFS_H nvlist_t *zpool_get_features(zpool_handle_t *);
 _LIBZFS_H int zpool_refresh_stats(zpool_handle_t *, boolean_t *);
 _LIBZFS_H int zpool_get_errlog(zpool_handle_t *, nvlist_t **);
+_LIBZFS_H void zpool_add_propname(zpool_handle_t *, const char *);
 
 /*
  * Import and export functions
@@ -504,6 +505,8 @@ _LIBZFS_H int zpool_checkpoint(zpool_handle_t *);
 _LIBZFS_H int zpool_discard_checkpoint(zpool_handle_t *);
 _LIBZFS_H boolean_t zpool_is_draid_spare(const char *);
 
+_LIBZFS_H int zpool_prefetch(zpool_handle_t *, zpool_prefetch_type_t);
+
 /*
  * Basic handle manipulations.  These functions do not create or destroy the
  * underlying datasets, only the references to them.

include/libzfs_core.h

@@ -147,6 +147,9 @@ _LIBZFS_CORE_H int lzc_pool_checkpoint_discard(const char *);
 _LIBZFS_CORE_H int lzc_wait(const char *, zpool_wait_activity_t, boolean_t *);
 _LIBZFS_CORE_H int lzc_wait_tag(const char *, zpool_wait_activity_t, uint64_t,
     boolean_t *);
+
+_LIBZFS_CORE_H int lzc_pool_prefetch(const char *, zpool_prefetch_type_t);
+
 _LIBZFS_CORE_H int lzc_wait_fs(const char *, zfs_wait_activity_t, boolean_t *);
 
 _LIBZFS_CORE_H int lzc_set_bootenv(const char *, const nvlist_t *);

include/sys/arc.h

@@ -250,6 +250,16 @@ typedef struct arc_buf_info {
 	enum zio_compress	abi_l2arc_compress;
 } arc_buf_info_t;
 
+/*
+ * Flags returned by arc_cached; describes which part of the arc
+ * the block is cached in.
+ */
+#define	ARC_CACHED_EMBEDDED	(1U << 0)
+#define	ARC_CACHED_IN_L1	(1U << 1)
+#define	ARC_CACHED_IN_MRU	(1U << 2)
+#define	ARC_CACHED_IN_MFU	(1U << 3)
+#define	ARC_CACHED_IN_L2	(1U << 4)
+
 void arc_space_consume(uint64_t space, arc_space_type_t type);
 void arc_space_return(uint64_t space, arc_space_type_t type);
 boolean_t arc_is_metadata(arc_buf_t *buf);
@@ -310,6 +320,7 @@ zio_t *arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
 arc_prune_t *arc_add_prune_callback(arc_prune_func_t *func, void *priv);
 void arc_remove_prune_callback(arc_prune_t *p);
 void arc_freed(spa_t *spa, const blkptr_t *bp);
+int arc_cached(spa_t *spa, const blkptr_t *bp);
 
 void arc_flush(spa_t *spa, boolean_t retry);
 void arc_tempreserve_clear(uint64_t reserve);

include/sys/ddt.h

@@ -236,15 +236,17 @@ extern void ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total);
 
 extern uint64_t ddt_get_dedup_dspace(spa_t *spa);
 extern uint64_t ddt_get_pool_dedup_ratio(spa_t *spa);
+extern int ddt_get_pool_dedup_cached(spa_t *spa, uint64_t *psize);
 
 extern ddt_t *ddt_select(spa_t *spa, const blkptr_t *bp);
 extern void ddt_enter(ddt_t *ddt);
 extern void ddt_exit(ddt_t *ddt);
 extern void ddt_init(void);
 extern void ddt_fini(void);
 extern ddt_entry_t *ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add);
-extern void ddt_prefetch(spa_t *spa, const blkptr_t *bp);
 extern void ddt_remove(ddt_t *ddt, ddt_entry_t *dde);
+extern void ddt_prefetch(spa_t *spa, const blkptr_t *bp);
+extern void ddt_prefetch_all(spa_t *spa);
 
 extern boolean_t ddt_class_contains(spa_t *spa, ddt_class_t max_class,
     const blkptr_t *bp);

include/sys/ddt_impl.h

@@ -47,6 +47,7 @@ typedef struct {
 	    const ddt_key_t *ddk);
 	void (*ddt_op_prefetch)(objset_t *os, uint64_t object,
 	    const ddt_key_t *ddk);
+	void (*ddt_op_prefetch_all)(objset_t *os, uint64_t object);
 	int (*ddt_op_update)(objset_t *os, uint64_t object,
 	    const ddt_key_t *ddk, const ddt_phys_t *phys, size_t psize,
 	    dmu_tx_t *tx);

include/sys/dmu.h

@@ -505,6 +505,12 @@ void dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
 void dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
     dmu_tx_t *tx);
 
+/*
+ * Get an estimated cache size for an object. Caller must expect races.
+ */
+int dmu_object_cached_size(objset_t *os, uint64_t object,
+    uint64_t *l1sz, uint64_t *l2sz);
+
 void dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
     void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
     int compressed_size, int byteorder, dmu_tx_t *tx);
@@ -903,6 +909,8 @@ void dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
 void dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
 	uint64_t len, enum zio_priority pri);
 void dmu_prefetch_dnode(objset_t *os, uint64_t object, enum zio_priority pri);
+int dmu_prefetch_wait(objset_t *os, uint64_t object, uint64_t offset,
+    uint64_t size);
 
 typedef struct dmu_object_info {
 	/* All sizes are in bytes unless otherwise indicated. */