Overhaul block device interfaces

This reshapes how block devices (frontends) interact with their attached
backends.  Rather than associating a backend with a device when it is
created, backend and device are attached after the fact, when each has
had an opportunity to be initialized.

Through its attachment handle, the device is able to notify the backend
of new requests, but also query sizing parameters (previously specified
during device construction), pause and unpause request retrieval, and
(in the future) perform other tasks such as cache mode alteration.

The backend has a corresponding attachment handle through which it
retrieves pending requests from the device -- behavior which is largely
unchanged from the original structure.

Rather than store device-specific information required to issue request
completions to the guest, the device emulation is expecting to use a
`Tracking` structure which will store the completion data to be
retrieved using an ID injected into the Request, which is passed back
with its result when processed by the backend.  This tracking structure
also implements several generic USDT probes for tracing block events,
rather than requiring the use of per-device probes.

Cargo.toml

@@ -116,6 +116,7 @@ lazy_static = "1.4"
 libc = "0.2"
 mockall = "0.11"
 num_enum = "0.5.11"
+pin-project-lite = "0.2.13"
 proc-macro2 = "1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
 quote = "1.0"

bin/propolis-server/src/lib/config.rs

@@ -4,67 +4,8 @@
 
 //! Describes a server config which may be parsed from a TOML file.
 
-use std::num::NonZeroUsize;
-use std::sync::Arc;
-
-use propolis::block;
-use propolis::inventory;
 pub use propolis_server_config::*;
 
-pub fn create_backend_for_block(
-    config: &Config,
-    name: &str,
-    log: slog::Logger,
-) -> Result<(Arc<dyn block::Backend>, inventory::ChildRegister), ParseError> {
-    let entry = config.block_devs.get(name).ok_or_else(|| {
-        ParseError::KeyNotFound(name.to_string(), "block_dev".to_string())
-    })?;
-    blockdev_backend(entry, log)
-}
-
-fn blockdev_backend(
-    dev: &BlockDevice,
-    log: slog::Logger,
-) -> Result<(Arc<dyn block::Backend>, inventory::ChildRegister), ParseError> {
-    let opts = propolis::block::BackendOpts {
-        block_size: dev.opts.block_size,
-        read_only: dev.opts.read_only,
-        skip_flush: dev.opts.skip_flush,
-    };
-    match &dev.bdtype as &str {
-        "file" => {
-            let path = dev
-                .options
-                .get("path")
-                .ok_or_else(|| {
-                    ParseError::KeyNotFound(
-                        "path".to_string(),
-                        "options".to_string(),
-                    )
-                })?
-                .as_str()
-                .ok_or_else(|| {
-                    ParseError::AsError(
-                        "path".to_string(),
-                        "as_str".to_string(),
-                    )
-                })?;
-
-            let nworkers = NonZeroUsize::new(8).unwrap();
-            let be = propolis::block::FileBackend::create(
-                path, opts, nworkers, log,
-            )?;
-            let child =
-                inventory::ChildRegister::new(&be, Some(path.to_string()));
-
-            Ok((be, child))
-        }
-        _ => {
-            panic!("unrecognized block dev type {}!", dev.bdtype);
-        }
-    }
-}
-
 #[cfg(not(feature = "omicron-build"))]
 pub fn reservoir_decide(log: &slog::Logger) -> bool {
     // Automatically enable use of the memory reservoir (rather than transient

bin/propolis-server/src/lib/initializer.rs

@@ -335,9 +335,6 @@ impl<'a> MachineInitializer<'a> {
                         ..Default::default()
                     },
                     nworkers,
-                    self.log.new(
-                        slog::o!("component" => format!("file-{}", spec.path)),
-                    ),
                 )?;
 
                 let child =
@@ -363,13 +360,15 @@ impl<'a> MachineInitializer<'a> {
                 info!(self.log, "Creating in-memory disk backend";
                       "len" => bytes.len());
 
+                let nworkers = NonZeroUsize::new(8).unwrap();
                 let be = propolis::block::InMemoryBackend::create(
                     bytes,
                     propolis::block::BackendOpts {
                         block_size: Some(512),
                         read_only: Some(spec.readonly),
                         ..Default::default()
                     },
+                    nworkers,
                 )?;
 
                 let child = inventory::ChildRegister::new(
@@ -447,28 +446,26 @@ impl<'a> MachineInitializer<'a> {
                 )
             })?;
 
-            let be_info = backend.info();
             match device_interface {
                 DeviceInterface::Virtio => {
-                    let vioblk = virtio::PciVirtioBlock::new(0x100, be_info);
+                    let vioblk = virtio::PciVirtioBlock::new(0x100);
                     let id =
                         self.inv.register_instance(&vioblk, bdf.to_string())?;
                     let _ = self.inv.register_child(child, id).unwrap();
-                    backend.attach(vioblk.clone())?;
+                    block::attach(backend, vioblk.clone());
                     chipset.device().pci_attach(bdf, vioblk);
                 }
                 DeviceInterface::Nvme => {
                     let nvme = nvme::PciNvme::create(
                         name.to_string(),
-                        be_info,
                         self.log.new(
                             slog::o!("component" => format!("nvme-{}", name)),
                         ),
                     );
                     let id =
                         self.inv.register_instance(&nvme, bdf.to_string())?;
                     let _ = self.inv.register_child(child, id).unwrap();
-                    backend.attach(nvme.clone())?;
+                    block::attach(backend, nvme.clone());
                     chipset.device().pci_attach(bdf, nvme);
                 }
             };

bin/propolis-standalone/src/cidata.rs

@@ -98,6 +98,7 @@ pub(crate) fn build_cidata_be(
             read_only: Some(true),
             ..Default::default()
         },
+        std::num::NonZeroUsize::new(8).unwrap(),
     )
     .context("could not create block backend")
 }

bin/propolis-standalone/src/config.rs

@@ -65,7 +65,6 @@ pub fn block_backend(
                     parsed.workers.unwrap_or(DEFAULT_WORKER_COUNT),
                 )
                 .unwrap(),
-                log.clone(),
             )
             .unwrap();
 

bin/propolis-standalone/src/main.rs

@@ -872,12 +872,11 @@ fn setup_instance(
                 let (backend, creg) = config::block_backend(&config, dev, log);
                 let bdf = bdf.unwrap();
 
-                let info = backend.info();
-                let vioblk = hw::virtio::PciVirtioBlock::new(0x100, info);
+                let vioblk = hw::virtio::PciVirtioBlock::new(0x100);
                 let id = inv.register_instance(&vioblk, bdf.to_string())?;
                 let _be_id = inv.register_child(creg, id)?;
 
-                backend.attach(vioblk.clone() as Arc<dyn block::Device>)?;
+                block::attach(backend, vioblk.clone());
 
                 chipset.pci_attach(bdf, vioblk);
             }
@@ -902,14 +901,13 @@ fn setup_instance(
                     .as_str()
                     .unwrap()
                     .to_string();
-                let info = backend.info();
                 let log = log.new(slog::o!("dev" => format!("nvme-{}", name)));
-                let nvme = hw::nvme::PciNvme::create(dev_serial, info, log);
+                let nvme = hw::nvme::PciNvme::create(dev_serial, log);
 
                 let id = inv.register_instance(&nvme, bdf.to_string())?;
                 let _be_id = inv.register_child(creg, id)?;
 
-                backend.attach(nvme.clone())?;
+                block::attach(backend, nvme.clone());
 
                 chipset.pci_attach(bdf, nvme);
             }

lib/propolis/Cargo.toml

@@ -20,6 +20,7 @@ propolis_types.workspace = true
 usdt = { workspace = true, features = ["asm"] }
 tokio = { workspace = true, features = ["full"] }
 futures.workspace = true
+pin-project-lite.workspace = true
 anyhow.workspace = true
 rfb.workspace = true
 slog.workspace = true