From 422d91589f17fb02b021962b6f6375b71a2497db Mon Sep 17 00:00:00 2001 From: zyno42 <83068959+zyno42@users.noreply.github.com> Date: Mon, 6 Sep 2021 17:37:14 +0200 Subject: [PATCH] Extend libtapasco with Monitor DMA Engine Implement suggestion of @jahofmann in: https://github.com/esa-tu-darmstadt/tapasco/issues/296#issuecomment-900076385 When a new device is created a dummy implementation of the DMA Engine is used that simply does nothing. The actual DMA Engine is initialized later when the access mode is changed from monitor to exclusive mode. This is necessary to prevent a monitoring application like `tapasco-debug` to allocate all DMA Buffers of the kernel driver. --- runtime/libtapasco/src/device.rs | 125 ++++++++++++++++++++++--------- runtime/libtapasco/src/dma.rs | 20 +++++ 2 files changed, 109 insertions(+), 36 deletions(-) diff --git a/runtime/libtapasco/src/device.rs b/runtime/libtapasco/src/device.rs index 48994898..34a0479a 100644 --- a/runtime/libtapasco/src/device.rs +++ b/runtime/libtapasco/src/device.rs @@ -20,7 +20,7 @@ use crate::allocator::{Allocator, DriverAllocator, GenericAllocator, VfioAllocator}; use crate::debug::DebugGenerator; -use crate::dma::{DMAControl, DirectDMA, DriverDMA, VfioDMA}; +use crate::dma::{DMAControl, DirectDMA, DriverDMA, MonitorDMA, VfioDMA}; use crate::dma_user_space::UserSpaceDMA; use crate::job::Job; use crate::pe::PEId; @@ -330,20 +330,9 @@ impl Device { if name == "pcie" { info!("Allocating the default of 4GB at 0x0 for a PCIe platform"); let mut dma_offset = 0; - let mut dma_interrupt_read = 0; - let mut dma_interrupt_write = 1; for comp in &s.platform { if comp.name == "PLATFORM_COMPONENT_DMA0" { dma_offset = comp.offset; - for v in &comp.interrupts { - if v.name == "READ" { - dma_interrupt_read = v.mapping as usize; - } else if v.name == "WRITE" { - dma_interrupt_write = v.mapping as usize; - } else { - trace!("Unknown DMA interrupt: {}.", v.name); - } - } } } if dma_offset == 0 { @@ -357,28 +346,7 @@ impl Device { allocator: Mutex::new(Box::new( GenericAllocator::new(0, 4 * 1024 * 1024 * 1024, 64).context(AllocatorError)?, )), - dma: Box::new( - UserSpaceDMA::new( - &tlkm_dma_file, - dma_offset as usize, - dma_interrupt_read, - dma_interrupt_write, - &platform, - settings - .get::("dma.read_buffer_size") - .context(ConfigError)?, - settings - .get::("dma.read_buffers") - .context(ConfigError)?, - settings - .get::("dma.write_buffer_size") - .context(ConfigError)?, - settings - .get::("dma.write_buffers") - .context(ConfigError)?, - ) - .context(DMAError)?, - ), + dma: Box::new(MonitorDMA {}), })); } else if name == "zynq" || (name == "zynqmp" && !zynqmp_vfio_mode) { info!("Using driver allocation for Zynq/ZynqMP based platform."); @@ -386,7 +354,7 @@ impl Device { allocator: Mutex::new(Box::new( DriverAllocator::new(&tlkm_dma_file).context(AllocatorError)?, )), - dma: Box::new(DriverDMA::new(&tlkm_dma_file)), + dma: Box::new(MonitorDMA {}), })); } else if name == "zynqmp" { info!("Using VFIO mode for ZynqMP based platform."); @@ -397,7 +365,7 @@ impl Device { allocator: Mutex::new(Box::new( VfioAllocator::new(&vfio_dev).context(AllocatorError)?, )), - dma: Box::new(VfioDMA::new(&tlkm_dma_file, &vfio_dev)), + dma: Box::new(MonitorDMA {}), })); } else { return Err(Error::DeviceType { name: name }); @@ -510,11 +478,96 @@ impl Device { self.access = access; if access == tlkm_access::TlkmAccessExclusive { + trace!("Re-initializing DMAEngine for exclusive access mode."); + + // Now re-initialize the DMAEngine. + // TODO: I don't know if this way of doing things is correct. + // First remove the MonitorDMA: + self.offchip_memory.clear(); + + // Currently falls back to PCIe and Zynq allocation using the default 4GB at 0x0. + // This will be replaced with proper dynamic initialization after the status core + // has been updated to contain the required information. + info!("Again using static memory allocation due to lack of dynamic data in the status core."); + let zynqmp_vfio_mode = true; + if self.name == "pcie" { + info!("Allocating the default of 4GB at 0x0 for a PCIe platform"); + let mut dma_offset = 0; + let mut dma_interrupt_read = 0; + let mut dma_interrupt_write = 1; + for comp in &self.status.platform { + if comp.name == "PLATFORM_COMPONENT_DMA0" { + dma_offset = comp.offset; + for v in &comp.interrupts { + if v.name == "READ" { + dma_interrupt_read = v.mapping as usize; + } else if v.name == "WRITE" { + dma_interrupt_write = v.mapping as usize; + } else { + trace!("Unknown DMA interrupt: {}.", v.name); + } + } + } + } + if dma_offset == 0 { + trace!("Could not find DMA engine."); + return Err(Error::DMAEngineMissing {}); + } + + self.offchip_memory.push(Arc::new(OffchipMemory { + allocator: Mutex::new(Box::new( + GenericAllocator::new(0, 4 * 1024 * 1024 * 1024, 64).context(AllocatorError)?, + )), + dma: Box::new( + UserSpaceDMA::new( + &self.tlkm_device_file, + dma_offset as usize, + dma_interrupt_read, + dma_interrupt_write, + &self.platform, + self.settings + .get::("dma.read_buffer_size") + .context(ConfigError)?, + self.settings + .get::("dma.read_buffers") + .context(ConfigError)?, + self.settings + .get::("dma.write_buffer_size") + .context(ConfigError)?, + self.settings + .get::("dma.write_buffers") + .context(ConfigError)?, + ) + .context(DMAError)?, + ), + })); + } else if self.name == "zynq" || (self.name == "zynqmp" && !zynqmp_vfio_mode) { + info!("Using driver allocation for Zynq/ZynqMP based platform."); + self.offchip_memory.push(Arc::new(OffchipMemory { + allocator: Mutex::new(Box::new( + DriverAllocator::new(&self.tlkm_device_file).context(AllocatorError)?, + )), + dma: Box::new(DriverDMA::new(&self.tlkm_device_file)), + })); + } else if self.name == "zynqmp" { + info!("Using VFIO mode for ZynqMP based platform."); + let vfio_dev = Arc::new(init_vfio(self.settings.clone()).context(VfioInitError)?); + self.offchip_memory.push(Arc::new(OffchipMemory { + allocator: Mutex::new(Box::new( + VfioAllocator::new(&vfio_dev).context(AllocatorError)?, + )), + dma: Box::new(VfioDMA::new(&self.tlkm_device_file, &vfio_dev)), + })); + } else { + return Err(Error::DeviceType { name: self.name.clone() }); + } + trace!("Access changed to exclusive, resetting all interrupts."); self.scheduler.reset_interrupts().context(SchedulerError)?; } trace!("Successfully acquired access."); + Ok(()) } diff --git a/runtime/libtapasco/src/dma.rs b/runtime/libtapasco/src/dma.rs index 502a35b3..229b0e8b 100644 --- a/runtime/libtapasco/src/dma.rs +++ b/runtime/libtapasco/src/dma.rs @@ -278,3 +278,23 @@ impl DMAControl for DirectDMA { Ok(()) } } + + +/// Use MonitorDMA for applications that only monitor other applications like tapasco-debug +/// This implementation simply does nothing +#[derive(Debug)] +pub struct MonitorDMA; + +impl DMAControl for MonitorDMA { + fn copy_to(&self, _data: &[u8], _ptr: DeviceAddress) -> Result<()> { + trace!("MonitorDMA copy_to: Doing nothing."); + + Ok(()) + } + + fn copy_from(&self, _ptr: DeviceAddress, _data: &mut [u8]) -> Result<()> { + trace!("MonitorDMA copy_to: Doing nothing."); + + Ok(()) + } +}