Skip to content

Commit

Permalink
F #6492: Index PCI passthrough devices with bus
Browse files Browse the repository at this point in the history
If q35 machine type is detected the slot of the pci device is set to 0
and the bus to pci_id + 1.

Q35 models uses pcie-root-ports to attach PCI devices. Each PCI port is
selected by the bus parameter of the PCI address and it that does not accept a
slot number greater than 0.

Example:

A VM with 2 X710 VFs is defined OpenNebula as:

PCI=[
  ADDRESS="0000:44:0a:0",
  BUS="44",
  CLASS="0200",
  DEVICE="154c",
  DOMAIN="0000",
  FUNCTION="0",
  NUMA_NODE="0",
  PCI_ID="0",
  SHORT_ADDRESS="44:0a.0",
  SLOT="0a",
  VENDOR="8086",
  VM_ADDRESS="01:00.0",
  VM_BUS="0x01",
  VM_DOMAIN="0x0000",
  VM_FUNCTION="0",
  VM_SLOT="0000" ]

PCI=[
  ADDRESS="0000:44:0a:1",
  BUS="44",
  CLASS="0200",
  DEVICE="154c",
  DOMAIN="0000",
  FUNCTION="1",
  NUMA_NODE="0",
  PCI_ID="1",
  SHORT_ADDRESS="44:0a.1",
  SLOT="0a",
  VENDOR="8086",
  VM_ADDRESS="02:00.0",
  VM_BUS="0x02",
  VM_DOMAIN="0x0000",
  VM_FUNCTION="0",
  VM_SLOT="0000" ]

Each PCI VFs is attached to different pcie-root-port, selected with the
VM_BUS parameter:

00:02.0 PCI bridge: Red Hat, Inc. QEMU PCIe Root port
00:02.1 PCI bridge: Red Hat, Inc. QEMU PCIe Root port

The PCI topology is:

-[0000:00]-+-00.0  Intel Corporation 82G33/G31/P35/P31 Express DRAM Controller
           +-01.0  Cirrus Logic GD 5446
           +-02.0-[01]----00.0  Intel Corporation Ethernet Virtual Function 700 Series
           +-02.1-[02]----00.0  Intel Corporation Ethernet Virtual Function 700 Series
           +-02.2-[03-04]----00.0-[04]--
           +-02.3-[05]----00.0  Red Hat, Inc. Virtio network device
           +-02.4-[06]----00.0  Red Hat, Inc. Virtio SCSI
           +-02.5-[07]----00.0  Red Hat, Inc. QEMU XHCI Host Controller
           +-02.6-[08]----00.0  Red Hat, Inc. Virtio console
           +-02.7-[09]----00.0  Red Hat, Inc. Virtio memory balloon
           +-03.0-[0a]--
           +-03.1-[0b]--
           +-03.2-[0c]--
           +-03.3-[0d]--
           +-03.4-[0e]--
           +-03.5-[0f]--
           +-03.6-[10]--
           +-03.7-[11]--
           +-1f.0  Intel Corporation 82801IB (ICH9) LPC Interface Controller
           +-1f.2  Intel Corporation 82801IR/IO/IH (ICH9R/DO/DH) 6 port SATA Controller [AHCI mode]
           \-1f.3  Intel Corporation 82801I (ICH9 Family) SMBus Controller
  • Loading branch information
rsmontero committed Feb 8, 2024
1 parent 1b0a0a5 commit 4ce7340
Show file tree
Hide file tree
Showing 5 changed files with 43 additions and 13 deletions.
8 changes: 7 additions & 1 deletion include/HostSharePCI.h
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@ class HostSharePCI : public Template
* - VM_FUNCTION: 0
* - VM_ADDRESS: BUS:SLOT.0
*
* When the machine type is q35, SLOT needs to be 0, and devices are attached
* to a different bus. Use param slot_index to select this behavior
* - VM_SLOT: 0
* - VM_BUS: PCI_ID + 1
*
* Cleans internal attributes:
* - NUMA_NODE
* - UUID
Expand All @@ -139,10 +144,11 @@ class HostSharePCI : public Template
* @param pci_device to set the address in
* @param default_bus if not set in PCI attribute (PCI_PASSTHROUGH_BUS
* in oned.conf)
* @param bus_index when true devices uses slot = 0 and bus = pci_id + 1
* @return -1 if wrong bus 0 on success
*/
static int set_pci_address(VectorAttribute * pci_device, const std::string& dbus,
bool clean);
bool bus_index, bool clean);

private:
/**
Expand Down
18 changes: 16 additions & 2 deletions include/VirtualMachine.h
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ class VirtualMachine : public PoolObjectSQL
{
VectorAttribute * os = obj_template->get("OS");

if ( os == 0 )
if ( os == nullptr )
{
return;
}
Expand All @@ -362,14 +362,28 @@ class VirtualMachine : public PoolObjectSQL
{
VectorAttribute * os = obj_template->get("OS");

if ( os == 0 )
if ( os == nullptr )
{
return;
}

os->replace("INITRD", initrd);
};

bool test_machine_type(const std::string& machine_type) const
{
VectorAttribute * os = obj_template->get("OS");

if ( os == nullptr )
{
return false;
}

const std::string machine = os->vector_value("MACHINE");

return machine.find(machine_type) != std::string::npos;
}

// ------------------------------------------------------------------------
// Access to VM locations
// ------------------------------------------------------------------------
Expand Down
21 changes: 14 additions & 7 deletions src/host/HostSharePCI.cc
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ int HostSharePCI::get_pci_value(const char * name,
/* ------------------------------------------------------------------------*/

int HostSharePCI::set_pci_address(VectorAttribute * pci_device,
const string& dbus, bool clean)
const string& dbus, bool bus_index, bool clean)
{
string bus;
ostringstream oss;
Expand Down Expand Up @@ -553,21 +553,28 @@ int HostSharePCI::set_pci_address(VectorAttribute * pci_device,
return -1;
}

oss << showbase << internal << setfill('0') << hex << setw(4) << ibus;

pci_device->replace("VM_BUS", oss.str());

// --------------------- SLOT (PCI_ID +1) -----------------------
oss.str("");

pci_device->vector_value("PCI_ID", slot);

slot = slot + 1;

if ( bus_index )
{
ibus = slot;
slot = 0;
}

// Set PCI attributes
oss << showbase << internal << setfill('0') << hex << setw(4) << slot;

pci_device->replace("VM_SLOT", oss.str());

oss.str("");

oss << showbase << internal << setfill('0') << hex << setw(4) << ibus;

pci_device->replace("VM_BUS", oss.str());

// ------------------- ADDRESS (BUS:SLOT.0) ---------------------
oss.str("");

Expand Down
6 changes: 4 additions & 2 deletions src/vm/VirtualMachine.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3678,7 +3678,8 @@ int VirtualMachine::set_up_attach_nic(VirtualMachineTemplate * tmpl, string& err

nd.get_configuration_attribute("PCI_PASSTHROUGH_BUS", bus);

if ( HostSharePCI::set_pci_address(_new_nic.get(), bus, false) != 0 )
if ( HostSharePCI::set_pci_address(_new_nic.get(), bus,
test_machine_type("q35"), false) != 0 )
{
err = "Wrong BUS in PCI attribute";
return -1;
Expand Down Expand Up @@ -3828,7 +3829,8 @@ int VirtualMachine::attach_pci(VectorAttribute * vpci, string& err)

nd.get_configuration_attribute("PCI_PASSTHROUGH_BUS", bus);

if ( HostSharePCI::set_pci_address(_new_pci.get(), bus, false) != 0 )
if ( HostSharePCI::set_pci_address(_new_pci.get(), bus,
test_machine_type("q35"), false) != 0 )
{
err = "Wrong BUS in PCI attribute";
return -1;
Expand Down
3 changes: 2 additions & 1 deletion src/vm/VirtualMachineParser.cc
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,8 @@ int VirtualMachine::parse_pci(string& error_str, Template * tmpl)
return -1;
}

if ( HostSharePCI::set_pci_address(attr, default_bus, true) != 0 )
if ( HostSharePCI::set_pci_address(attr, default_bus,
test_machine_type("q35"), true) != 0 )
{
error_str = "Wrong BUS in PCI attribute";
return -1;
Expand Down

0 comments on commit 4ce7340

Please sign in to comment.