improving network capabilities

An update of NixOS/nixops#922

- Replaced `deployment.libvirtd.networks` option with a submodule to allow not
  only (libvirt) network names, but other networking types as well.
- Domain XML was adjusted accordingly to incorporate the parameters from the new
  `networks` submodule.
- Added the qemu guest agent to guests to allow for out-of-band
  communication (no need for network connectivity) with the hypervisor.
- Guest IP (for provisioning after guest has started) is no longer determined by
  waiting for the guest to get a DHCP lease in the hypervisor libvirt network.
  If the guest has a static IP, it won't ask for a DHCP lease. Also, for bridged
  networking, we probably will not have access to the DHCP server.
- Instead, the address of the first interface is retrieved from libvirt using
  the `VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT` method, which can now be done
  because of the newly added qemu guest agent.

nix/libvirtd.nix

@@ -82,9 +82,11 @@ in
  };
 
  deployment.libvirtd.networks = mkOption {
- default = [ "default" ];
- type = types.listOf types.str;
- description = "Names of libvirt networks to attach the VM to.";
+ type = types.listOf (types.submodule (import ./network-options.nix {
+ inherit lib;
+ }));
+ default = [{ source = "default"; type= "virtual"; }];
+ description = "Describe network interfaces.";
  };
 
  deployment.libvirtd.extraDevicesXML = mkOption {
@@ -142,6 +144,8 @@ in
  services.openssh.extraConfig = "UseDNS no";
 
  deployment.hasFastConnection = true;
+
+ services.qemuGuest.enable = true;
 };
 
 }

nix/network-options.nix

@@ -0,0 +1,23 @@
+{ lib } :
+
+with lib;
+{
+ options = {
+
+ source = mkOption {
+ type = types.str;
+ default = "default";
+ description = ''
+ '';
+ };
+
+ type = mkOption {
+ type = types.enum [ "bridge" "virtual" ];
+ default = "virtual";
+ description = ''
+ '';
+ };
+
+ };
+
+}

nixopsvirtd/backends/libvirtd.py

@@ -18,6 +18,29 @@
 # to prevent libvirt errors from appearing on screen, see
 # https://www.redhat.com/archives/libvirt-users/2017-August/msg00011.html
 
+
+class LibvirtdNetwork:
+
+ INTERFACE_TYPES = {
+ 'virtual': 'network',
+ 'bridge': 'bridge',
+ }
+
+ def __init__(self, **kwargs):
+ self.type = kwargs['type']
+ self.source = kwargs['source']
+
+ @property
+ def interface_type(self):
+ return self.INTERFACE_TYPES[self.type]
+
+ @classmethod
+ def from_xml(cls, x):
+ type = x.find("attr[@name='type']/string").get("value")
+ source = x.find("attr[@name='source']/string").get("value")
+ return cls(type=type, source=source)
+
+
 class LibvirtdDefinition(MachineDefinition):
  """Definition of a trivial machine."""
 
@@ -35,6 +58,9 @@ def __init__(self, xml, config):
  self.extra_devices = x.find("attr[@name='extraDevicesXML']/string").get("value")
  self.extra_domain = x.find("attr[@name='extraDomainXML']/string").get("value")
  self.headless = x.find("attr[@name='headless']/bool").get("value") == 'true'
+ self.image_dir = x.find("attr[@name='imageDir']/string")
+ if self.image_dir:
+ self.image_dir = self.image_dir.get("value")
  self.domain_type = x.find("attr[@name='domainType']/string").get("value")
  self.kernel = x.find("attr[@name='kernel']/string").get("value")
  self.initrd = x.find("attr[@name='initrd']/string").get("value")
@@ -43,17 +69,21 @@ def __init__(self, xml, config):
  self.uri = x.find("attr[@name='URI']/string").get("value")
 
  self.networks = [
- k.get("value")
- for k in x.findall("attr[@name='networks']/list/string")]
+ LibvirtdNetwork.from_xml(n)
+ for n in x.findall("attr[@name='networks']/list/*")
+ ]
+
+ print("%r" % self.networks)
+ # print("%r" % self.networks[0])
+ # print("%r" % self.networks[1])
+ print("len=%d" % len(self.networks))
  assert len(self.networks) > 0
 
 
 class LibvirtdState(MachineState):
  private_ipv4 = nixops.util.attr_property("privateIpv4", None)
  client_public_key = nixops.util.attr_property("libvirtd.clientPublicKey", None)
  client_private_key = nixops.util.attr_property("libvirtd.clientPrivateKey", None)
- primary_net = nixops.util.attr_property("libvirtd.primaryNet", None)
- primary_mac = nixops.util.attr_property("libvirtd.primaryMAC", None)
  domain_xml = nixops.util.attr_property("libvirtd.domainXML", None)
  disk_path = nixops.util.attr_property("libvirtd.diskPath", None)
  storage_volume_name = nixops.util.attr_property("libvirtd.storageVolume", None)
@@ -134,17 +164,9 @@ def address_to(self, m):
  def _vm_id(self):
  return "nixops-{0}-{1}".format(self.depl.uuid, self.name)
 
- def _generate_primary_mac(self):
- mac = [0x52, 0x54, 0x00,
- random.randint(0x00, 0x7f),
- random.randint(0x00, 0xff),
- random.randint(0x00, 0xff)]
- self.primary_mac = ':'.join(map(lambda x: "%02x" % x, mac))
-
  def create(self, defn, check, allow_reboot, allow_recreate):
  assert isinstance(defn, LibvirtdDefinition)
  self.set_common_state(defn)
- self.primary_net = defn.networks[0]
  self.storage_pool_name = defn.storage_pool_name
  self.uri = defn.uri
 
@@ -153,14 +175,13 @@ def create(self, defn, check, allow_reboot, allow_recreate):
  if self.conn.getLibVersion() < 1002007:
  raise Exception('libvirt 1.2.7 or newer is required at the target host')
 
- if not self.primary_mac:
- self._generate_primary_mac()
+ self.storage_pool_name = defn.storage_pool_name
 
  if not self.client_public_key:
  (self.client_private_key, self.client_public_key) = nixops.util.create_key_pair()
 
  if self.storage_volume_name is None:
- self._prepare_storage_volume()
+ self._prepare_storage_volume(defn)
  self.storage_volume_name = self.vol.name()
 
  self.domain_xml = self._make_domain_xml(defn)
@@ -178,7 +199,7 @@ def create(self, defn, check, allow_reboot, allow_recreate):
  self.start()
  return True
 
- def _prepare_storage_volume(self):
+ def _prepare_storage_volume(self, defn):
  self.logger.log("preparing disk image...")
  newEnv = copy.deepcopy(os.environ)
  newEnv["NIXOPS_LIBVIRTD_PUBKEY"] = self.client_public_key
@@ -196,27 +217,40 @@ def _prepare_storage_volume(self):
 
  self.logger.log("uploading disk image...")
  image_info = self._get_image_info(temp_disk_path)
- self._vol = self._create_volume(image_info['virtual-size'], image_info['actual-size'])
+ self._vol = self._create_volume(image_info['virtual-size'], image_info['actual-size'], defn.image_dir)
  self._upload_volume(temp_disk_path, image_info['actual-size'])
 
  def _get_image_info(self, filename):
  output = self._logged_exec(["qemu-img", "info", "--output", "json", filename], capture_stdout=True)
  return json.loads(output)
 
- def _create_volume(self, virtual_size, actual_size):
+ def _create_volume(self, virtual_size, actual_size, path=None):
+ # according to https://libvirt.org/formatstorage.html#StoragePoolTarget
+ # files should be created with rights depending on parent folder but 
+ # this doesn't seem true
+ # here I hardcode permission rights (BAD)
  xml = '''
  <volume>
  <name>{name}</name>
  <capacity>{virtual_size}</capacity>
  <allocation>{actual_size}</allocation>
  <target>
  <format type="qcow2"/>
+ <permissions>
+ <owner>1000</owner>
+ <group>100</group>
+ <mode>0744</mode>
+ <label>virt_image_t</label>
+ </permissions>
+ {eventual_path}
  </target>
  </volume>
  '''.format(
  name="{}.qcow2".format(self._vm_id()),
  virtual_size=virtual_size,
  actual_size=actual_size,
+ # eventual_path= "<path >%s</path>" % path if path else ""
+ eventual_path= ""
  )
  vol = self.pool.createXML(xml)
  self._vol = vol
@@ -243,19 +277,15 @@ def _get_qemu_executable(self):
  def _make_domain_xml(self, defn):
  qemu = self._get_qemu_executable()
 
- def maybe_mac(n):
- if n == self.primary_net:
- return '<mac address="' + self.primary_mac + '" />'
- else:
- return ""
-
  def iface(n):
  return "\n".join([
- ' <interface type="network">',
- maybe_mac(n),
- ' <source network="{0}"/>',
+ ' <interface type="{interface_type}">',
+ ' <source {interface_type}="{source}"/>',
  ' </interface>',
- ]).format(n)
+ ]).format(
+ interface_type=n.interface_type,
+ source=n.source,
+ )
 
  def _make_os(defn):
  return [
@@ -283,6 +313,10 @@ def _make_os(defn):
  ' <graphics type="vnc" port="-1" autoport="yes"/>' if not defn.headless else "",
  ' <input type="keyboard" bus="usb"/>',
  ' <input type="mouse" bus="usb"/>',
+ ' <channel type="unix">',
+ ' <target type="virtio" name="org.qemu.guest_agent.0"/>',
+ ' <address type="virtio-serial" controller="0" bus="0" port="1"/>',
+ ' </channel>',
  defn.extra_devices,
  ' </devices>',
  defn.extra_domain,
@@ -302,19 +336,39 @@ def _parse_ip(self):
  """
  return an ip v4
  """
- # alternative is VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE if qemu agent is available
- ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE, 0)
- if ifaces is None:
- self.log("Failed to get domain interfaces")
+
+ try:
+ ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT, 0)
+ # if ifaces is None:
+ # self.log("Failed to get domain interfaces")
+ # return
+ # print("The interface IP addresses:")
+ from ipaddress import ip_address
+ for (name, val) in ifaces.iteritems():
+ self.log("Parsing interface %s..." % name)
+
+ if val['addrs']:
+ for ipaddr in val['addrs']:
+ curaddr = ip_address(unicode(ipaddr['addr']))
+
+ if ipaddr['type'] == libvirt.VIR_IP_ADDR_TYPE_IPV4:
+ print(ipaddr['addr'] + " VIR_IP_ADDR_TYPE_IPV4")
+ if curaddr.is_loopback:
+ continue
+ self.success("Found address")
+ return ipaddr['addr']
+
+ else:
+ pass
+
+ except libvirt.libvirtError as e:
+ self.log(str(e))
  return
 
- for (name, val) in ifaces.iteritems():
- if val['addrs']:
- for ipaddr in val['addrs']:
- return ipaddr['addr']
+ return False
+
 
  def _wait_for_ip(self, prev_time):
- self.log_start("waiting for IP address to appear in DHCP leases...")
  while True:
  ip = self._parse_ip()
  if ip:
@@ -334,7 +388,6 @@ def _is_running(self):
  def start(self):
  assert self.vm_id
  assert self.domain_xml
- assert self.primary_net
  if self._is_running():
  self.log("connecting...")
  self.private_ipv4 = self._parse_ip()